diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 5577e496a57..00000000000 --- a/.coveragerc +++ /dev/null @@ -1,9 +0,0 @@ -[report] -omit = - src/blib2to3/* - tests/data/* - */site-packages/* - .tox/* - -[run] -relative_files = True diff --git a/.flake8 b/.flake8 index 7bc346a09c1..85f51cf9f05 100644 --- a/.flake8 +++ b/.flake8 @@ -1,6 +1,6 @@ [flake8] # B905 should be enabled when we drop support for 3.9 -ignore = E203, E266, E501, W503, B905, B907 +ignore = E203, E266, E501, E704, W503, B905, B907 # line length is intentionally set to 80 here because black uses Bugbear # See https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#line-length for more details max-line-length = 80 diff --git a/.git_archival.txt b/.git_archival.txt new file mode 100644 index 00000000000..8fb235d7045 --- /dev/null +++ b/.git_archival.txt @@ -0,0 +1,4 @@ +node: $Format:%H$ +node-date: $Format:%cI$ +describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$ +ref-names: $Format:%D$ diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000000..00a7b00c94e --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +.git_archival.txt export-subst diff --git a/.github/ISSUE_TEMPLATE/style_issue.md b/.github/ISSUE_TEMPLATE/style_issue.md index 2e4343a3527..a9ce85fd977 100644 --- a/.github/ISSUE_TEMPLATE/style_issue.md +++ b/.github/ISSUE_TEMPLATE/style_issue.md @@ -1,8 +1,8 @@ --- -name: Style issue -about: Help us improve the Black style +name: Code style issue +about: Help us improve the Black code style title: "" -labels: "T: design" +labels: "T: style" assignees: "" --- diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index b3e1f0b9024..a1804597d7d 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Grep CHANGES.md for PR number if: contains(github.event.pull_request.labels.*.name, 'skip news') != true diff --git a/.github/workflows/diff_shades.yml b/.github/workflows/diff_shades.yml index a126756f102..8d8be2550b0 100644 --- a/.github/workflows/diff_shades.yml +++ b/.github/workflows/diff_shades.yml @@ -19,14 +19,14 @@ jobs: matrix: ${{ steps.set-config.outputs.matrix }} steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: "*" + python-version: "3.11" - name: Install diff-shades and support dependencies run: | - python -m pip install click packaging urllib3 + python -m pip install 'click==8.1.3' packaging urllib3 python -m pip install https://github.com/ichard26/diff-shades/archive/stable.zip - name: Calculate run configuration & metadata @@ -44,7 +44,7 @@ jobs: HATCH_BUILD_HOOKS_ENABLE: "1" # Clang is less picky with the C code it's given than gcc (and may # generate faster binaries too). - CC: clang-12 + CC: clang-14 strategy: fail-fast: false matrix: @@ -52,19 +52,19 @@ jobs: steps: - name: Checkout this repository (full clone) - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: # The baseline revision could be rather old so a full clone is ideal. fetch-depth: 0 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: - python-version: "*" + python-version: "3.11" - name: Install diff-shades and support dependencies run: | python -m pip install https://github.com/ichard26/diff-shades/archive/stable.zip - python -m pip install click packaging urllib3 + python -m pip install 'click==8.1.3' packaging urllib3 # After checking out old revisions, this might not exist so we'll use a copy. cat scripts/diff_shades_gha_helper.py > helper.py git config user.name "diff-shades-gha" diff --git a/.github/workflows/diff_shades_comment.yml b/.github/workflows/diff_shades_comment.yml index 26d06090919..9b3b4b579da 100644 --- a/.github/workflows/diff_shades_comment.yml +++ b/.github/workflows/diff_shades_comment.yml @@ -12,8 +12,8 @@ jobs: comment: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: "*" @@ -33,7 +33,7 @@ jobs: - name: Try to find pre-existing PR comment if: steps.metadata.outputs.needs-comment == 'true' id: find-comment - uses: peter-evans/find-comment@81e2da3af01c92f83cb927cf3ace0e085617c556 + uses: peter-evans/find-comment@a54c31d7fa095754bfef525c0c8e5e5674c4b4b1 with: issue-number: ${{ steps.metadata.outputs.pr-number }} comment-author: "github-actions[bot]" @@ -41,7 +41,7 @@ jobs: - name: Create or update PR comment if: steps.metadata.outputs.needs-comment == 'true' - uses: peter-evans/create-or-update-comment@5adcb0bb0f9fb3f95ef05400558bdb3f329ee808 + uses: peter-evans/create-or-update-comment@23ff15729ef2fc348714a3bb66d2f655ca9066f2 with: comment-id: ${{ steps.find-comment.outputs.comment-id }} issue-number: ${{ steps.metadata.outputs.pr-number }} diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index fc94dea62d9..006991a16d8 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -21,10 +21,10 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up latest Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "*" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 04e30e727bd..ee858236fcf 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -16,16 +16,16 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -36,7 +36,7 @@ jobs: latest_non_release)" >> $GITHUB_ENV - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . platforms: linux/amd64,linux/arm64 @@ -47,7 +47,7 @@ jobs: if: ${{ github.event_name == 'release' && github.event.action == 'published' && !github.event.release.prerelease }} - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . platforms: linux/amd64,linux/arm64 @@ -58,7 +58,7 @@ jobs: if: ${{ github.event_name == 'release' && github.event.action == 'published' && github.event.release.prerelease }} - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 373e1500ee9..42a399fd0aa 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -22,13 +22,13 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 064d4745a53..2d016cef7a6 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,4 +1,4 @@ -name: Lint +name: Lint + format ourselves on: [push, pull_request] @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Assert PR target is main if: github.event_name == 'pull_request' && github.repository == 'psf/black' @@ -24,14 +24,14 @@ jobs: fi - name: Set up latest Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "*" - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install -e '.[d]' + python -m pip install -e '.' python -m pip install tox - name: Run pre-commit hooks diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index 6b3eb903d84..8e3eb67a10d 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -1,8 +1,12 @@ -name: Publish to PyPI +name: Build and publish on: release: types: [published] + pull_request: + push: + branches: + - main permissions: contents: read @@ -11,12 +15,13 @@ jobs: main: name: sdist + pure wheel runs-on: ubuntu-latest + if: github.event_name == 'release' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up latest Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "*" @@ -28,39 +33,65 @@ jobs: - name: Build wheel and source distributions run: python -m build - - name: Upload to PyPI via Twine + - if: github.event_name == 'release' + name: Upload to PyPI via Twine env: TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} run: twine upload --verbose -u '__token__' dist/* + generate_wheels_matrix: + name: generate wheels matrix + runs-on: ubuntu-latest + outputs: + include: ${{ steps.set-matrix.outputs.include }} + steps: + - uses: actions/checkout@v4 + - name: Install cibuildwheel and pypyp + run: | + pipx install cibuildwheel==2.15.0 + pipx install pypyp==1 + - name: generate matrix + if: github.event_name != 'pull_request' + run: | + { + cibuildwheel --print-build-identifiers --platform linux \ + | pyp 'json.dumps({"only": x, "os": "ubuntu-latest"})' \ + && cibuildwheel --print-build-identifiers --platform macos \ + | pyp 'json.dumps({"only": x, "os": "macos-latest"})' \ + && cibuildwheel --print-build-identifiers --platform windows \ + | pyp 'json.dumps({"only": x, "os": "windows-latest"})' + } | pyp 'json.dumps(list(map(json.loads, lines)))' > /tmp/matrix + env: + CIBW_ARCHS_LINUX: x86_64 + CIBW_ARCHS_MACOS: x86_64 arm64 + CIBW_ARCHS_WINDOWS: AMD64 + - name: generate matrix (PR) + if: github.event_name == 'pull_request' + run: | + { + cibuildwheel --print-build-identifiers --platform linux \ + | pyp 'json.dumps({"only": x, "os": "ubuntu-latest"})' + } | pyp 'json.dumps(list(map(json.loads, lines)))' > /tmp/matrix + env: + CIBW_BUILD: "cp38-* cp312-*" + CIBW_ARCHS_LINUX: x86_64 + - id: set-matrix + run: echo "include=$(cat /tmp/matrix)" | tee -a $GITHUB_OUTPUT + mypyc: - name: mypyc wheels (${{ matrix.name }}) + name: mypyc wheels ${{ matrix.only }} + needs: generate_wheels_matrix runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - include: - - os: ubuntu-latest - name: linux-x86_64 - - os: windows-2019 - name: windows-amd64 - - os: macos-11 - name: macos-x86_64 - macos_arch: "x86_64" - - os: macos-11 - name: macos-arm64 - macos_arch: "arm64" - - os: macos-11 - name: macos-universal2 - macos_arch: "universal2" + include: ${{ fromJson(needs.generate_wheels_matrix.outputs.include) }} steps: - - uses: actions/checkout@v3 - - - name: Build wheels via cibuildwheel - uses: pypa/cibuildwheel@v2.11.4 - env: - CIBW_ARCHS_MACOS: "${{ matrix.macos_arch }}" + - uses: actions/checkout@v4 + - uses: pypa/cibuildwheel@v2.16.2 + with: + only: ${{ matrix.only }} - name: Upload wheels as workflow artifacts uses: actions/upload-artifact@v3 @@ -68,7 +99,8 @@ jobs: name: ${{ matrix.name }}-mypyc-wheels path: ./wheelhouse/*.whl - - name: Upload wheels to PyPI via Twine + - if: github.event_name == 'release' + name: Upload wheels to PyPI via Twine env: TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} run: pipx run twine upload --verbose -u '__token__' wheelhouse/*.whl @@ -77,17 +109,19 @@ jobs: name: Update stable branch needs: [main, mypyc] runs-on: ubuntu-latest + if: github.event_name == 'release' permissions: contents: write steps: - name: Checkout stable branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: stable fetch-depth: 0 - - name: Update stable branch to release tag & push + - if: github.event_name == 'release' + name: Update stable branch to release tag & push run: | git reset --hard ${{ github.event.release.tag_name }} git push diff --git a/.github/workflows/release_tests.yml b/.github/workflows/release_tests.yml new file mode 100644 index 00000000000..192ba004f81 --- /dev/null +++ b/.github/workflows/release_tests.yml @@ -0,0 +1,56 @@ +name: Release tool CI + +on: + push: + paths: + - .github/workflows/release_tests.yml + - release.py + - release_tests.py + pull_request: + paths: + - .github/workflows/release_tests.yml + - release.py + - release_tests.py + +jobs: + build: + # We want to run on external PRs, but not on our own internal PRs as they'll be run + # by the push to the branch. Without this if check, checks are duplicated since + # internal PRs match both the push and pull_request events. + if: + github.event_name == 'push' || github.event.pull_request.head.repo.full_name != + github.repository + + name: Running python ${{ matrix.python-version }} on ${{matrix.os}} + runs-on: ${{ matrix.os }} + strategy: + matrix: + python-version: ["3.12"] + os: [macOS-latest, ubuntu-latest, windows-latest] + + steps: + - uses: actions/checkout@v4 + with: + # Give us all history, branches and tags + fetch-depth: 0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + + - name: Print Python Version + run: python --version --version && which python + + - name: Print Git Version + run: git --version && which git + + - name: Update pip, setuptools + wheels + run: | + python -m pip install --upgrade pip setuptools wheel + + - name: Run unit tests via coverage + print report + run: | + python -m pip install coverage + coverage run scripts/release_tests.py + coverage report --show-missing diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3ca2a469147..55359a23303 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,14 +31,14 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "pypy-3.7", "pypy-3.8"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "pypy-3.9"] os: [ubuntu-latest, macOS-latest, windows-latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -49,7 +49,8 @@ jobs: - name: Unit tests if: "!startsWith(matrix.python-version, 'pypy')" - run: tox -e ci-py -- -v --color=yes + run: + tox -e ci-py$(echo ${{ matrix.python-version }} | tr -d '.') -- -v --color=yes - name: Unit tests (pypy) if: "startsWith(matrix.python-version, 'pypy')" @@ -58,8 +59,10 @@ jobs: - name: Upload coverage to Coveralls # Upload coverage if we are on the main repository and # we're running on Linux (this action only supports Linux) - if: github.repository == 'psf/black' && matrix.os == 'ubuntu-latest' - uses: AndreMiras/coveralls-python-action@v20201129 + if: + github.repository == 'psf/black' && matrix.os == 'ubuntu-latest' && + !startsWith(matrix.python-version, 'pypy') + uses: AndreMiras/coveralls-python-action@8799c9f4443ac4201d2e2f2c725d577174683b99 with: github-token: ${{ secrets.GITHUB_TOKEN }} parallel: true @@ -72,9 +75,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Send finished signal to Coveralls - uses: AndreMiras/coveralls-python-action@v20201129 + uses: AndreMiras/coveralls-python-action@8799c9f4443ac4201d2e2f2c725d577174683b99 with: parallel-finished: true debug: true @@ -90,10 +93,10 @@ jobs: os: [ubuntu-latest, macOS-latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up latest Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "*" @@ -103,4 +106,4 @@ jobs: python -m pip install -e ".[uvloop]" - name: Format ourselves - run: python -m black --check src/ + run: python -m black --check . diff --git a/.github/workflows/upload_binary.yml b/.github/workflows/upload_binary.yml index 22535a64c67..06e55cfe93a 100644 --- a/.github/workflows/upload_binary.yml +++ b/.github/workflows/upload_binary.yml @@ -29,10 +29,10 @@ jobs: executable_mime: "application/x-mach-binary" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up latest Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "*" diff --git a/.gitignore b/.gitignore index 249499b135e..4a4f1b738ad 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ _build .DS_Store .vscode +.python-version docs/_static/pypi.svg .tox __pycache__ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a69fb645238..2896489d724 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ # Note: don't use this config for your own repositories. Instead, see # "Version control integration" in docs/integrations/source_version_control.md -exclude: ^(src/blib2to3/|profiling/|tests/data/) +exclude: ^(profiling/|tests/data/) repos: - repo: local hooks: @@ -12,7 +12,7 @@ repos: additional_dependencies: &version_check_dependencies [ commonmark==0.9.1, - pyyaml==5.4.1, + pyyaml==6.0.1, beautifulsoup4==4.9.3, ] @@ -29,38 +29,45 @@ repos: - id: isort - repo: https://github.com/pycqa/flake8 - rev: 4.0.1 + rev: 6.1.0 hooks: - id: flake8 additional_dependencies: - flake8-bugbear - flake8-comprehensions - flake8-simplify + exclude: ^src/blib2to3/ - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.991 + rev: v1.7.1 hooks: - id: mypy exclude: ^docs/conf.py + args: ["--config-file", "pyproject.toml"] additional_dependencies: - - types-dataclasses >= 0.1.3 - types-PyYAML - tomli >= 0.2.6, < 2.0.0 - - types-typed-ast >= 1.4.1 - - click >= 8.1.0 + - click >= 8.1.0, != 8.1.4, != 8.1.5 - packaging >= 22.0 - platformdirs >= 2.1.0 - pytest - hypothesis + - aiohttp >= 3.7.4 + - types-commonmark + - urllib3 + - hypothesmith - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.7.1 + rev: v3.0.3 hooks: - id: prettier exclude: \.github/workflows/diff_shades\.yml - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.4.0 hooks: - id: end-of-file-fixer - id: trailing-whitespace + +ci: + autoupdate_schedule: quarterly diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 137957045a6..54a03efe7a1 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -1,7 +1,10 @@ +# Note that we recommend using https://github.com/psf/black-pre-commit-mirror instead +# This will work about 2x as fast as using the hooks in this repository - id: black name: black description: "Black: The uncompromising Python code formatter" entry: black + stages: [pre-commit, pre-merge-commit, pre-push, manual] language: python minimum_pre_commit_version: 2.9.2 require_serial: true @@ -11,6 +14,7 @@ description: "Black: The uncompromising Python code formatter (with Jupyter Notebook support)" entry: black + stages: [pre-commit, pre-merge-commit, pre-push, manual] language: python minimum_pre_commit_version: 2.9.2 require_serial: true diff --git a/.readthedocs.yaml b/.readthedocs.yaml index fff2d6ed341..fa612668850 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -6,7 +6,7 @@ formats: build: os: ubuntu-22.04 tools: - python: "3.8" + python: "3.11" python: install: diff --git a/AUTHORS.md b/AUTHORS.md index ab3f30b8821..e0511bb9b7c 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -13,6 +13,7 @@ Maintained with: - [Richard Si](mailto:sichard26@gmail.com) - [Felix Hildén](mailto:felix.hilden@gmail.com) - [Batuhan Taskaya](mailto:batuhan@python.org) +- [Shantanu Jain](mailto:hauntsaninja@gmail.com) Multiple contributions by: diff --git a/CHANGES.md b/CHANGES.md index 2071eb3f800..d0c9e567457 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,51 +1,335 @@ # Change Log -## Unreleased +## 23.12.1 + +### Packaging + +- Fixed a bug that included dependencies from the `d` extra by default (#4108) + +## 23.12.0 + +### Highlights + +It's almost 2024, which means it's time for a new edition of _Black_'s stable style! +Together with this release, we'll put out an alpha release 24.1a1 showcasing the draft +2024 stable style, which we'll finalize in the January release. Please try it out and +[share your feedback](https://github.com/psf/black/issues/4042). + +This release (23.12.0) will still produce the 2023 style. Most but not all of the +changes in `--preview` mode will be in the 2024 stable style. + +### Stable style + +- Fix bug where `# fmt: off` automatically dedents when used with the `--line-ranges` + option, even when it is not within the specified line range. (#4084) +- Fix feature detection for parenthesized context managers (#4104) + +### Preview style + +- Prefer more equal signs before a break when splitting chained assignments (#4010) +- Standalone form feed characters at the module level are no longer removed (#4021) +- Additional cases of immediately nested tuples, lists, and dictionaries are now + indented less (#4012) +- Allow empty lines at the beginning of all blocks, except immediately before a + docstring (#4060) +- Fix crash in preview mode when using a short `--line-length` (#4086) +- Keep suites consisting of only an ellipsis on their own lines if they are not + functions or class definitions (#4066) (#4103) + +### Configuration + +- `--line-ranges` now skips _Black_'s internal stability check in `--safe` mode. This + avoids a crash on rare inputs that have many unformatted same-content lines. (#4034) + +### Packaging + +- Upgrade to mypy 1.7.1 (#4049) (#4069) +- Faster compiled wheels are now available for CPython 3.12 (#4070) + +### Integrations + +- Enable 3.12 CI (#4035) +- Build docker images in parallel (#4054) +- Build docker images with 3.12 (#4055) + +## 23.11.0 + +### Highlights + +- Support formatting ranges of lines with the new `--line-ranges` command-line option + (#4020) + +### Stable style + +- Fix crash on formatting bytes strings that look like docstrings (#4003) +- Fix crash when whitespace followed a backslash before newline in a docstring (#4008) +- Fix standalone comments inside complex blocks crashing Black (#4016) +- Fix crash on formatting code like `await (a ** b)` (#3994) +- No longer treat leading f-strings as docstrings. This matches Python's behaviour and + fixes a crash (#4019) + +### Preview style + +- Multiline dicts and lists that are the sole argument to a function are now indented + less (#3964) +- Multiline unpacked dicts and lists as the sole argument to a function are now also + indented less (#3992) +- In f-string debug expressions, quote types that are visible in the final string are + now preserved (#4005) +- Fix a bug where long `case` blocks were not split into multiple lines. Also enable + general trailing comma rules on `case` blocks (#4024) +- Keep requiring two empty lines between module-level docstring and first function or + class definition (#4028) +- Add support for single-line format skip with other comments on the same line (#3959) + +### Configuration + +- Consistently apply force exclusion logic before resolving symlinks (#4015) +- Fix a bug in the matching of absolute path names in `--include` (#3976) + +### Performance + +- Fix mypyc builds on arm64 on macOS (#4017) + +### Integrations + +- Black's pre-commit integration will now run only on git hooks appropriate for a code + formatter (#3940) + +## 23.10.1 + +### Highlights + +- Maintenance release to get a fix out for GitHub Action edge case (#3957) + +### Preview style + +- Fix merging implicit multiline strings that have inline comments (#3956) +- Allow empty first line after block open before a comment or compound statement (#3967) + +### Packaging + +- Change Dockerfile to hatch + compile black (#3965) + +### Integrations + +- The summary output for GitHub workflows is now suppressible using the `summary` + parameter. (#3958) +- Fix the action failing when Black check doesn't pass (#3957) + +### Documentation + +- It is known Windows documentation CI is broken + https://github.com/psf/black/issues/3968 + +## 23.10.0 + +### Stable style + +- Fix comments getting removed from inside parenthesized strings (#3909) + +### Preview style + +- Fix long lines with power operators getting split before the line length (#3942) +- Long type hints are now wrapped in parentheses and properly indented when split across + multiple lines (#3899) +- Magic trailing commas are now respected in return types. (#3916) +- Require one empty line after module-level docstrings. (#3932) +- Treat raw triple-quoted strings as docstrings (#3947) + +### Configuration + +- Fix cache versioning logic when `BLACK_CACHE_DIR` is set (#3937) + +### Parser + +- Fix bug where attributes named `type` were not accepted inside `match` statements + (#3950) +- Add support for PEP 695 type aliases containing lambdas and other unusual expressions + (#3949) + +### Output + +- Black no longer attempts to provide special errors for attempting to format Python 2 + code (#3933) +- Black will more consistently print stacktraces on internal errors in verbose mode + (#3938) + +### Integrations + +- The action output displayed in the job summary is now wrapped in Markdown (#3914) + +## 23.9.1 + +Due to various issues, the previous release (23.9.0) did not include compiled mypyc +wheels, which make Black significantly faster. These issues have now been fixed, and +this release should come with compiled wheels once again. + +There will be no wheels for Python 3.12 due to a bug in mypyc. We will provide 3.12 +wheels in a future release as soon as the mypyc bug is fixed. + +### Packaging + +- Upgrade to mypy 1.5.1 (#3864) + +### Performance + +- Store raw tuples instead of NamedTuples in Black's cache, improving performance and + decreasing the size of the cache (#3877) + +## 23.9.0 + +### Preview style + +- More concise formatting for dummy implementations (#3796) +- In stub files, add a blank line between a statement with a body (e.g an + `if sys.version_info > (3, x):`) and a function definition on the same level (#3862) +- Fix a bug whereby spaces were removed from walrus operators within subscript(#3823) + +### Configuration + +- Black now applies exclusion and ignore logic before resolving symlinks (#3846) + +### Performance + +- Avoid importing `IPython` if notebook cells do not contain magics (#3782) +- Improve caching by comparing file hashes as fallback for mtime and size (#3821) + +### _Blackd_ + +- Fix an issue in `blackd` with single character input (#3558) + +### Integrations + +- Black now has an + [official pre-commit mirror](https://github.com/psf/black-pre-commit-mirror). Swapping + `https://github.com/psf/black` to `https://github.com/psf/black-pre-commit-mirror` in + your `.pre-commit-config.yaml` will make Black about 2x faster (#3828) +- The `.black.env` folder specified by `ENV_PATH` will now be removed on the completion + of the GitHub Action (#3759) + +## 23.7.0 ### Highlights - +- Runtime support for Python 3.7 has been removed. Formatting 3.7 code will still be + supported until further notice (#3765) ### Stable style - +- Fix a bug where an illegal trailing comma was added to return type annotations using + PEP 604 unions (#3735) +- Fix several bugs and crashes where comments in stub files were removed or mishandled + under some circumstances (#3745) +- Fix a crash with multi-line magic comments like `type: ignore` within parentheses + (#3740) +- Fix error in AST validation when _Black_ removes trailing whitespace in a type comment + (#3773) ### Preview style - +- Implicitly concatenated strings used as function args are no longer wrapped inside + parentheses (#3640) +- Remove blank lines between a class definition and its docstring (#3692) ### Configuration - +- The `--workers` argument to _Black_ can now be specified via the `BLACK_NUM_WORKERS` + environment variable (#3743) +- `.pytest_cache`, `.ruff_cache` and `.vscode` are now excluded by default (#3691) +- Fix _Black_ not honouring `pyproject.toml` settings when running `--stdin-filename` + and the `pyproject.toml` found isn't in the current working directory (#3719) +- _Black_ will now error if `exclude` and `extend-exclude` have invalid data types in + `pyproject.toml`, instead of silently doing the wrong thing (#3764) ### Packaging - +- Upgrade mypyc from 0.991 to 1.3 (#3697) +- Remove patching of Click that mitigated errors on Python 3.6 with `LANG=C` (#3768) ### Parser - +- Add support for the new PEP 695 syntax in Python 3.12 (#3703) ### Performance - +- Speed up _Black_ significantly when the cache is full (#3751) +- Avoid importing `IPython` in a case where we wouldn't need it (#3748) ### Output - +- Use aware UTC datetimes internally, avoids deprecation warning on Python 3.12 (#3728) +- Change verbose logging to exactly mirror _Black_'s logic for source discovery (#3749) ### _Blackd_ - +- The `blackd` argument parser now shows the default values for options in their help + text (#3712) ### Integrations - +- Black is now tested with + [`PYTHONWARNDEFAULTENCODING = 1`](https://docs.python.org/3/library/io.html#io-encoding-warning) + (#3763) +- Update GitHub Action to display black output in the job summary (#3688) ### Documentation - +- Add a CITATION.cff file to the root of the repository, containing metadata on how to + cite this software (#3723) +- Update the _classes_ and _exceptions_ documentation in Developer reference to match + the latest code base (#3755) + +## 23.3.0 + +### Highlights + +This release fixes a longstanding confusing behavior in Black's GitHub action, where the +version of the action did not determine the version of Black being run (issue #3382). In +addition, there is a small bug fix around imports and a number of improvements to the +preview style. + +Please try out the +[preview style](https://black.readthedocs.io/en/stable/the_black_code_style/future_style.html#preview-style) +with `black --preview` and tell us your feedback. All changes in the preview style are +expected to become part of Black's stable style in January 2024. + +### Stable style + +- Import lines with `# fmt: skip` and `# fmt: off` no longer have an extra blank line + added when they are right after another import line (#3610) + +### Preview style + +- Add trailing commas to collection literals even if there's a comment after the last + entry (#3393) +- `async def`, `async for`, and `async with` statements are now formatted consistently + compared to their non-async version. (#3609) +- `with` statements that contain two context managers will be consistently wrapped in + parentheses (#3589) +- Let string splitters respect [East Asian Width](https://www.unicode.org/reports/tr11/) + (#3445) +- Now long string literals can be split after East Asian commas and periods (`、` U+3001 + IDEOGRAPHIC COMMA, `。` U+3002 IDEOGRAPHIC FULL STOP, & `，` U+FF0C FULLWIDTH COMMA) + besides before spaces (#3445) +- For stubs, enforce one blank line after a nested class with a body other than just + `...` (#3564) +- Improve handling of multiline strings by changing line split behavior (#1879) + +### Parser + +- Added support for formatting files with invalid type comments (#3594) + +### Integrations + +- Update GitHub Action to use the version of Black equivalent to action's version if + version input is not specified (#3543) +- Fix missing Python binary path in autoload script for vim (#3508) + +### Documentation + +- Document that only the most recent release is supported for security issues; + vulnerabilities should be reported through Tidelift (#3612) ## 23.1.0 @@ -66,8 +350,6 @@ versions separately. ### Stable style - - - Introduce the 2023 stable style, which incorporates most aspects of last year's preview style (#3418). Specific changes: - Enforce empty lines before classes and functions with sticky leading comments @@ -101,8 +383,6 @@ versions separately. ### Preview style - - - Format hex codes in unicode escape sequences in string literals (#2916) - Add parentheses around `if`-`else` expressions (#2278) - Improve performance on large expressions that contain many strings (#3467) @@ -133,15 +413,11 @@ versions separately. ### Configuration - - - Black now tries to infer its `--target-version` from the project metadata specified in `pyproject.toml` (#3219) ### Packaging - - - Upgrade mypyc from `0.971` to `0.991` so mypycified _Black_ can be built on armv7 (#3380) - This also fixes some crashes while using compiled Black with a debug build of @@ -154,8 +430,6 @@ versions separately. ### Output - - - Calling `black --help` multiple times will return the same help contents each time (#3516) - Verbose logging now shows the values of `pyproject.toml` configuration variables @@ -165,25 +439,18 @@ versions separately. ### Integrations - - - Move 3.11 CI to normal flow now that all dependencies support 3.11 (#3446) - Docker: Add new `latest_prerelease` tag automation to follow latest black alpha release on docker images (#3465) ### Documentation - - - Expand `vim-plug` installation instructions to offer more explicit options (#3468) ## 22.12.0 ### Preview style - - - Enforce empty lines before classes and functions with sticky leading comments (#3302) - Reformat empty and whitespace-only files as either an empty file (if no newline is present) or as a single newline character (if a newline is present) (#3348) @@ -196,8 +463,6 @@ versions separately. ### Configuration - - - Fix incorrectly applied `.gitignore` rules by considering the `.gitignore` location and the relative path to the target file (#3338) - Fix incorrectly ignoring `.gitignore` presence when more than one source directory is @@ -205,16 +470,12 @@ versions separately. ### Parser - - - Parsing support has been added for walruses inside generator expression that are passed as function args (for example, `any(match := my_re.match(text) for text in texts)`) (#3327). ### Integrations - - - Vim plugin: Optionally allow using the system installation of Black via `let g:black_use_virtualenv = 0`(#3309) @@ -717,7 +978,7 @@ and the first release covered by our new [`master`](https://github.com/psf/black/tree/main) branch with the [`main`](https://github.com/psf/black/tree/main) branch. Some additional changes in the source code were also made. (#2210) -- Sigificantly reorganized the documentation to make much more sense. Check them out by +- Significantly reorganized the documentation to make much more sense. Check them out by heading over to [the stable docs on RTD](https://black.readthedocs.io/en/stable/). (#2174) diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 00000000000..7ff0e3ca9bc --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,22 @@ +cff-version: 1.2.0 +title: "Black: The uncompromising Python code formatter" +message: >- + If you use this software, please cite it using the metadata from this file. +type: software +authors: + - family-names: Langa + given-names: Łukasz + - name: "contributors to Black" +repository-code: "https://github.com/psf/black" +url: "https://black.readthedocs.io/en/stable/" +abstract: >- + Black is the uncompromising Python code formatter. By using it, you agree to cede + control over minutiae of hand-formatting. In return, Black gives you speed, + determinism, and freedom from pycodestyle nagging about formatting. You will save time + and mental energy for more important matters. + + Blackened code looks the same regardless of the project you're reading. Formatting + becomes transparent after a while and you can focus on the content instead. + + Black makes code review faster by producing the smallest diffs possible. +license: MIT diff --git a/Dockerfile b/Dockerfile index 4e8f12f9798..ab961a2f491 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,18 @@ -FROM python:3-slim AS builder +FROM python:3.12-slim AS builder RUN mkdir /src COPY . /src/ ENV VIRTUAL_ENV=/opt/venv +ENV HATCH_BUILD_HOOKS_ENABLE=1 +# Install build tools to compile black + dependencies +RUN apt update && apt install -y build-essential git python3-dev RUN python -m venv $VIRTUAL_ENV -RUN . /opt/venv/bin/activate && pip install --no-cache-dir --upgrade pip setuptools wheel \ - # Install build tools to compile dependencies that don't have prebuilt wheels - && apt update && apt install -y git build-essential \ - && cd /src \ - && pip install --no-cache-dir .[colorama,d] +RUN python -m pip install --no-cache-dir hatch hatch-fancy-pypi-readme hatch-vcs +RUN . /opt/venv/bin/activate && pip install --no-cache-dir --upgrade pip setuptools \ + && cd /src && hatch build -t wheel \ + && pip install --no-cache-dir dist/*-cp*[colorama,d,uvloop] -FROM python:3-slim +FROM python:3.12-slim # copy only Python packages to limit the image size COPY --from=builder /opt/venv /opt/venv diff --git a/README.md b/README.md index b12ddfb1290..fb8170b626a 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Coverage Status License: MIT PyPI -Downloads +Downloads conda-forge Code style: black

@@ -38,7 +38,7 @@ Try it out now using the [Black Playground](https://black.vercel.app). Watch the ### Installation -_Black_ can be installed by running `pip install black`. It requires Python 3.7+ to run. +_Black_ can be installed by running `pip install black`. It requires Python 3.8+ to run. If you want to format Jupyter Notebooks, install with `pip install "black[jupyter]"`. If you can't wait for the latest _hotness_ and want to install from GitHub, use: @@ -68,7 +68,7 @@ projects, small and big. _Black_ has a comprehensive test suite, with efficient tests, and our own auto formatting and parallel Continuous Integration runner. Now that we have become stable, you should not expect large formatting changes in the future. Stylistic changes will mostly be responses to bug reports and support for new Python -syntax. For more information please refer to the +syntax. For more information please refer to [The Black Code Style](https://black.readthedocs.io/en/stable/the_black_code_style/index.html). Also, as a safety measure which slows down processing, _Black_ will check that the @@ -86,7 +86,7 @@ take previous formatting into account (see for exceptions). Our documentation covers the current _Black_ code style, but planned changes to it are -also documented. They're both worth taking a look: +also documented. They're both worth taking a look at: - [The _Black_ Code Style: Current style](https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html) - [The _Black_ Code Style: Future style](https://black.readthedocs.io/en/stable/the_black_code_style/future_style.html) @@ -137,8 +137,8 @@ SQLAlchemy, Poetry, PyPA applications (Warehouse, Bandersnatch, Pipenv, virtuale pandas, Pillow, Twisted, LocalStack, every Datadog Agent Integration, Home Assistant, Zulip, Kedro, OpenOA, FLORIS, ORBIT, WOMBAT, and many more. -The following organizations use _Black_: Facebook, Dropbox, KeepTruckin, Mozilla, Quora, -Duolingo, QuantumBlack, Tesla, Archer Aviation. +The following organizations use _Black_: Facebook, Dropbox, KeepTruckin, Lyft, Mozilla, +Quora, Duolingo, QuantumBlack, Tesla, Archer Aviation. Are we missing anyone? Let us know. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000000..47049501183 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,11 @@ +# Security Policy + +## Supported Versions + +Only the latest non-prerelease version is supported. + +## Security contact information + +To report a security vulnerability, please use the +[Tidelift security contact](https://tidelift.com/security). Tidelift will coordinate the +fix and disclosure. diff --git a/action.yml b/action.yml index 35705e99414..49cd6b06e4c 100644 --- a/action.yml +++ b/action.yml @@ -27,18 +27,42 @@ inputs: description: 'Python Version specifier (PEP440) - e.g. "21.5b1"' required: false default: "" + summary: + description: "Whether to add the output to the workflow summary" + required: false + default: true branding: color: "black" icon: "check-circle" runs: using: composite steps: - - run: | + - name: black + run: | + # Even when black fails, do not close the shell + set +e + if [ "$RUNNER_OS" == "Windows" ]; then - python $GITHUB_ACTION_PATH/action/main.py + runner="python" else - python3 $GITHUB_ACTION_PATH/action/main.py + runner="python3" + fi + + out=$(${runner} $GITHUB_ACTION_PATH/action/main.py) + exit_code=$? + + # Display the raw output in the step + echo "${out}" + + if [ "${{ inputs.summary }}" == "true" ]; then + # Display the Markdown output in the job summary + echo "\`\`\`python" >> $GITHUB_STEP_SUMMARY + echo "${out}" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY fi + + # Exit with the exit-code returned by Black + exit ${exit_code} env: # TODO: Remove once https://github.com/actions/runner/issues/665 is fixed. INPUT_OPTIONS: ${{ inputs.options }} diff --git a/action/main.py b/action/main.py index ff9d4112aed..c0af3930dbb 100644 --- a/action/main.py +++ b/action/main.py @@ -1,5 +1,6 @@ import os import shlex +import shutil import sys from pathlib import Path from subprocess import PIPE, STDOUT, run @@ -22,24 +23,57 @@ extra_deps = "[colorama,jupyter]" else: extra_deps = "[colorama]" -req = f"black{extra_deps}{version_specifier}" +if version_specifier: + req = f"black{extra_deps}{version_specifier}" +else: + describe_name = "" + with open(ACTION_PATH / ".git_archival.txt", encoding="utf-8") as fp: + for line in fp: + if line.startswith("describe-name: "): + describe_name = line[len("describe-name: ") :].rstrip() + break + if not describe_name: + print("::error::Failed to detect action version.", file=sys.stderr, flush=True) + sys.exit(1) + # expected format is one of: + # - 23.1.0 + # - 23.1.0-51-g448bba7 + if describe_name.count("-") < 2: + # the action's commit matches a tag exactly, install exact version from PyPI + req = f"black{extra_deps}=={describe_name}" + else: + # the action's commit does not match any tag, install from the local git repo + req = f".{extra_deps}" +print(f"Installing {req}...", flush=True) pip_proc = run( [str(ENV_BIN / "python"), "-m", "pip", "install", req], stdout=PIPE, stderr=STDOUT, encoding="utf-8", + cwd=ACTION_PATH, ) if pip_proc.returncode: print(pip_proc.stdout) - print("::error::Failed to install Black.", flush=True) + print("::error::Failed to install Black.", file=sys.stderr, flush=True) sys.exit(pip_proc.returncode) base_cmd = [str(ENV_BIN / "black")] if BLACK_ARGS: # TODO: remove after a while since this is deprecated in favour of SRC + OPTIONS. - proc = run([*base_cmd, *shlex.split(BLACK_ARGS)]) + proc = run( + [*base_cmd, *shlex.split(BLACK_ARGS)], + stdout=PIPE, + stderr=STDOUT, + encoding="utf-8", + ) else: - proc = run([*base_cmd, *shlex.split(OPTIONS), *shlex.split(SRC)]) - + proc = run( + [*base_cmd, *shlex.split(OPTIONS), *shlex.split(SRC)], + stdout=PIPE, + stderr=STDOUT, + encoding="utf-8", + ) +shutil.rmtree(ENV_PATH, ignore_errors=True) +print(proc.stdout) sys.exit(proc.returncode) diff --git a/autoload/black.vim b/autoload/black.vim index 5aec8725bd0..051fea05c3b 100644 --- a/autoload/black.vim +++ b/autoload/black.vim @@ -9,7 +9,7 @@ def strtobool(text): return True if text.lower() in ['n', 'no', 'f', 'false', 'off', '0']: return False - raise ValueError(f"{text} is not convertable to boolean") + raise ValueError(f"{text} is not convertible to boolean") class Flag(collections.namedtuple("FlagBase", "name, cast")): @property @@ -34,7 +34,7 @@ FLAGS = [ ] -def _get_python_binary(exec_prefix): +def _get_python_binary(exec_prefix, pyver): try: default = vim.eval("g:pymode_python").strip() except vim.error: @@ -43,7 +43,15 @@ def _get_python_binary(exec_prefix): return default if sys.platform[:3] == "win": return exec_prefix / 'python.exe' - return exec_prefix / 'bin' / 'python3' + bin_path = exec_prefix / "bin" + exec_path = (bin_path / f"python{pyver[0]}.{pyver[1]}").resolve() + if exec_path.exists(): + return exec_path + # It is possible that some environments may only have python3 + exec_path = (bin_path / f"python3").resolve() + if exec_path.exists(): + return exec_path + raise ValueError("python executable not found") def _get_pip(venv_path): if sys.platform[:3] == "win": @@ -67,8 +75,8 @@ def _initialize_black_env(upgrade=False): return True pyver = sys.version_info[:3] - if pyver < (3, 7): - print("Sorry, Black requires Python 3.7+ to run.") + if pyver < (3, 8): + print("Sorry, Black requires Python 3.8+ to run.") return False from pathlib import Path @@ -82,7 +90,7 @@ def _initialize_black_env(upgrade=False): _executable = sys.executable _base_executable = getattr(sys, "_base_executable", _executable) try: - executable = str(_get_python_binary(Path(sys.exec_prefix))) + executable = str(_get_python_binary(Path(sys.exec_prefix), pyver)) sys.executable = executable sys._base_executable = executable print(f'Creating a virtualenv in {virtualenv_path}...') diff --git a/docs/conf.py b/docs/conf.py index 7fc4f8f589e..52a849d06a4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -149,15 +149,13 @@ def make_pypi_svg(version: str) -> None: # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). -latex_documents = [ - ( - master_doc, - "black.tex", - "Documentation for Black", - "Łukasz Langa and contributors to Black", - "manual", - ) -] +latex_documents = [( + master_doc, + "black.tex", + "Documentation for Black", + "Łukasz Langa and contributors to Black", + "manual", +)] # -- Options for manual page output ------------------------------------------ @@ -172,17 +170,15 @@ def make_pypi_svg(version: str) -> None: # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) -texinfo_documents = [ - ( - master_doc, - "Black", - "Documentation for Black", - author, - "Black", - "The uncompromising Python code formatter", - "Miscellaneous", - ) -] +texinfo_documents = [( + master_doc, + "Black", + "Documentation for Black", + author, + "Black", + "The uncompromising Python code formatter", + "Miscellaneous", +)] # -- Options for Epub output ------------------------------------------------- @@ -210,7 +206,14 @@ def make_pypi_svg(version: str) -> None: autodoc_member_order = "bysource" +# -- sphinx-copybutton configuration ---------------------------------------- +copybutton_prompt_text = ( + r">>> |\.\.\. |> |\$ |\# | In \[\d*\]: | {2,5}\.\.\.: | {5,8}: " +) +copybutton_prompt_is_regexp = True +copybutton_remove_prompts = True + # -- Options for intersphinx extension --------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {"https://docs.python.org/3/": None} +intersphinx_mapping = {"": ("https://docs.python.org/3/", None)} diff --git a/docs/contributing/index.md b/docs/contributing/index.md index f56e57c9e90..3314c8eaa39 100644 --- a/docs/contributing/index.md +++ b/docs/contributing/index.md @@ -24,7 +24,8 @@ not very). This is deliberate. _Black_ aims to provide a consistent style and ta opportunities for arguing about style. Bug reports and fixes are always welcome! Please follow the -[issue template on GitHub](https://github.com/psf/black/issues/new) for best results. +[issue templates on GitHub](https://github.com/psf/black/issues/new/choose) for best +results. Before you suggest a new feature or configuration knob, ask yourself why you want it. If it enables better integration with some workflow, fixes an inconsistency, speeds things diff --git a/docs/contributing/issue_triage.md b/docs/contributing/issue_triage.md index 865a47935ed..89cfff76f7f 100644 --- a/docs/contributing/issue_triage.md +++ b/docs/contributing/issue_triage.md @@ -1,6 +1,6 @@ # Issue triage -Currently, _Black_ uses the issue tracker for bugs, feature requests, proposed design +Currently, _Black_ uses the issue tracker for bugs, feature requests, proposed style modifications, and general user support. Each of these issues have to be triaged so they can be eventually be resolved somehow. This document outlines the triaging process and also the current guidelines and recommendations. @@ -53,13 +53,13 @@ The lifecycle of a bug report or user support issue typically goes something lik - the issue has been fixed - duplicate of another pre-existing issue or is invalid -For enhancement, documentation, and design issues, the lifecycle looks very similar but +For enhancement, documentation, and style issues, the lifecycle looks very similar but the details are different: 1. _the issue is waiting for triage_ 2. **identified** - has been marked with a type label and other relevant labels 3. **discussion** - the merits of the suggested changes are currently being discussed, a - PR would be acceptable but would be at sigificant risk of being rejected + PR would be acceptable but would be at significant risk of being rejected 4. **accepted & awaiting PR** - it's been determined the suggested changes are OK and a PR would be welcomed (`S: accepted`) 5. **closed**: - the issue has been resolved, reasons include: diff --git a/docs/contributing/reference/reference_classes.rst b/docs/contributing/reference/reference_classes.rst index 3931e0e0072..dc615579e30 100644 --- a/docs/contributing/reference/reference_classes.rst +++ b/docs/contributing/reference/reference_classes.rst @@ -3,6 +3,9 @@ *Contents are subject to change.* +Black Classes +~~~~~~~~~~~~~~ + .. currentmodule:: black :class:`BracketTracker` @@ -18,6 +21,12 @@ :members: :special-members: __str__, __bool__ +:class:`RHSResult` +------------------------- + +.. autoclass:: black.lines.RHSResult + :members: + :class:`LinesBlock` ------------------------- @@ -43,6 +52,12 @@ .. autoclass:: black.comments.ProtoComment :members: +:class:`Mode` +--------------------- + +.. autoclass:: black.mode.Mode + :members: + :class:`Report` --------------- @@ -50,6 +65,20 @@ :members: :special-members: __str__ +:class:`Ok` +--------------- + +.. autoclass:: black.rusty.Ok + :show-inheritance: + :members: + +:class:`Err` +--------------- + +.. autoclass:: black.rusty.Err + :show-inheritance: + :members: + :class:`Visitor` ---------------- @@ -57,20 +86,122 @@ :show-inheritance: :members: -Enums -===== +:class:`StringTransformer` +---------------------------- -:class:`Changed` ----------------- +.. autoclass:: black.trans.StringTransformer + :show-inheritance: + :members: + +:class:`CustomSplit` +---------------------------- -.. autoclass:: black.Changed +.. autoclass:: black.trans.CustomSplit + :members: + +:class:`CustomSplitMapMixin` +----------------------------- + +.. autoclass:: black.trans.CustomSplitMapMixin :show-inheritance: :members: -:class:`Mode` ------------------ +:class:`StringMerger` +---------------------- + +.. autoclass:: black.trans.StringMerger + :show-inheritance: + :members: + +:class:`StringParenStripper` +----------------------------- + +.. autoclass:: black.trans.StringParenStripper + :show-inheritance: + :members: -.. autoclass:: black.Mode +:class:`BaseStringSplitter` +----------------------------- + +.. autoclass:: black.trans.BaseStringSplitter + :show-inheritance: + :members: + +:class:`StringSplitter` +----------------------------- + +.. autoclass:: black.trans.StringSplitter + :show-inheritance: + :members: + +:class:`StringParenWrapper` +----------------------------- + +.. autoclass:: black.trans.StringParenWrapper + :show-inheritance: + :members: + +:class:`StringParser` +----------------------------- + +.. autoclass:: black.trans.StringParser + :members: + +:class:`DebugVisitor` +------------------------ + +.. autoclass:: black.debug.DebugVisitor + :show-inheritance: + :members: + +:class:`Replacement` +------------------------ + +.. autoclass:: black.handle_ipynb_magics.Replacement + :members: + +:class:`CellMagic` +------------------------ + +.. autoclass:: black.handle_ipynb_magics.CellMagic + :members: + +:class:`CellMagicFinder` +------------------------ + +.. autoclass:: black.handle_ipynb_magics.CellMagicFinder + :show-inheritance: + :members: + +:class:`OffsetAndMagic` +------------------------ + +.. autoclass:: black.handle_ipynb_magics.OffsetAndMagic + :members: + +:class:`MagicFinder` +------------------------ + +.. autoclass:: black.handle_ipynb_magics.MagicFinder + :show-inheritance: + :members: + +:class:`Cache` +------------------------ + +.. autoclass:: black.cache.Cache + :show-inheritance: + :members: + +Enum Classes +~~~~~~~~~~~~~ + +Classes inherited from Python `Enum `_ class. + +:class:`Changed` +---------------- + +.. autoclass:: black.report.Changed :show-inheritance: :members: @@ -80,3 +211,24 @@ Enums .. autoclass:: black.WriteBack :show-inheritance: :members: + +:class:`TargetVersion` +---------------------- + +.. autoclass:: black.mode.TargetVersion + :show-inheritance: + :members: + +:class:`Feature` +------------------ + +.. autoclass:: black.mode.Feature + :show-inheritance: + :members: + +:class:`Preview` +------------------ + +.. autoclass:: black.mode.Preview + :show-inheritance: + :members: diff --git a/docs/contributing/reference/reference_exceptions.rst b/docs/contributing/reference/reference_exceptions.rst index aafe61e5017..ab46ebdb628 100644 --- a/docs/contributing/reference/reference_exceptions.rst +++ b/docs/contributing/reference/reference_exceptions.rst @@ -5,8 +5,14 @@ .. currentmodule:: black +.. autoexception:: black.trans.CannotTransform + .. autoexception:: black.linegen.CannotSplit -.. autoexception:: black.NothingChanged +.. autoexception:: black.brackets.BracketMatchError + +.. autoexception:: black.report.NothingChanged + +.. autoexception:: black.parsing.InvalidInput -.. autoexception:: black.InvalidInput +.. autoexception:: black.mode.Deprecated diff --git a/docs/contributing/reference/reference_functions.rst b/docs/contributing/reference/reference_functions.rst index 3bda5de1774..ebadf6975a7 100644 --- a/docs/contributing/reference/reference_functions.rst +++ b/docs/contributing/reference/reference_functions.rst @@ -94,18 +94,10 @@ Split functions Caching ------- -.. autofunction:: black.cache.filter_cached - .. autofunction:: black.cache.get_cache_dir .. autofunction:: black.cache.get_cache_file -.. autofunction:: black.cache.get_cache_info - -.. autofunction:: black.cache.read_cache - -.. autofunction:: black.cache.write_cache - Utilities --------- @@ -157,7 +149,7 @@ Utilities .. autofunction:: black.numerics.normalize_numeric_literal -.. autofunction:: black.linegen.normalize_prefix +.. autofunction:: black.comments.normalize_trailing_prefix .. autofunction:: black.strings.normalize_string_prefix @@ -165,8 +157,6 @@ Utilities .. autofunction:: black.linegen.normalize_invisible_parens -.. autofunction:: black.patch_click - .. autofunction:: black.nodes.preceding_leaf .. autofunction:: black.re_compile_maybe_verbose @@ -178,3 +168,5 @@ Utilities .. autofunction:: black.strings.sub_twice .. autofunction:: black.nodes.whitespace + +.. autofunction:: black.nodes.make_simple_prefix diff --git a/docs/contributing/reference/reference_summary.rst b/docs/contributing/reference/reference_summary.rst index f6ff4681557..c6163d897b6 100644 --- a/docs/contributing/reference/reference_summary.rst +++ b/docs/contributing/reference/reference_summary.rst @@ -3,8 +3,11 @@ Developer reference .. note:: - The documentation here is quite outdated and has been neglected. Many objects worthy - of inclusion aren't documented. Contributions are appreciated! + As of June 2023, the documentation of *Black classes* and *Black exceptions* + has been updated to the latest available version. + + The documentation of *Black functions* is quite outdated and has been neglected. Many + functions worthy of inclusion aren't documented. Contributions are appreciated! *Contents are subject to change.* diff --git a/docs/contributing/release_process.md b/docs/contributing/release_process.md index be9b08a6c82..c66ffae8ace 100644 --- a/docs/contributing/release_process.md +++ b/docs/contributing/release_process.md @@ -32,19 +32,29 @@ The 10,000 foot view of the release process is that you prepare a release PR and publish a [GitHub Release]. This triggers [release automation](#release-workflows) that builds all release artifacts and publishes them to the various platforms we publish to. +We now have a `scripts/release.py` script to help with cutting the release PRs. + +- `python3 scripts/release.py --help` is your friend. + - `release.py` has only been tested in Python 3.12 (so get with the times :D) + To cut a release: 1. Determine the release's version number - **_Black_ follows the [CalVer] versioning standard using the `YY.M.N` format** - So unless there already has been a release during this month, `N` should be `0` - Example: the first release in January, 2022 → `22.1.0` + - `release.py` will calculate this and log to stderr for you copy paste pleasure 1. File a PR editing `CHANGES.md` and the docs to version the latest changes + - Run `python3 scripts/release.py [--debug]` to generate most changes + - Sub headings in the template, if they have no bullet points need manual removal + _PR welcome to improve :D_ +1. If `release.py` fail manually edit; otherwise, yay, skip this step! 1. Replace the `## Unreleased` header with the version number 1. Remove any empty sections for the current release 1. (_optional_) Read through and copy-edit the changelog (eg. by moving entries, fixing typos, or rephrasing entries) - 1. Add a new empty template for the next release above - ([template below](#changelog-template)) + 1. Double-check that no changelog entries since the last release were put in the + wrong section (e.g., run `git diff CHANGES.md`) 1. Update references to the latest version in {doc}`/integrations/source_version_control` and {doc}`/usage_and_configuration/the_basics` @@ -61,6 +71,11 @@ To cut a release: description box 1. Publish the GitHub Release, triggering [release automation](#release-workflows) that will handle the rest +1. Once CI is done add + commit (git push - No review) a new empty template for the next + release to CHANGES.md _(Template is able to be copy pasted from release.py should we + fail)_ + 1. `python3 scripts/release.py --add-changes-template|-a [--debug]` + 1. Should that fail, please return to copy + paste 1. At this point, you're basically done. It's good practice to go and [watch and verify that all the release workflows pass][black-actions], although you will receive a GitHub notification should something fail. @@ -79,59 +94,6 @@ release is probably unnecessary. In the end, use your best judgement and ask other maintainers for their thoughts. ``` -### Changelog template - -Use the following template for a clean changelog after the release: - -``` -## Unreleased - -### Highlights - - - -### Stable style - - - -### Preview style - - - -### Configuration - - - -### Packaging - - - -### Parser - - - -### Performance - - - -### Output - - - -### _Blackd_ - - - -### Integrations - - - -### Documentation - - -``` - ## Release workflows All of _Black_'s release automation uses [GitHub Actions]. All workflows are therefore diff --git a/docs/contributing/the_basics.md b/docs/contributing/the_basics.md index 5fdcdd802bd..bc1680eecfd 100644 --- a/docs/contributing/the_basics.md +++ b/docs/contributing/the_basics.md @@ -12,7 +12,9 @@ example: ```console $ python3 -m venv .venv -$ source .venv/bin/activate +$ source .venv/bin/activate # activation for linux and mac +$ .venv\Scripts\activate # activation for windows + (.venv)$ pip install -r test_requirements.txt (.venv)$ pip install -e .[d] (.venv)$ pre-commit install @@ -30,8 +32,68 @@ the root of the black repo: # Optional Fuzz testing (.venv)$ tox -e fuzz + +# Format Black itself +(.venv)$ tox -e run_self +``` + +### Development + +Further examples of invoking the tests + +```console +# Run all of the above mentioned, in parallel +(.venv)$ tox --parallel=auto + +# Run tests on a specific python version +(.venv)$ tox -e py39 + +# pass arguments to pytest +(.venv)$ tox -e py -- --no-cov + +# print full tree diff, see documentation below +(.venv)$ tox -e py -- --print-full-tree + +# disable diff printing, see documentation below +(.venv)$ tox -e py -- --print-tree-diff=False ``` +### Testing + +All aspects of the _Black_ style should be tested. Normally, tests should be created as +files in the `tests/data/cases` directory. These files consist of up to three parts: + +- A line that starts with `# flags: ` followed by a set of command-line options. For + example, if the line is `# flags: --preview --skip-magic-trailing-comma`, the test + case will be run with preview mode on and the magic trailing comma off. The options + accepted are mostly a subset of those of _Black_ itself, except for the + `--minimum-version=` flag, which should be used when testing a grammar feature that + works only in newer versions of Python. This flag ensures that we don't try to + validate the AST on older versions and tests that we autodetect the Python version + correctly when the feature is used. For the exact flags accepted, see the function + `get_flags_parser` in `tests/util.py`. If this line is omitted, the default options + are used. +- A block of Python code used as input for the formatter. +- The line `# output`, followed by the output of _Black_ when run on the previous block. + If this is omitted, the test asserts that _Black_ will leave the input code unchanged. + +_Black_ has two pytest command-line options affecting test files in `tests/data/` that +are split into an input part, and an output part, separated by a line with`# output`. +These can be passed to `pytest` through `tox`, or directly into pytest if not using +`tox`. + +#### `--print-full-tree` + +Upon a failing test, print the full concrete syntax tree (CST) as it is after processing +the input ("actual"), and the tree that's yielded after parsing the output ("expected"). +Note that a test can fail with different output with the same CST. This used to be the +default, but now defaults to `False`. + +#### `--print-tree-diff` + +Upon a failing test, print the diff of the trees as described above. This is the +default. To turn it off pass `--print-tree-diff=False`. + ### News / Changelog Requirement `Black` has CI that will check for an entry corresponding to your PR in `CHANGES.md`. If @@ -62,7 +124,7 @@ If you make changes to docs, you can test they still build locally too. ```console (.venv)$ pip install -r docs/requirements.txt -(.venv)$ pip install [-e] .[d] +(.venv)$ pip install -e .[d] (.venv)$ sphinx-build -a -b html -W docs/ docs/_build/ ``` diff --git a/docs/faq.md b/docs/faq.md index bc9deccb756..c62e1b504b5 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -23,7 +23,7 @@ hooks, and scripting `unexpand` to run after applying _Black_. ## Does Black have an API? Not yet. _Black_ is fundamentally a command line tool. Many -[integrations](integrations/index.rst) are provided, but a Python interface is not one +[integrations](/integrations/index.md) are provided, but a Python interface is not one of them. A simple API is being [planned](https://github.com/psf/black/issues/779) though. @@ -39,7 +39,7 @@ other tools, such as `# noqa`, may be moved by _Black_. See below for more detai ## How stable is Black's style? Stable. _Black_ aims to enforce one style and one style only, with some room for -pragmatism. See [The Black Code Style](the_black_code_style/index.rst) for more details. +pragmatism. See [The Black Code Style](the_black_code_style/index.md) for more details. Starting in 2022, the formatting output will be stable for the releases made in the same year (other than unintentional bugs). It is possible to opt-in to the latest formatting @@ -57,8 +57,8 @@ _Black_ is timid about formatting Jupyter Notebooks. Cells containing any of the following will not be formatted: - automagics (e.g. `pip install black`) -- non-Python cell magics (e.g. `%%writeline`). These can be added with the flag - `--python-cell-magics`, e.g. `black --python-cell-magics writeline hello.ipynb`. +- non-Python cell magics (e.g. `%%writefile`). These can be added with the flag + `--python-cell-magics`, e.g. `black --python-cell-magics writefile hello.ipynb`. - multiline magics, e.g.: ```python @@ -86,7 +86,7 @@ disabled-by-default counterpart W504. E203 should be disabled while changes are ## Which Python versions does Black support? -Currently the runtime requires Python 3.7-3.11. Formatting is supported for files +Currently the runtime requires Python 3.8-3.11. Formatting is supported for files containing syntax from Python 3.3 to 3.11. We promise to support at least all Python versions that have not reached their end of life. This is the case for both running _Black_ and formatting code. @@ -95,7 +95,7 @@ Support for formatting Python 2 code was removed in version 22.0. While we've ma plans to stop supporting older Python 3 minor versions immediately, their support might also be removed some time in the future without a deprecation period. -Runtime support for 3.6 was removed in version 22.10.0. +Runtime support for 3.7 was removed in version 23.7.0. ## Why does my linter or typechecker complain after I format my code? @@ -107,7 +107,7 @@ codebase with _Black_. ## Can I run Black with PyPy? -Yes, there is support for PyPy 3.7 and higher. +Yes, there is support for PyPy 3.8 and higher. ## Why does Black not detect syntax errors in my code? diff --git a/docs/getting_started.md b/docs/getting_started.md index 33fb2f978bb..15b7646a509 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -16,7 +16,7 @@ Also, you can try out _Black_ online for minimal fuss on the ## Installation -_Black_ can be installed by running `pip install black`. It requires Python 3.7+ to run. +_Black_ can be installed by running `pip install black`. It requires Python 3.8+ to run. If you want to format Jupyter Notebooks, install with `pip install "black[jupyter]"`. If you can't wait for the latest _hotness_ and want to install from GitHub, use: diff --git a/docs/guides/introducing_black_to_your_project.md b/docs/guides/introducing_black_to_your_project.md index 9ae40a1928e..3927eb1a386 100644 --- a/docs/guides/introducing_black_to_your_project.md +++ b/docs/guides/introducing_black_to_your_project.md @@ -18,7 +18,8 @@ previous revision that modified those lines. So when migrating your project's code style to _Black_, reformat everything and commit the changes (preferably in one massive commit). Then put the full 40 characters commit -identifier(s) into a file. +identifier(s) into a file usually called `.git-blame-ignore-revs` at the root of your +project directory. ```text # Migrate code style to Black @@ -46,7 +47,6 @@ $ git config blame.ignoreRevsFile .git-blame-ignore-revs **The one caveat is that some online Git-repositories like GitLab do not yet support ignoring revisions using their native blame UI.** So blame information will be cluttered with a reformatting commit on those platforms. (If you'd like this feature, there's an -open issue for [GitLab](https://gitlab.com/gitlab-org/gitlab/-/issues/31423)). This is -however supported by -[GitHub](https://docs.github.com/en/repositories/working-with-files/using-files/viewing-a-file#ignore-commits-in-the-blame-view), -currently in beta. +open issue for [GitLab](https://gitlab.com/gitlab-org/gitlab/-/issues/31423)). +[GitHub supports `.git-blame-ignore-revs`](https://docs.github.com/en/repositories/working-with-files/using-files/viewing-a-file#ignore-commits-in-the-blame-view) +by default in blame views however. diff --git a/docs/guides/using_black_with_other_tools.md b/docs/guides/using_black_with_other_tools.md index 9356caaf0bd..22c641a7420 100644 --- a/docs/guides/using_black_with_other_tools.md +++ b/docs/guides/using_black_with_other_tools.md @@ -51,9 +51,9 @@ line_length = 88 #### Why those options above? -_Black_ wraps imports that surpass `line-length` by moving identifiers into their own -indented line. If that still doesn't fit the bill, it will put all of them in separate -lines and put a trailing comma. A more detailed explanation of this behaviour can be +_Black_ wraps imports that surpass `line-length` by moving identifiers onto separate +lines and by adding a trailing comma after each. A more detailed explanation of this +behaviour can be [found here](../the_black_code_style/current_style.md#how-black-wraps-lines). isort's default mode of wrapping imports that extend past the `line_length` limit is @@ -173,7 +173,7 @@ limit of `88`, _Black_'s default. This explains `max-line-length = 88`. ```ini [flake8] max-line-length = 88 -extend-ignore = E203 +extend-ignore = E203, E704 ``` diff --git a/docs/index.md b/docs/index.md index 9d0db465022..49a44ecca5a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -21,7 +21,7 @@ Try it out now using the [Black Playground](https://black.vercel.app). *Black* is [successfully used](https://github.com/psf/black#used-by) by many projects, small and big. *Black* has a comprehensive test suite, with efficient parallel tests, our own auto formatting and parallel Continuous Integration runner. -Now that we have become stable, you should not expect large formatting to changes in +Now that we have become stable, you should not expect large changes to formatting in the future. Stylistic changes will mostly be responses to bug reports and support for new Python syntax. diff --git a/docs/integrations/editors.md b/docs/integrations/editors.md index 74c6a283ab8..7d056160fcb 100644 --- a/docs/integrations/editors.md +++ b/docs/integrations/editors.md @@ -10,16 +10,26 @@ Options include the following: ## PyCharm/IntelliJ IDEA -There are three different ways you can use _Black_ from PyCharm: +There are several different ways you can use _Black_ from PyCharm: -1. As local server using the BlackConnect plugin -1. As external tool -1. As file watcher +1. Using the built-in _Black_ integration (PyCharm 2023.2 and later). This option is the + simplest to set up. +1. As local server using the BlackConnect plugin. This option formats the fastest. It + spins up {doc}`Black's HTTP server `, to + avoid the startup cost on subsequent formats. +1. As external tool. +1. As file watcher. -The first option is the simplest to set up and formats the fastest (by spinning up -{doc}`Black's HTTP server `, avoiding the -startup cost on subsequent formats), but if you would prefer to not install a -third-party plugin or blackd's extra dependencies, the other two are also great options. +### Built-in _Black_ integration + +1. Install `black`. + + ```console + $ pip install black + ``` + +1. Go to `Preferences or Settings -> Tools -> Black` and configure _Black_ to your + liking. ### As local server @@ -226,7 +236,7 @@ Configuration: #### Installation -This plugin **requires Vim 7.0+ built with Python 3.7+ support**. It needs Python 3.7 to +This plugin **requires Vim 7.0+ built with Python 3.8+ support**. It needs Python 3.8 to be able to run _Black_ inside the Vim process which is much faster than calling an external command. @@ -278,8 +288,8 @@ $ git checkout origin/stable -b stable ##### Arch Linux On Arch Linux, the plugin is shipped with the -[`python-black`](https://archlinux.org/packages/community/any/python-black/) package, so -you can start using it in Vim after install with no additional setup. +[`python-black`](https://archlinux.org/packages/extra/any/python-black/) package, so you +can start using it in Vim after install with no additional setup. ##### Vim 8 Native Plugin Management @@ -334,60 +344,6 @@ To run _Black_ on a key press (e.g. F9 below), add this: nnoremap :Black ``` -#### Troubleshooting - -**How to get Vim with Python 3.6?** On Ubuntu 17.10 Vim comes with Python 3.6 by -default. On macOS with Homebrew run: `brew install vim`. When building Vim from source, -use: `./configure --enable-python3interp=yes`. There's many guides online how to do -this. - -**I get an import error when using _Black_ from a virtual environment**: If you get an -error message like this: - -```text -Traceback (most recent call last): - File "", line 63, in - File "/home/gui/.vim/black/lib/python3.7/site-packages/black.py", line 45, in - from typed_ast import ast3, ast27 - File "/home/gui/.vim/black/lib/python3.7/site-packages/typed_ast/ast3.py", line 40, in - from typed_ast import _ast3 -ImportError: /home/gui/.vim/black/lib/python3.7/site-packages/typed_ast/_ast3.cpython-37m-x86_64-linux-gnu.so: undefined symbool: PyExc_KeyboardInterrupt -``` - -Then you need to install `typed_ast` directly from the source code. The error happens -because `pip` will download [Python wheels](https://pythonwheels.com/) if they are -available. Python wheels are a new standard of distributing Python packages and packages -that have Cython and extensions written in C are already compiled, so the installation -is much more faster. The problem here is that somehow the Python environment inside Vim -does not match with those already compiled C extensions and these kind of errors are the -result. Luckily there is an easy fix: installing the packages from the source code. - -The package that causes problems is: - -- [typed-ast](https://pypi.org/project/typed-ast/) - -Now remove those two packages: - -```console -$ pip uninstall typed-ast -y -``` - -And now you can install them with: - -```console -$ pip install --no-binary :all: typed-ast -``` - -The C extensions will be compiled and now Vim's Python environment will match. Note that -you need to have the GCC compiler and the Python development files installed (on -Ubuntu/Debian do `sudo apt-get install build-essential python3-dev`). - -If you later want to update _Black_, you should do it like this: - -```console -$ pip install -U black --no-binary typed-ast -``` - ### With ALE 1. Install [`ale`](https://github.com/dense-analysis/ale) @@ -435,7 +391,7 @@ close and reopen your File, _Black_ will be done with its job. - Use the [Python extension](https://marketplace.visualstudio.com/items?itemName=ms-python.python) - ([instructions](https://code.visualstudio.com/docs/python/editing#_formatting)). + ([instructions](https://code.visualstudio.com/docs/python/formatting)). - Alternatively the pre-release [Black Formatter](https://marketplace.visualstudio.com/items?itemName=ms-python.black-formatter) @@ -443,9 +399,10 @@ close and reopen your File, _Black_ will be done with its job. server for Black. Formatting is much more responsive using this extension, **but the minimum supported version of Black is 22.3.0**. -## SublimeText 3 +## SublimeText -Use [sublack plugin](https://github.com/jgirardet/sublack). +For SublimeText 3, use [sublack plugin](https://github.com/jgirardet/sublack). For +higher versions, it is recommended to use [LSP](#python-lsp-server) as documented below. ## Python LSP Server diff --git a/docs/integrations/github_actions.md b/docs/integrations/github_actions.md index ebfcc2d95a2..56b2cdd0586 100644 --- a/docs/integrations/github_actions.md +++ b/docs/integrations/github_actions.md @@ -24,7 +24,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: psf/black@stable ``` diff --git a/docs/integrations/source_version_control.md b/docs/integrations/source_version_control.md index d462e2cc18a..3b895193941 100644 --- a/docs/integrations/source_version_control.md +++ b/docs/integrations/source_version_control.md @@ -6,29 +6,48 @@ Use [pre-commit](https://pre-commit.com/). Once you ```yaml repos: - - repo: https://github.com/psf/black - rev: 23.1.0 + # Using this mirror lets us use mypyc-compiled black, which is about 2x faster + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 23.12.1 hooks: - id: black # It is recommended to specify the latest version of Python # supported by your project here, or alternatively use # pre-commit's default_language_version, see # https://pre-commit.com/#top_level-default_language_version - language_version: python3.9 + language_version: python3.11 ``` -Feel free to switch out the `rev` value to something else, like another -[tag/version][black-tags] or even a specific commit. Although we discourage the use of +Feel free to switch out the `rev` value to a different version of Black. + +Note if you'd like to use a specific commit in `rev`, you'll need to swap the repo +specified from the mirror to https://github.com/psf/black. We discourage the use of branches or other mutable refs since the hook [won't auto update as you may expect][pre-commit-mutable-rev]. -If you want support for Jupyter Notebooks as well, then replace `id: black` with -`id: black-jupyter`. +## Jupyter Notebooks + +There is an alternate hook `black-jupyter` that expands the targets of `black` to +include Jupyter Notebooks. To use this hook, simply replace the hook's `id: black` with +`id: black-jupyter` in the `.pre-commit-config.yaml`: + +```yaml +repos: + # Using this mirror lets us use mypyc-compiled black, which is about 2x faster + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 23.12.1 + hooks: + - id: black-jupyter + # It is recommended to specify the latest version of Python + # supported by your project here, or alternatively use + # pre-commit's default_language_version, see + # https://pre-commit.com/#top_level-default_language_version + language_version: python3.11 +``` ```{note} -The `black-jupyter` hook is only available from version 21.8b0 and onwards. +The `black-jupyter` hook became available in version 21.8b0. ``` -[black-tags]: https://github.com/psf/black/tags [pre-commit-mutable-rev]: https://pre-commit.com/#using-the-latest-version-for-a-repository diff --git a/docs/requirements.txt b/docs/requirements.txt index 9a269d02a75..b5b9e22fc84 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,9 +1,9 @@ # Used by ReadTheDocs; pinned requirements for stability. -myst-parser==0.18.1 -Sphinx==5.3.0 +myst-parser==2.0.0 +Sphinx==7.2.6 # Older versions break Sphinx even though they're declared to be supported. -docutils==0.19 +docutils==0.20.1 sphinxcontrib-programoutput==0.17 -sphinx_copybutton==0.5.1 -furo==2022.12.7 +sphinx_copybutton==0.5.2 +furo==2023.9.10 diff --git a/docs/the_black_code_style/current_style.md b/docs/the_black_code_style/current_style.md index 83f8785cc55..00bd81416dc 100644 --- a/docs/the_black_code_style/current_style.md +++ b/docs/the_black_code_style/current_style.md @@ -8,16 +8,9 @@ deliberately limited and rarely added. Previous formatting is taken into account little as possible, with rare exceptions like the magic trailing comma. The coding style used by _Black_ can be viewed as a strict subset of PEP 8. -_Black_ reformats entire files in place. It doesn't reformat lines that end with -`# fmt: skip` or blocks that start with `# fmt: off` and end with `# fmt: on`. -`# fmt: on/off` must be on the same level of indentation and in the same block, meaning -no unindents beyond the initial indentation level between them. It also recognizes -[YAPF](https://github.com/google/yapf)'s block comments to the same effect, as a -courtesy for straddling code. - -The rest of this document describes the current formatting style. If you're interested -in trying out where the style is heading, see [future style](./future_style.md) and try -running `black --preview`. +This document describes the current formatting style. If you're interested in trying out +where the style is heading, see [future style](./future_style.md) and try running +`black --preview`. ### How _Black_ wraps lines @@ -140,6 +133,8 @@ If you're reaching for backslashes, that's a clear signal that you can do better slightly refactor your code. I hope some of the examples above show you that there are many ways in which you can do it. +(labels/line-length)= + ### Line length You probably noticed the peculiar default line length. _Black_ defaults to 88 characters @@ -158,33 +153,35 @@ harder to work with line lengths exceeding 100 characters. It also adversely aff side-by-side diff review on typical screen resolutions. Long lines also make it harder to present code neatly in documentation or talk slides. -If you're using Flake8, you can bump `max-line-length` to 88 and mostly forget about it. -However, it's better if you use [Bugbear](https://github.com/PyCQA/flake8-bugbear)'s -B950 warning instead of E501, and bump the max line length to 88 (or the `--line-length` -you used for black), which will align more with black's _"try to respect -`--line-length`, but don't become crazy if you can't"_. You'd do it like this: - -```ini -[flake8] -max-line-length = 88 -... -select = C,E,F,W,B,B950 -extend-ignore = E203, E501 -``` +#### Flake8 -Explanation of why E203 is disabled can be found further in this documentation. And if -you're curious about the reasoning behind B950, -[Bugbear's documentation](https://github.com/PyCQA/flake8-bugbear#opinionated-warnings) -explains it. The tl;dr is "it's like highway speed limits, we won't bother you if you -overdo it by a few km/h". +If you use Flake8, you have a few options: -**If you're looking for a minimal, black-compatible flake8 configuration:** +1. Recommended is using [Bugbear](https://github.com/PyCQA/flake8-bugbear) and enabling + its B950 check instead of using Flake8's E501, because it aligns with Black's 10% + rule. Install Bugbear and use the following config: -```ini -[flake8] -max-line-length = 88 -extend-ignore = E203 -``` + ```ini + [flake8] + max-line-length = 80 + ... + select = C,E,F,W,B,B950 + extend-ignore = E203, E501, E704 + ``` + + The rationale for B950 is explained in + [Bugbear's documentation](https://github.com/PyCQA/flake8-bugbear#opinionated-warnings). + +2. For a minimally compatible config: + + ```ini + [flake8] + max-line-length = 88 + extend-ignore = E203, E704 + ``` + +An explanation of why E203 is disabled can be found in the [Slices section](#slices) of +this page. ### Empty lines @@ -250,11 +247,12 @@ required due to an inner function starting immediately after. _Black_ does not format comment contents, but it enforces two spaces between code and a comment on the same line, and a space before the comment text begins. Some types of -comments that require specific spacing rules are respected: doc comments (`#: comment`), -section comments with long runs of hashes, and Spyder cells. Non-breaking spaces after -hashes are also preserved. Comments may sometimes be moved because of formatting -changes, which can break tools that assign special meaning to them. See -[AST before and after formatting](#ast-before-and-after-formatting) for more discussion. +comments that require specific spacing rules are respected: shebangs (`#! comment`), doc +comments (`#: comment`), section comments with long runs of hashes, and Spyder cells. +Non-breaking spaces after hashes are also preserved. Comments may sometimes be moved +because of formatting changes, which can break tools that assign special meaning to +them. See [AST before and after formatting](#ast-before-and-after-formatting) for more +discussion. ### Trailing commas @@ -273,6 +271,8 @@ A pre-existing trailing comma informs _Black_ to always explode contents of the bracket pair into one item per line. Read more about this in the [Pragmatism](#pragmatism) section below. +(labels/strings)= + ### Strings _Black_ prefers double quotes (`"` and `"""`) over single quotes (`'` and `'''`). It @@ -457,6 +457,8 @@ there were not many users anyway. Not many edge cases were reported. As a mature _Black_ does make some exceptions to rules it otherwise holds. This section documents what those exceptions are and why this is the case. +(labels/magic-trailing-comma)= + ### The magic trailing comma _Black_ in general does not take existing formatting into account. @@ -493,6 +495,8 @@ default by (among others) GitHub and Visual Studio Code, differentiates between r-strings and R-strings. The former are syntax highlighted as regular expressions while the latter are treated as true raw strings with no special semantics. +(labels/ast-changes)= + ### AST before and after formatting When run with `--safe` (the default), _Black_ checks that the code before and after is diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index 6d289d460a7..f55ea5f60a9 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -47,11 +47,13 @@ with contextlib.ExitStack() as exit_stack: ... ``` +(labels/preview-style)= + ## Preview style Experimental, potentially disruptive style changes are gathered under the `--preview` CLI flag. At the end of each year, these changes may be adopted into the default style, -as described in [The Black Code Style](./index.rst). Because the functionality is +as described in [The Black Code Style](index.md). Because the functionality is experimental, feedback and issue reports are highly encouraged! ### Improved string processing @@ -62,3 +64,246 @@ plain strings. User-made splits are respected when they do not exceed the line l limit. Line continuation backslashes are converted into parenthesized strings. Unnecessary parentheses are stripped. The stability and status of this feature is tracked in [this issue](https://github.com/psf/black/issues/2188). + +### Improved line breaks + +For assignment expressions, _Black_ now prefers to split and wrap the right side of the +assignment instead of left side. For example: + +```python +some_dict[ + "with_a_long_key" +] = some_looooooooong_module.some_looooooooooooooong_function_name( + first_argument, second_argument, third_argument +) +``` + +will be changed to: + +```python +some_dict["with_a_long_key"] = ( + some_looooooooong_module.some_looooooooooooooong_function_name( + first_argument, second_argument, third_argument + ) +) +``` + +### Improved parentheses management + +For dict literals with long values, they are now wrapped in parentheses. Unnecessary +parentheses are now removed. For example: + +```python +my_dict = { + "a key in my dict": a_very_long_variable + * and_a_very_long_function_call() + / 100000.0, + "another key": (short_value), +} +``` + +will be changed to: + +```python +my_dict = { + "a key in my dict": ( + a_very_long_variable * and_a_very_long_function_call() / 100000.0 + ), + "another key": short_value, +} +``` + +### Improved multiline dictionary and list indentation for sole function parameter + +For better readability and less verticality, _Black_ now pairs parentheses ("(", ")") +with braces ("{", "}") and square brackets ("[", "]") on the same line. For example: + +```python +foo( + [ + 1, + 2, + 3, + ] +) + +nested_array = [ + [ + 1, + 2, + 3, + ] +] +``` + +will be changed to: + +```python +foo([ + 1, + 2, + 3, +]) + +nested_array = [[ + 1, + 2, + 3, +]] +``` + +This also applies to list and dictionary unpacking: + +```python +foo( + *[ + a_long_function_name(a_long_variable_name) + for a_long_variable_name in some_generator + ] +) +``` + +will become: + +```python +foo(*[ + a_long_function_name(a_long_variable_name) + for a_long_variable_name in some_generator +]) +``` + +You can use a magic trailing comma to avoid this compacting behavior; by default, +_Black_ will not reformat the following code: + +```python +foo( + [ + 1, + 2, + 3, + ], +) +``` + +### Improved multiline string handling + +_Black_ is smarter when formatting multiline strings, especially in function arguments, +to avoid introducing extra line breaks. Previously, it would always consider multiline +strings as not fitting on a single line. With this new feature, _Black_ looks at the +context around the multiline string to decide if it should be inlined or split to a +separate line. For example, when a multiline string is passed to a function, _Black_ +will only split the multiline string if a line is too long or if multiple arguments are +being passed. + +For example, _Black_ will reformat + +```python +textwrap.dedent( + """\ + This is a + multiline string +""" +) +``` + +to: + +```python +textwrap.dedent("""\ + This is a + multiline string +""") +``` + +And: + +```python +MULTILINE = """ +foobar +""".replace( + "\n", "" +) +``` + +to: + +```python +MULTILINE = """ +foobar +""".replace("\n", "") +``` + +Implicit multiline strings are special, because they can have inline comments. Strings +without comments are merged, for example + +```python +s = ( + "An " + "implicit " + "multiline " + "string" +) +``` + +becomes + +```python +s = "An implicit multiline string" +``` + +A comment on any line of the string (or between two string lines) will block the +merging, so + +```python +s = ( + "An " # Important comment concerning just this line + "implicit " + "multiline " + "string" +) +``` + +and + +```python +s = ( + "An " + "implicit " + # Comment in between + "multiline " + "string" +) +``` + +will not be merged. Having the comment after or before the string lines (but still +inside the parens) will merge the string. For example + +```python +s = ( # Top comment + "An " + "implicit " + "multiline " + "string" + # Bottom comment +) +``` + +becomes + +```python +s = ( # Top comment + "An implicit multiline string" + # Bottom comment +) +``` + +======= + +### Form feed characters + +_Black_ will now retain form feed characters on otherwise empty lines at the module +level. Only one form feed is retained for a group of consecutive empty lines. Where +there are two empty lines in a row, the form feed will be placed on the second line. + +_Black_ already retained form feed literals inside a comment or inside a string. This +remains the case. diff --git a/docs/the_black_code_style/index.md b/docs/the_black_code_style/index.md index e5967be2db4..1719347eec8 100644 --- a/docs/the_black_code_style/index.md +++ b/docs/the_black_code_style/index.md @@ -17,7 +17,7 @@ Python language and, occasionally, in response to user feedback. Large-scale sty preferences presented in {doc}`current_style` are very unlikely to change, but minor style aspects and details might change according to the stability policy presented below. Ongoing style considerations are tracked on GitHub with the -[design](https://github.com/psf/black/labels/T%3A%20design) issue label. +[style](https://github.com/psf/black/labels/T%3A%20style) issue label. (labels/stability-policy)= diff --git a/docs/usage_and_configuration/black_docker_image.md b/docs/usage_and_configuration/black_docker_image.md index 85aec91ef1c..c97c25af328 100644 --- a/docs/usage_and_configuration/black_docker_image.md +++ b/docs/usage_and_configuration/black_docker_image.md @@ -24,6 +24,8 @@ created for all unreleased [commits on the `main` branch](https://github.com/psf/black/commits/main). This tag is not meant to be used by external users. +From version 23.11.0 the Docker image installs a compiled black into the image. + ## Usage A permanent container doesn't have to be created to use _Black_ as a Docker image. It's diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 2b41c187766..4f9856c6a47 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -12,7 +12,8 @@ _Black_ is a well-behaved Unix-style command-line tool: ## Usage -To get started right away with sensible defaults: +_Black_ will reformat entire files in place. To get started right away with sensible +defaults: ```sh black {source_file_or_directory} @@ -24,66 +25,134 @@ You can run _Black_ as a package if running it as a script doesn't work: python -m black {source_file_or_directory} ``` +### Ignoring sections + +Black will not reformat lines that contain `# fmt: skip` or blocks that start with +`# fmt: off` and end with `# fmt: on`. `# fmt: skip` can be mixed with other +pragmas/comments either with multiple comments (e.g. `# fmt: skip # pylint # noqa`) or +as a semicolon separated list (e.g. `# fmt: skip; pylint; noqa`). `# fmt: on/off` must +be on the same level of indentation and in the same block, meaning no unindents beyond +the initial indentation level between them. Black also recognizes +[YAPF](https://github.com/google/yapf)'s block comments to the same effect, as a +courtesy for straddling code. + ### Command line options -The CLI options of _Black_ can be displayed by expanding the view below or by running -`black --help`. While _Black_ has quite a few knobs these days, it is still opinionated -so style options are deliberately limited and rarely added. +The CLI options of _Black_ can be displayed by running `black --help`. All options are +also covered in more detail below. -
+While _Black_ has quite a few knobs these days, it is still opinionated so style options +are deliberately limited and rarely added. + +Note that all command-line options listed above can also be configured using a +`pyproject.toml` file (more on that below). + +#### `-h`, `--help` + +Show available command-line options and exit. -CLI reference +#### `-c`, `--code` -```{program-output} black --help +Format the code passed in as a string. +```console +$ black --code "print ( 'hello, world' )" +print("hello, world") ``` -
+#### `-l`, `--line-length` -Note that all command-line options listed above can also be configured using a -`pyproject.toml` file (more on that below). +How many characters per line to allow. The default is 88. -### Code input alternatives +See also [the style documentation](labels/line-length). -#### Standard Input +#### `-t`, `--target-version` -_Black_ supports formatting code via stdin, with the result being printed to stdout. -Just let _Black_ know with `-` as the path. +Python versions that should be supported by Black's output. You can run `black --help` +and look for the `--target-version` option to see the full list of supported versions. +You should include all versions that your code supports. If you support Python 3.8 +through 3.11, you should write: ```console -$ echo "print ( 'hello, world' )" | black - -print("hello, world") -reformatted - -All done! ✨ 🍰 ✨ -1 file reformatted. +$ black -t py38 -t py39 -t py310 -t py311 ``` -**Tip:** if you need _Black_ to treat stdin input as a file passed directly via the CLI, -use `--stdin-filename`. Useful to make sure _Black_ will respect the `--force-exclude` -option on some editors that rely on using stdin. +In a [configuration file](#configuration-via-a-file), you can write: -#### As a string +```toml +target-version = ["py38", "py39", "py310", "py311"] +``` -You can also pass code as a string using the `-c` / `--code` option. +By default, Black will infer target versions from the project metadata in +`pyproject.toml`, specifically the `[project.requires-python]` field. If this does not +yield conclusive results, Black will use per-file auto-detection. + +_Black_ uses this option to decide what grammar to use to parse your code. In addition, +it may use it to decide what style to use. For example, support for a trailing comma +after `*args` in a function call was added in Python 3.5, so _Black_ will add this comma +only if the target versions are all Python 3.5 or higher: ```console -$ black --code "print ( 'hello, world' )" -print("hello, world") +$ black --line-length=10 --target-version=py35 -c 'f(a, *args)' +f( + a, + *args, +) +$ black --line-length=10 --target-version=py34 -c 'f(a, *args)' +f( + a, + *args +) +$ black --line-length=10 --target-version=py34 --target-version=py35 -c 'f(a, *args)' +f( + a, + *args +) ``` -### Writeback and reporting +#### `--pyi` -By default _Black_ reformats the files given and/or found in place. Sometimes you need -_Black_ to just tell you what it _would_ do without actually rewriting the Python files. +Format all input files like typing stubs regardless of file extension. This is useful +when piping source on standard input. -There's two variations to this mode that are independently enabled by their respective -flags. Both variations can be enabled at once. +#### `--ipynb` + +Format all input files like Jupyter Notebooks regardless of file extension. This is +useful when piping source on standard input. + +#### `--python-cell-magics` + +When processing Jupyter Notebooks, add the given magic to the list of known python- +magics. Useful for formatting cells with custom python magics. + +#### `-x, --skip-source-first-line` + +Skip the first line of the source code. + +#### `-S, --skip-string-normalization` + +By default, _Black_ uses double quotes for all strings and normalizes string prefixes, +as described in [the style documentation](labels/strings). If this option is given, +strings are left unchanged instead. + +#### `-C, --skip-magic-trailing-comma` + +By default, _Black_ uses existing trailing commas as an indication that short lines +should be left separate, as described in +[the style documentation](labels/magic-trailing-comma). If this option is given, the +magic trailing comma is ignored. + +#### `--preview` + +Enable potentially disruptive style changes that may be added to Black's main +functionality in the next major release. Read more about +[our preview style](labels/preview-style). (labels/exit-code)= -#### Exit code +#### `--check` -Passing `--check` will make _Black_ exit with: +Don't write the files back, just return the status. _Black_ will exit with: - code 0 if nothing would change; - code 1 if some files would be reformatted; or @@ -111,17 +180,17 @@ $ echo $? 123 ``` -#### Diffs +#### `--diff` -Passing `--diff` will make _Black_ print out diffs that indicate what changes _Black_ -would've made. They are printed to stdout so capturing them is simple. +Don't write the files back, just output a diff to indicate what changes _Black_ would've +made. They are printed to stdout so capturing them is simple. -If you'd like colored diffs, you can enable them with the `--color`. +If you'd like colored diffs, you can enable them with `--color`. ```console $ black test.py --diff ---- test.py 2021-03-08 22:23:40.848954 +0000 -+++ test.py 2021-03-08 22:23:47.126319 +0000 +--- test.py 2021-03-08 22:23:40.848954+00:00 ++++ test.py 2021-03-08 22:23:47.126319+00:00 @@ -1 +1 @@ -print ( 'hello, world' ) +print("hello, world") @@ -130,25 +199,119 @@ All done! ✨ 🍰 ✨ 1 file would be reformatted. ``` -### Output verbosity +#### `--color` / `--no-color` -_Black_ in general tries to produce the right amount of output, balancing between -usefulness and conciseness. By default, _Black_ emits files modified and error messages, -plus a short summary. +Show (or do not show) colored diff. Only applies when `--diff` is given. + +#### `--line-ranges` + +When specified, _Black_ will try its best to only format these lines. + +This option can be specified multiple times, and a union of the lines will be formatted. +Each range must be specified as two integers connected by a `-`: `-`. The +`` and `` integer indices are 1-based and inclusive on both ends. + +_Black_ may still format lines outside of the ranges for multi-line statements. +Formatting more than one file or any ipynb files with this option is not supported. This +option cannot be specified in the `pyproject.toml` config. + +Example: `black --line-ranges=1-10 --line-ranges=21-30 test.py` will format lines from +`1` to `10` and `21` to `30`. + +This option is mainly for editor integrations, such as "Format Selection". + +```{note} +Due to [#4052](https://github.com/psf/black/issues/4052), `--line-ranges` might format +extra lines outside of the ranges when ther are unformatted lines with the exact +content. It also disables _Black_'s formatting stability check in `--safe` mode. +``` + +#### `--fast` / `--safe` + +By default, _Black_ performs [an AST safety check](labels/ast-changes) after formatting +your code. The `--fast` flag turns off this check and the `--safe` flag explicitly +enables it. + +#### `--required-version` + +Require a specific version of _Black_ to be running. This is useful for ensuring that +all contributors to your project are using the same version, because different versions +of _Black_ may format code a little differently. This option can be set in a +configuration file for consistent results across environments. ```console -$ black src/ +$ black --version +black, 23.12.1 (compiled: yes) +$ black --required-version 23.12.1 -c "format = 'this'" +format = "this" +$ black --required-version 31.5b2 -c "still = 'beta?!'" +Oh no! 💥 💔 💥 The required version does not match the running version! +``` + +You can also pass just the major version: + +```console +$ black --required-version 22 -c "format = 'this'" +format = "this" +$ black --required-version 31 -c "still = 'beta?!'" +Oh no! 💥 💔 💥 The required version does not match the running version! +``` + +Because of our [stability policy](../the_black_code_style/index.md), this will guarantee +stable formatting, but still allow you to take advantage of improvements that do not +affect formatting. + +#### `--exclude` + +A regular expression that matches files and directories that should be excluded on +recursive searches. An empty value means no paths are excluded. Use forward slashes for +directories on all platforms (Windows, too). By default, Black also ignores all paths +listed in `.gitignore`. Changing this value will override all default exclusions. + +#### `--extend-exclude` + +Like `--exclude`, but adds additional files and directories on top of the default values +instead of overriding them. + +#### `--force-exclude` + +Like `--exclude`, but files and directories matching this regex will be excluded even +when they are passed explicitly as arguments. This is useful when invoking Black +programmatically on changed files, such as in a pre-commit hook or editor plugin. + +#### `--stdin-filename` + +The name of the file when passing it through stdin. Useful to make sure Black will +respect the `--force-exclude` option on some editors that rely on using stdin. + +#### `--include` + +A regular expression that matches files and directories that should be included on +recursive searches. An empty value means all files are included regardless of the name. +Use forward slashes for directories on all platforms (Windows, too). Overrides all +exclusions, including from `.gitignore` and command line options. + +#### `-W`, `--workers` + +When _Black_ formats multiple files, it may use a process pool to speed up formatting. +This option controls the number of parallel workers. This can also be specified via the +`BLACK_NUM_WORKERS` environment variable. Defaults to the number of CPUs in the system. + +#### `-q`, `--quiet` + +Stop emitting all non-critical output. Error messages will still be emitted (which can +silenced by `2>/dev/null`). + +```console +$ black src/ -q error: cannot format src/black_primer/cli.py: Cannot parse: 5:6: mport asyncio -reformatted src/black_primer/lib.py -reformatted src/blackd/__init__.py -reformatted src/black/__init__.py -Oh no! 💥 💔 💥 -3 files reformatted, 2 files left unchanged, 1 file failed to reformat. ``` -Passing `-v` / `--verbose` will cause _Black_ to also emit messages about files that -were not changed or were ignored due to exclusion patterns. If _Black_ is using a -configuration file, a blue message detailing which one it is using will be emitted. +#### `-v`, `--verbose` + +Emit messages about files that were not changed or were ignored due to exclusion +patterns. If _Black_ is using a configuration file, a message detailing which one it is +using will be emitted. ```console $ black src/ -v @@ -164,35 +327,82 @@ Oh no! 💥 💔 💥 3 files reformatted, 2 files left unchanged, 1 file failed to reformat ``` -Passing `-q` / `--quiet` will cause _Black_ to stop emitting all non-critial output. -Error messages will still be emitted (which can silenced by `2>/dev/null`). +#### `--version` + +You can check the version of _Black_ you have installed using the `--version` flag. ```console -$ black src/ -q -error: cannot format src/black_primer/cli.py: Cannot parse: 5:6: mport asyncio +$ black --version +black, 23.12.1 ``` -### Versions +#### `--config` -You can check the version of _Black_ you have installed using the `--version` flag. +Read configuration options from a configuration file. See +[below](#configuration-via-a-file) for more details on the configuration file. + +### Environment variable options + +_Black_ supports the following configuration via environment variables. + +#### `BLACK_CACHE_DIR` + +The directory where _Black_ should store its cache. + +#### `BLACK_NUM_WORKERS` + +The number of parallel workers _Black_ should use. The command line option `-W` / +`--workers` takes precedence over this environment variable. + +### Code input alternatives + +_Black_ supports formatting code via stdin, with the result being printed to stdout. +Just let _Black_ know with `-` as the path. ```console -$ black --version -black, version 23.1.0 +$ echo "print ( 'hello, world' )" | black - +print("hello, world") +reformatted - +All done! ✨ 🍰 ✨ +1 file reformatted. ``` -An option to require a specific version to be running is also provided. +**Tip:** if you need _Black_ to treat stdin input as a file passed directly via the CLI, +use `--stdin-filename`. Useful to make sure _Black_ will respect the `--force-exclude` +option on some editors that rely on using stdin. + +You can also pass code as a string using the `--code` option. + +### Writeback and reporting + +By default _Black_ reformats the files given and/or found in place. Sometimes you need +_Black_ to just tell you what it _would_ do without actually rewriting the Python files. + +There's two variations to this mode that are independently enabled by their respective +flags: + +- `--check` (exit with code 1 if any file would be reformatted) +- `--diff` (print a diff instead of reformatting files) + +Both variations can be enabled at once. + +### Output verbosity + +_Black_ in general tries to produce the right amount of output, balancing between +usefulness and conciseness. By default, _Black_ emits files modified and error messages, +plus a short summary. ```console -$ black --required-version 21.9b0 -c "format = 'this'" -format = "this" -$ black --required-version 31.5b2 -c "still = 'beta?!'" -Oh no! 💥 💔 💥 The required version does not match the running version! +$ black src/ +error: cannot format src/black_primer/cli.py: Cannot parse: 5:6: mport asyncio +reformatted src/black_primer/lib.py +reformatted src/blackd/__init__.py +reformatted src/black/__init__.py +Oh no! 💥 💔 💥 +3 files reformatted, 2 files left unchanged, 1 file failed to reformat. ``` -This is useful for example when running _Black_ in multiple environments that haven't -necessarily installed the correct version. This option can be set in a configuration -file for consistent results across environments. +The `--quiet` and `--verbose` flags control output verbosity. ## Configuration via a file @@ -241,8 +451,7 @@ refers to the path to your home directory. On Windows, this will be something li You can also explicitly specify the path to a particular file that you want with `--config`. In this situation _Black_ will not look for any other file. -If you're running with `--verbose`, you will see a blue message if a file was found and -used. +If you're running with `--verbose`, you will see a message if a file was found and used. Please note `blackd` will not use `pyproject.toml` configuration. diff --git a/gallery/gallery.py b/gallery/gallery.py index 38e52e34795..ba5d6f65fbe 100755 --- a/gallery/gallery.py +++ b/gallery/gallery.py @@ -243,11 +243,9 @@ def format_repos(repos: Tuple[Path, ...], options: Namespace) -> None: def main() -> None: - parser = ArgumentParser( - description="""Black Gallery is a script that + parser = ArgumentParser(description="""Black Gallery is a script that automates the process of applying different Black versions to a selected - PyPI package and seeing the results between versions.""" - ) + PyPI package and seeing the results between versions.""") group = parser.add_mutually_exclusive_group(required=True) group.add_argument("-p", "--pypi-package", help="PyPI package to download.") diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 58bb7536173..00000000000 --- a/mypy.ini +++ /dev/null @@ -1,41 +0,0 @@ -[mypy] -# Specify the target platform details in config, so your developers are -# free to run mypy on Windows, Linux, or macOS and get consistent -# results. -python_version=3.7 - -mypy_path=src - -show_column_numbers=True -show_error_codes=True - -# be strict -strict=True - -# except for... -no_implicit_reexport = False - -# Unreachable blocks have been an issue when compiling mypyc, let's try -# to avoid 'em in the first place. -warn_unreachable=True - -[mypy-blib2to3.driver.*] -ignore_missing_imports = True - -[mypy-IPython.*] -ignore_missing_imports = True - -[mypy-colorama.*] -ignore_missing_imports = True - -[mypy-pathspec.*] -ignore_missing_imports = True - -[mypy-tokenize_rt.*] -ignore_missing_imports = True - -[mypy-uvloop.*] -ignore_missing_imports = True - -[mypy-_black_version.*] -ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml index 435626ac8f4..24b9c07674d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,13 +7,12 @@ [tool.black] line-length = 88 -target-version = ['py37', 'py38'] +target-version = ['py38'] include = '\.pyi?$' extend-exclude = ''' /( # The following are specific to Black, you probably don't want those. - | blib2to3 - | tests/data + tests/data | profiling )/ ''' @@ -26,14 +25,14 @@ preview = true # NOTE: You don't need this in your own Black configuration. [build-system] -requires = ["hatchling>=1.8.0", "hatch-vcs", "hatch-fancy-pypi-readme"] +requires = ["hatchling>=1.20.0", "hatch-vcs", "hatch-fancy-pypi-readme"] build-backend = "hatchling.build" [project] name = "black" description = "The uncompromising code formatter." license = { text = "MIT" } -requires-python = ">=3.7" +requires-python = ">=3.8" authors = [ { name = "Łukasz Langa", email = "lukasz@langa.pl" }, ] @@ -54,11 +53,11 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Software Development :: Quality Assurance", ] @@ -69,8 +68,7 @@ dependencies = [ "pathspec>=0.9.0", "platformdirs>=2", "tomli>=1.1.0; python_version < '3.11'", - "typed-ast>=1.4.2; python_version < '3.8' and implementation_name == 'cpython'", - "typing_extensions>=3.10.0.0; python_version < '3.10'", + "typing_extensions>=4.0.1; python_version < '3.11'", ] dynamic = ["readme", "version"] @@ -78,7 +76,8 @@ dynamic = ["readme", "version"] colorama = ["colorama>=0.4.3"] uvloop = ["uvloop>=0.15.2"] d = [ - "aiohttp>=3.7.4", + "aiohttp>=3.7.4; sys_platform != 'win32' or implementation_name != 'pypy'", + "aiohttp>=3.7.4, !=3.9.0; sys_platform == 'win32' and implementation_name == 'pypy'", ] jupyter = [ "ipython>=7.8.0", @@ -115,14 +114,15 @@ exclude = ["/profiling"] [tool.hatch.build.targets.wheel] only-include = ["src"] sources = ["src"] +# Note that we change the behaviour of this flag below +macos-max-compat = true [tool.hatch.build.targets.wheel.hooks.mypyc] enable-by-default = false dependencies = [ - "hatch-mypyc>=0.13.0", - "mypy==0.991", - # Required stubs to be removed when the packages support PEP 561 themselves - "types-typed-ast>=1.4.2", + "hatch-mypyc>=0.16.0", + "mypy==1.7.1", + "click==8.1.3", # avoid https://github.com/pallets/click/issues/2558 ] require-runtime-dependencies = true exclude = [ @@ -140,16 +140,17 @@ exclude = [ # Compiled modules can't be run directly and that's a problem here: "/src/black/__main__.py", ] +mypy-args = ["--ignore-missing-imports"] options = { debug_level = "0" } [tool.cibuildwheel] build-verbosity = 1 # So these are the environments we target: -# - Python: CPython 3.7+ only +# - Python: CPython 3.8+ only # - Architecture (64-bit only): amd64 / x86_64, universal2, and arm64 # - OS: Linux (no musl), Windows, and macOS -build = "cp3*-*" -skip = ["*-manylinux_i686", "*-musllinux_*", "*-win32", "pp-*"] +build = "cp3*" +skip = ["*-manylinux_i686", "*-musllinux_*", "*-win32", "pp*"] # This is the bare minimum needed to run the test suite. Pulling in the full # test_requirements.txt would download a bunch of other packages not necessary # here and would slow down the testing step a fair bit. @@ -177,16 +178,25 @@ before-build = [ HATCH_BUILD_HOOKS_ENABLE = "1" MYPYC_OPT_LEVEL = "3" MYPYC_DEBUG_LEVEL = "0" +AIOHTTP_NO_EXTENSIONS = "1" + # Black needs Clang to compile successfully on Linux. CC = "clang" -AIOHTTP_NO_EXTENSIONS = "1" + +[tool.cibuildwheel.macos] +build-frontend = { name = "build", args = ["--no-isolation"] } +# Unfortunately, hatch doesn't respect MACOSX_DEPLOYMENT_TARGET +before-build = [ + "python -m pip install 'hatchling==1.20.0' hatch-vcs hatch-fancy-pypi-readme 'hatch-mypyc>=0.16.0' 'mypy==1.7.1' 'click==8.1.3'", + """sed -i '' -e "600,700s/'10_16'/os.environ['MACOSX_DEPLOYMENT_TARGET'].replace('.', '_')/" $(python -c 'import hatchling.builders.wheel as h; print(h.__file__)') """, +] [tool.isort] atomic = true profile = "black" line_length = 88 skip_gitignore = true -skip_glob = ["src/blib2to3", "tests/data", "profiling"] +skip_glob = ["tests/data", "profiling"] known_first_party = ["black", "blib2to3", "blackd", "_black_version"] [tool.pytest.ini_options] @@ -208,10 +218,45 @@ filterwarnings = [ # this is mitigated by a try/catch in https://github.com/psf/black/pull/3198/ # this ignore can be removed when support for aiohttp 3.x is dropped. '''ignore:Middleware decorator is deprecated since 4\.0 and its behaviour is default, you can simply remove this decorator:DeprecationWarning''', - # this is mitigated by https://github.com/python/cpython/issues/79071 in python 3.8+ - # this ignore can be removed when support for 3.7 is dropped. - '''ignore:Bare functions are deprecated, use async ones:DeprecationWarning''', # aiohttp is using deprecated cgi modules - Safe to remove when fixed: # https://github.com/aio-libs/aiohttp/issues/6905 '''ignore:'cgi' is deprecated and slated for removal in Python 3.13:DeprecationWarning''', + # Work around https://github.com/pytest-dev/pytest/issues/10977 for Python 3.12 + '''ignore:(Attribute s|Attribute n|ast.Str|ast.Bytes|ast.NameConstant|ast.Num) is deprecated and will be removed in Python 3.14:DeprecationWarning''', + # Will be fixed with aiohttp 3.9.0 + # https://github.com/aio-libs/aiohttp/pull/7302 + "ignore:datetime.*utcfromtimestamp\\(\\) is deprecated and scheduled for removal:DeprecationWarning", +] +[tool.coverage.report] +omit = [ + "src/blib2to3/*", + "tests/data/*", + "*/site-packages/*", + ".tox/*" ] +[tool.coverage.run] +relative_files = true +branch = true + +[tool.mypy] +# Specify the target platform details in config, so your developers are +# free to run mypy on Windows, Linux, or macOS and get consistent +# results. +python_version = "3.8" +mypy_path = "src" +strict = true +# Unreachable blocks have been an issue when compiling mypyc, let's try to avoid 'em in the first place. +warn_unreachable = true +implicit_reexport = true +show_error_codes = true +show_column_numbers = true + +[[tool.mypy.overrides]] +module = ["pathspec.*", "IPython.*", "colorama.*", "tokenize_rt.*", "uvloop.*", "_black_version.*"] +ignore_missing_imports = true + +# CI only checks src/, but in case users are running LSP or similar we explicitly ignore +# errors in test data files. +[[tool.mypy.overrides]] +module = ["tests.data.*"] +ignore_errors = true diff --git a/scripts/check_pre_commit_rev_in_example.py b/scripts/check_pre_commit_rev_in_example.py index 9560b3b8401..cc45a31e1ed 100644 --- a/scripts/check_pre_commit_rev_in_example.py +++ b/scripts/check_pre_commit_rev_in_example.py @@ -14,7 +14,7 @@ import commonmark import yaml -from bs4 import BeautifulSoup +from bs4 import BeautifulSoup # type: ignore[import-untyped] def main(changes: str, source_version_control: str) -> None: diff --git a/scripts/check_version_in_basics_example.py b/scripts/check_version_in_basics_example.py index c62780d97ab..c90fdb43da5 100644 --- a/scripts/check_version_in_basics_example.py +++ b/scripts/check_version_in_basics_example.py @@ -8,7 +8,7 @@ import sys import commonmark -from bs4 import BeautifulSoup +from bs4 import BeautifulSoup # type: ignore[import-untyped] def main(changes: str, the_basics: str) -> None: @@ -20,20 +20,21 @@ def main(changes: str, the_basics: str) -> None: the_basics_html = commonmark.commonmark(the_basics) the_basics_soup = BeautifulSoup(the_basics_html, "html.parser") - (version_example,) = [ + version_examples = [ code_block.string for code_block in the_basics_soup.find_all(class_="language-console") if "$ black --version" in code_block.string ] for tag in tags: - if tag in version_example and tag != latest_tag: - print( - "Please set the version in the ``black --version`` " - "example from ``the_basics.md`` to be the latest one.\n" - f"Expected {latest_tag}, got {tag}.\n" - ) - sys.exit(1) + for version_example in version_examples: + if tag in version_example and tag != latest_tag: + print( + "Please set the version in the ``black --version`` " + "examples from ``the_basics.md`` to be the latest one.\n" + f"Expected {latest_tag}, got {tag}.\n" + ) + sys.exit(1) if __name__ == "__main__": diff --git a/scripts/diff_shades_gha_helper.py b/scripts/diff_shades_gha_helper.py index b5fea5a817d..8cd8ba155ce 100644 --- a/scripts/diff_shades_gha_helper.py +++ b/scripts/diff_shades_gha_helper.py @@ -21,6 +21,7 @@ import subprocess import sys import zipfile +from base64 import b64encode from io import BytesIO from pathlib import Path from typing import Any @@ -52,7 +53,17 @@ def set_output(name: str, value: str) -> None: print(f"[INFO]: setting '{name}' to '{value}'") else: print(f"[INFO]: setting '{name}' to [{len(value)} chars]") - print(f"::set-output name={name}::{value}") + + if "GITHUB_OUTPUT" in os.environ: + if "\n" in value: + # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings + delimiter = b64encode(os.urandom(16)).decode() + value = f"{delimiter}\n{value}\n{delimiter}" + command = f"{name}<<{value}" + else: + command = f"{name}={value}" + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + print(command, file=f) def http_get(url: str, *, is_json: bool = True, **kwargs: Any) -> Any: @@ -108,7 +119,7 @@ def main() -> None: @main.command("config", help="Acquire run configuration and metadata.") @click.argument("event", type=click.Choice(["push", "pull_request"])) def config(event: Literal["push", "pull_request"]) -> None: - import diff_shades + import diff_shades # type: ignore[import-not-found] if event == "push": jobs = [{"mode": "preview-changes", "force-flag": "--force-preview-style"}] @@ -218,9 +229,7 @@ def comment_details(run_id: str) -> None: # while it's still in progress seems impossible). body = body.replace("$workflow-run-url", data["html_url"]) body = body.replace("$job-diff-url", diff_url) - # https://github.community/t/set-output-truncates-multiline-strings/16852/3 - escaped = body.replace("%", "%25").replace("\n", "%0A").replace("\r", "%0D") - set_output("comment-body", escaped) + set_output("comment-body", body) if __name__ == "__main__": diff --git a/scripts/fuzz.py b/scripts/fuzz.py index 25362c927d4..018b66e0ea2 100644 --- a/scripts/fuzz.py +++ b/scripts/fuzz.py @@ -21,7 +21,7 @@ max_examples=1000, # roughly 1k tests/minute, or half that under coverage derandomize=True, # deterministic mode to avoid CI flakiness deadline=None, # ignore Hypothesis' health checks; we already know that - suppress_health_check=HealthCheck.all(), # this is slow and filter-heavy. + suppress_health_check=list(HealthCheck), # this is slow and filter-heavy. ) @given( # Note that while Hypothesmith might generate code unlike that written by @@ -80,7 +80,7 @@ def test_idempotent_any_syntatically_valid_python( try: import sys - import atheris + import atheris # type: ignore[import-not-found] except ImportError: pass else: diff --git a/scripts/make_width_table.py b/scripts/make_width_table.py new file mode 100644 index 00000000000..1b53c1b2cc9 --- /dev/null +++ b/scripts/make_width_table.py @@ -0,0 +1,66 @@ +"""Generates a width table for Unicode characters. + +This script generates a width table for Unicode characters that are not +narrow (width 1). The table is written to src/black/_width_table.py (note +that although this file is generated, it is checked into Git) and is used +by the char_width() function in src/black/strings.py. + +You should run this script when you upgrade wcwidth, which is expected to +happen when a new Unicode version is released. The generated table contains +the version of wcwidth and Unicode that it was generated for. + +In order to run this script, you need to install the latest version of wcwidth. +You can do this by running: + + pip install -U wcwidth + +""" + +import sys +from os.path import basename, dirname, join +from typing import Iterable, Tuple + +import wcwidth # type: ignore[import-not-found] + + +def make_width_table() -> Iterable[Tuple[int, int, int]]: + start_codepoint = -1 + end_codepoint = -1 + range_width = -2 + for codepoint in range(0, sys.maxunicode + 1): + width = wcwidth.wcwidth(chr(codepoint)) + if width <= 1: + # Ignore narrow characters along with zero-width characters so that + # they are treated as single-width. Note that treating zero-width + # characters as single-width is consistent with the heuristics built + # on top of str.isascii() in the str_width() function in strings.py. + continue + if start_codepoint < 0: + start_codepoint = codepoint + range_width = width + elif width != range_width or codepoint != end_codepoint + 1: + yield (start_codepoint, end_codepoint, range_width) + start_codepoint = codepoint + range_width = width + end_codepoint = codepoint + if start_codepoint >= 0: + yield (start_codepoint, end_codepoint, range_width) + + +def main() -> None: + table_path = join(dirname(__file__), "..", "src", "black", "_width_table.py") + with open(table_path, "w") as f: + f.write(f"""# Generated by {basename(__file__)} +# wcwidth {wcwidth.__version__} +# Unicode {wcwidth.list_versions()[-1]} +from typing import Final, List, Tuple + +WIDTH_TABLE: Final[List[Tuple[int, int, int]]] = [ +""") + for triple in make_width_table(): + f.write(f" {triple!r},\n") + f.write("]\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/release.py b/scripts/release.py new file mode 100755 index 00000000000..d588429c2d3 --- /dev/null +++ b/scripts/release.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +""" +Tool to help automate changes needed in commits during and after releases +""" + +import argparse +import logging +import sys +from datetime import datetime +from pathlib import Path +from subprocess import PIPE, run +from typing import List + +LOG = logging.getLogger(__name__) +NEW_VERSION_CHANGELOG_TEMPLATE = """\ +## Unreleased + +### Highlights + + + +### Stable style + + + +### Preview style + + + +### Configuration + + + +### Packaging + + + +### Parser + + + +### Performance + + + +### Output + + + +### _Blackd_ + + + +### Integrations + + + +### Documentation + + +""" + + +class NoGitTagsError(Exception): ... # noqa: E701,E761 + + +# TODO: Do better with alpha + beta releases +# Maybe we vendor packaging library +def get_git_tags(versions_only: bool = True) -> List[str]: + """Pull out all tags or calvers only""" + cp = run(["git", "tag"], stdout=PIPE, stderr=PIPE, check=True, encoding="utf8") + if not cp.stdout: + LOG.error(f"Returned no git tags stdout: {cp.stderr}") + raise NoGitTagsError + git_tags = cp.stdout.splitlines() + if versions_only: + return [t for t in git_tags if t[0].isdigit()] + return git_tags + + +# TODO: Support sorting alhpa/beta releases correctly +def tuple_calver(calver: str) -> tuple[int, ...]: # mypy can't notice maxsplit below + """Convert a calver string into a tuple of ints for sorting""" + try: + return tuple(map(int, calver.split(".", maxsplit=2))) + except ValueError: + return (0, 0, 0) + + +class SourceFiles: + def __init__(self, black_repo_dir: Path): + # File path fun all pathlib to be platform agnostic + self.black_repo_path = black_repo_dir + self.changes_path = self.black_repo_path / "CHANGES.md" + self.docs_path = self.black_repo_path / "docs" + self.version_doc_paths = ( + self.docs_path / "integrations" / "source_version_control.md", + self.docs_path / "usage_and_configuration" / "the_basics.md", + ) + self.current_version = self.get_current_version() + self.next_version = self.get_next_version() + + def __str__(self) -> str: + return f"""\ +> SourceFiles ENV: + Repo path: {self.black_repo_path} + CHANGES.md path: {self.changes_path} + docs path: {self.docs_path} + Current version: {self.current_version} + Next version: {self.next_version} +""" + + def add_template_to_changes(self) -> int: + """Add the template to CHANGES.md if it does not exist""" + LOG.info(f"Adding template to {self.changes_path}") + + with self.changes_path.open("r") as cfp: + changes_string = cfp.read() + + if "## Unreleased" in changes_string: + LOG.error(f"{self.changes_path} already has unreleased template") + return 1 + + templated_changes_string = changes_string.replace( + "# Change Log\n", + f"# Change Log\n\n{NEW_VERSION_CHANGELOG_TEMPLATE}", + ) + + with self.changes_path.open("w") as cfp: + cfp.write(templated_changes_string) + + LOG.info(f"Added template to {self.changes_path}") + return 0 + + def cleanup_changes_template_for_release(self) -> None: + LOG.info(f"Cleaning up {self.changes_path}") + + with self.changes_path.open("r") as cfp: + changes_string = cfp.read() + + # Change Unreleased to next version + versioned_changes = changes_string.replace( + "## Unreleased", f"## {self.next_version}" + ) + + # Remove all comments (subheadings are harder - Human required still) + no_comments_changes = [] + for line in versioned_changes.splitlines(): + if line.startswith(""): + continue + no_comments_changes.append(line) + + with self.changes_path.open("w") as cfp: + cfp.write("\n".join(no_comments_changes) + "\n") + + LOG.debug(f"Finished Cleaning up {self.changes_path}") + + def get_current_version(self) -> str: + """Get the latest git (version) tag as latest version""" + return sorted(get_git_tags(), key=lambda k: tuple_calver(k))[-1] + + def get_next_version(self) -> str: + """Workout the year and month + version number we need to move to""" + base_calver = datetime.today().strftime("%y.%m") + calver_parts = base_calver.split(".") + base_calver = f"{calver_parts[0]}.{int(calver_parts[1])}" # Remove leading 0 + git_tags = get_git_tags() + same_month_releases = [t for t in git_tags if t.startswith(base_calver)] + if len(same_month_releases) < 1: + return f"{base_calver}.0" + same_month_version = same_month_releases[-1].split(".", 2)[-1] + return f"{base_calver}.{int(same_month_version) + 1}" + + def update_repo_for_release(self) -> int: + """Update CHANGES.md + doc files ready for release""" + self.cleanup_changes_template_for_release() + self.update_version_in_docs() + return 0 # return 0 if no exceptions hit + + def update_version_in_docs(self) -> None: + for doc_path in self.version_doc_paths: + LOG.info(f"Updating black version to {self.next_version} in {doc_path}") + + with doc_path.open("r") as dfp: + doc_string = dfp.read() + + next_version_doc = doc_string.replace( + self.current_version, self.next_version + ) + + with doc_path.open("w") as dfp: + dfp.write(next_version_doc) + + LOG.debug( + f"Finished updating black version to {self.next_version} in {doc_path}" + ) + + +def _handle_debug(debug: bool) -> None: + """Turn on debugging if asked otherwise INFO default""" + log_level = logging.DEBUG if debug else logging.INFO + logging.basicConfig( + format="[%(asctime)s] %(levelname)s: %(message)s (%(filename)s:%(lineno)d)", + level=log_level, + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument( + "-a", + "--add-changes-template", + action="store_true", + help="Add the Unreleased template to CHANGES.md", + ) + parser.add_argument( + "-d", "--debug", action="store_true", help="Verbose debug output" + ) + args = parser.parse_args() + _handle_debug(args.debug) + return args + + +def main() -> int: + args = parse_args() + + # Need parent.parent cause script is in scripts/ directory + sf = SourceFiles(Path(__file__).parent.parent) + + if args.add_changes_template: + return sf.add_template_to_changes() + + LOG.info(f"Current version detected to be {sf.current_version}") + LOG.info(f"Next version will be {sf.next_version}") + return sf.update_repo_for_release() + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/scripts/release_tests.py b/scripts/release_tests.py new file mode 100644 index 00000000000..bd72cb4b48a --- /dev/null +++ b/scripts/release_tests.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +import unittest +from pathlib import Path +from shutil import rmtree +from tempfile import TemporaryDirectory +from typing import Any +from unittest.mock import Mock, patch + +from release import SourceFiles, tuple_calver # type: ignore + + +class FakeDateTime: + """Used to mock the date to test generating next calver function""" + + def today(*args: Any, **kwargs: Any) -> "FakeDateTime": # noqa + return FakeDateTime() + + # Add leading 0 on purpose to ensure we remove it + def strftime(*args: Any, **kwargs: Any) -> str: # noqa + return "69.01" + + +class TestRelease(unittest.TestCase): + def setUp(self) -> None: + # We only test on >= 3.12 + self.tempdir = TemporaryDirectory(delete=False) # type: ignore + self.tempdir_path = Path(self.tempdir.name) + self.sf = SourceFiles(self.tempdir_path) + + def tearDown(self) -> None: + rmtree(self.tempdir.name) + return super().tearDown() + + @patch("release.get_git_tags") + def test_get_current_version(self, mocked_git_tags: Mock) -> None: + mocked_git_tags.return_value = ["1.1.0", "69.1.0", "69.1.1", "2.2.0"] + self.assertEqual("69.1.1", self.sf.get_current_version()) + + @patch("release.get_git_tags") + @patch("release.datetime", FakeDateTime) + def test_get_next_version(self, mocked_git_tags: Mock) -> None: + # test we handle no args + mocked_git_tags.return_value = [] + self.assertEqual( + "69.1.0", + self.sf.get_next_version(), + "Unable to get correct next version with no git tags", + ) + + # test we handle + mocked_git_tags.return_value = ["1.1.0", "69.1.0", "69.1.1", "2.2.0"] + self.assertEqual( + "69.1.2", + self.sf.get_next_version(), + "Unable to get correct version with 2 previous versions released this" + " month", + ) + + def test_tuple_calver(self) -> None: + first_month_release = tuple_calver("69.1.0") + second_month_release = tuple_calver("69.1.1") + self.assertEqual((69, 1, 0), first_month_release) + self.assertEqual((0, 0, 0), tuple_calver("69.1.1a0")) # Hack for alphas/betas + self.assertTrue(first_month_release < second_month_release) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/black/__init__.py b/src/black/__init__.py index 4ebf28821c3..735ba713b8f 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -7,12 +7,13 @@ import traceback from contextlib import contextmanager from dataclasses import replace -from datetime import datetime +from datetime import datetime, timezone from enum import Enum from json.decoder import JSONDecodeError from pathlib import Path from typing import ( Any, + Collection, Dict, Generator, Iterator, @@ -34,7 +35,7 @@ from pathspec.patterns.gitwildmatch import GitWildMatchPatternError from _black_version import version as __version__ -from black.cache import Cache, get_cache_info, read_cache, write_cache +from black.cache import Cache from black.comments import normalize_fmt_off from black.const import ( DEFAULT_EXCLUDES, @@ -50,6 +51,7 @@ get_gitignore, normalize_path_maybe_ignore, parse_pyproject_toml, + path_is_excluded, wrap_stream_for_windows, ) from black.handle_ipynb_magics import ( @@ -63,14 +65,9 @@ ) from black.linegen import LN, LineGenerator, transform_line from black.lines import EmptyLineTracker, LinesBlock -from black.mode import ( - FUTURE_FLAG_TO_FEATURE, - VERSION_TO_FEATURES, - Feature, - Mode, - TargetVersion, - supports_feature, -) +from black.mode import FUTURE_FLAG_TO_FEATURE, VERSION_TO_FEATURES, Feature +from black.mode import Mode as Mode # re-exported +from black.mode import TargetVersion, supports_feature from black.nodes import ( STARS, is_number_token, @@ -81,6 +78,7 @@ from black.output import color_diff, diff, dump_to_file, err, ipynb_diff, out from black.parsing import InvalidInput # noqa F401 from black.parsing import lib2to3_parse, parse_ast, stringify_ast +from black.ranges import adjusted_lines, convert_unchanged_lines, parse_line_ranges from black.report import Changed, NothingChanged, Report from black.trans import iter_fexpr_spans from blib2to3.pgen2 import token @@ -127,7 +125,9 @@ def read_pyproject_toml( otherwise. """ if not value: - value = find_pyproject_toml(ctx.params.get("src", ())) + value = find_pyproject_toml( + ctx.params.get("src", ()), ctx.params.get("stdin_filename", None) + ) if value is None: return None @@ -155,6 +155,22 @@ def read_pyproject_toml( "target-version", "Config key target-version must be a list" ) + exclude = config.get("exclude") + if exclude is not None and not isinstance(exclude, str): + raise click.BadOptionUsage("exclude", "Config key exclude must be a string") + + extend_exclude = config.get("extend_exclude") + if extend_exclude is not None and not isinstance(extend_exclude, str): + raise click.BadOptionUsage( + "extend-exclude", "Config key extend-exclude must be a string" + ) + + line_ranges = config.get("line_ranges") + if line_ranges is not None: + raise click.BadOptionUsage( + "line-ranges", "Cannot use line-ranges in the pyproject.toml file." + ) + default_map: Dict[str, Any] = {} if ctx.default_map: default_map.update(ctx.default_map) @@ -219,25 +235,26 @@ def validate_regex( callback=target_version_option_callback, multiple=True, help=( - "Python versions that should be supported by Black's output. By default, Black" - " will try to infer this from the project metadata in pyproject.toml. If this" - " does not yield conclusive results, Black will use per-file auto-detection." + "Python versions that should be supported by Black's output. You should" + " include all versions that your code supports. By default, Black will infer" + " target versions from the project metadata in pyproject.toml. If this does" + " not yield conclusive results, Black will use per-file auto-detection." ), ) @click.option( "--pyi", is_flag=True, help=( - "Format all input files like typing stubs regardless of file extension (useful" - " when piping source on standard input)." + "Format all input files like typing stubs regardless of file extension. This" + " is useful when piping source on standard input." ), ) @click.option( "--ipynb", is_flag=True, help=( - "Format all input files like Jupyter Notebooks regardless of file extension " - "(useful when piping source on standard input)." + "Format all input files like Jupyter Notebooks regardless of file extension." + "This is useful when piping source on standard input." ), ) @click.option( @@ -294,39 +311,48 @@ def validate_regex( @click.option( "--diff", is_flag=True, - help="Don't write the files back, just output a diff for each file on stdout.", + help=( + "Don't write the files back, just output a diff to indicate what changes" + " Black would've made. They are printed to stdout so capturing them is simple." + ), ) @click.option( "--color/--no-color", is_flag=True, - help="Show colored diff. Only applies when `--diff` is given.", + help="Show (or do not show) colored diff. Only applies when --diff is given.", ) @click.option( - "--fast/--safe", - is_flag=True, - help="If --fast given, skip temporary sanity checks. [default: --safe]", + "--line-ranges", + multiple=True, + metavar="START-END", + help=( + "When specified, Black will try its best to only format these lines. This" + " option can be specified multiple times, and a union of the lines will be" + " formatted. Each range must be specified as two integers connected by a `-`:" + " `-`. The `` and `` integer indices are 1-based and" + " inclusive on both ends." + ), + default=(), ) @click.option( - "--required-version", - type=str, + "--fast/--safe", + is_flag=True, help=( - "Require a specific version of Black to be running (useful for unifying results" - " across many environments e.g. with a pyproject.toml file). It can be" - " either a major version number or an exact version." + "By default, Black performs an AST safety check after formatting your code." + " The --fast flag turns off this check and the --safe flag explicitly enables" + " it. [default: --safe]" ), ) @click.option( - "--include", + "--required-version", type=str, - default=DEFAULT_INCLUDES, - callback=validate_regex, help=( - "A regular expression that matches files and directories that should be" - " included on recursive searches. An empty value means all files are included" - " regardless of the name. Use forward slashes for directories on all platforms" - " (Windows, too). Exclusions are calculated first, inclusions later." + "Require a specific version of Black to be running. This is useful for" + " ensuring that all contributors to your project are using the same" + " version, because different versions of Black may format code a little" + " differently. This option can be set in a configuration file for consistent" + " results across environments." ), - show_default=True, ) @click.option( "--exclude", @@ -336,8 +362,8 @@ def validate_regex( "A regular expression that matches files and directories that should be" " excluded on recursive searches. An empty value means no paths are excluded." " Use forward slashes for directories on all platforms (Windows, too)." - " Exclusions are calculated first, inclusions later. [default:" - f" {DEFAULT_EXCLUDES}]" + " By default, Black also ignores all paths listed in .gitignore. Changing this" + f" value will override all default exclusions. [default: {DEFAULT_EXCLUDES}]" ), show_default=False, ) @@ -347,7 +373,7 @@ def validate_regex( callback=validate_regex, help=( "Like --exclude, but adds additional files and directories on top of the" - " excluded ones. (Useful if you simply want to add to the default)" + " default values instead of overriding them." ), ) @click.option( @@ -355,33 +381,55 @@ def validate_regex( type=str, callback=validate_regex, help=( - "Like --exclude, but files and directories matching this regex will be " - "excluded even when they are passed explicitly as arguments." + "Like --exclude, but files and directories matching this regex will be excluded" + " even when they are passed explicitly as arguments. This is useful when" + " invoking Black programmatically on changed files, such as in a pre-commit" + " hook or editor plugin." ), ) @click.option( "--stdin-filename", type=str, + is_eager=True, + help=( + "The name of the file when passing it through stdin. Useful to make sure Black" + " will respect the --force-exclude option on some editors that rely on using" + " stdin." + ), +) +@click.option( + "--include", + type=str, + default=DEFAULT_INCLUDES, + callback=validate_regex, help=( - "The name of the file when passing it through stdin. Useful to make " - "sure Black will respect --force-exclude option on some " - "editors that rely on using stdin." + "A regular expression that matches files and directories that should be" + " included on recursive searches. An empty value means all files are included" + " regardless of the name. Use forward slashes for directories on all platforms" + " (Windows, too). Overrides all exclusions, including from .gitignore and" + " command line options." ), + show_default=True, ) @click.option( "-W", "--workers", type=click.IntRange(min=1), default=None, - help="Number of parallel workers [default: number of CPUs in the system]", + help=( + "When Black formats multiple files, it may use a process pool to speed up" + " formatting. This option controls the number of parallel workers. This can" + " also be specified via the BLACK_NUM_WORKERS environment variable. Defaults" + " to the number of CPUs in the system." + ), ) @click.option( "-q", "--quiet", is_flag=True, help=( - "Don't emit non-error messages to stderr. Errors are still emitted; silence" - " those with 2>/dev/null." + "Stop emitting all non-critical output. Error messages will still be emitted" + " (which can silenced by 2>/dev/null)." ), ) @click.option( @@ -389,8 +437,9 @@ def validate_regex( "--verbose", is_flag=True, help=( - "Also emit messages to stderr about files that were not changed or were ignored" - " due to exclusion patterns." + "Emit messages about files that were not changed or were ignored due to" + " exclusion patterns. If Black is using a configuration file, a message" + " detailing which one it is using will be emitted." ), ) @click.version_option( @@ -421,7 +470,7 @@ def validate_regex( ), is_eager=True, callback=read_pyproject_toml, - help="Read configuration from FILE path.", + help="Read configuration options from a configuration file.", ) @click.pass_context def main( # noqa: C901 @@ -431,6 +480,7 @@ def main( # noqa: C901 target_version: List[TargetVersion], check: bool, diff: bool, + line_ranges: Sequence[str], color: bool, fast: bool, pyi: bool, @@ -478,35 +528,13 @@ def main( # noqa: C901 fg="blue", ) - normalized = [ - ( - (source, source) - if source == "-" - else (normalize_path_maybe_ignore(Path(source), root), source) - ) - for source in src - ] - srcs_string = ", ".join( - [ - ( - f'"{_norm}"' - if _norm - else f'\033[31m"{source} (skipping - invalid)"\033[34m' - ) - for _norm, source in normalized - ] - ) - out(f"Sources to be formatted: {srcs_string}", fg="blue") - if config: config_source = ctx.get_parameter_source("config") user_level_config = str(find_user_pyproject_toml()) if config == user_level_config: out( - ( - "Using configuration from user-level config at " - f"'{user_level_config}'." - ), + "Using configuration from user-level config at " + f"'{user_level_config}'.", fg="blue", ) elif config_source in ( @@ -554,6 +582,18 @@ def main( # noqa: C901 python_cell_magics=set(python_cell_magics), ) + lines: List[Tuple[int, int]] = [] + if line_ranges: + if ipynb: + err("Cannot use --line-ranges with ipynb files.") + ctx.exit(1) + + try: + lines = parse_line_ranges(line_ranges) + except ValueError as e: + err(str(e)) + ctx.exit(1) + if code is not None: # Run in quiet mode by default with -c; the extra output isn't useful. # You can still pass -v to get verbose output. @@ -563,12 +603,18 @@ def main( # noqa: C901 if code is not None: reformat_code( - content=code, fast=fast, write_back=write_back, mode=mode, report=report + content=code, + fast=fast, + write_back=write_back, + mode=mode, + report=report, + lines=lines, ) else: + assert root is not None # root is only None if code is not None try: sources = get_sources( - ctx=ctx, + root=root, src=src, quiet=quiet, verbose=verbose, @@ -597,10 +643,14 @@ def main( # noqa: C901 write_back=write_back, mode=mode, report=report, + lines=lines, ) else: from black.concurrency import reformat_many + if lines: + err("Cannot use --line-ranges to format multiple files.") + ctx.exit(1) reformat_many( sources=sources, fast=fast, @@ -621,7 +671,7 @@ def main( # noqa: C901 def get_sources( *, - ctx: click.Context, + root: Path, src: Tuple[str, ...], quiet: bool, verbose: bool, @@ -634,7 +684,6 @@ def get_sources( ) -> Set[Path]: """Compute the set of files to be formatted.""" sources: Set[Path] = set() - root = ctx.obj["root"] using_default_exclude = exclude is None exclude = re_compile_maybe_verbose(DEFAULT_EXCLUDES) if exclude is None else exclude @@ -643,47 +692,58 @@ def get_sources( for s in src: if s == "-" and stdin_filename: - p = Path(stdin_filename) + path = Path(stdin_filename) is_stdin = True else: - p = Path(s) + path = Path(s) is_stdin = False - if is_stdin or p.is_file(): - normalized_path = normalize_path_maybe_ignore(p, ctx.obj["root"], report) - if normalized_path is None: - continue + # Compare the logic here to the logic in `gen_python_files`. + if is_stdin or path.is_file(): + root_relative_path = path.absolute().relative_to(root).as_posix() + + root_relative_path = "/" + root_relative_path - normalized_path = "/" + normalized_path # Hard-exclude any files that matches the `--force-exclude` regex. - if force_exclude: - force_exclude_match = force_exclude.search(normalized_path) - else: - force_exclude_match = None - if force_exclude_match and force_exclude_match.group(0): - report.path_ignored(p, "matches the --force-exclude regular expression") + if path_is_excluded(root_relative_path, force_exclude): + report.path_ignored( + path, "matches the --force-exclude regular expression" + ) + continue + + normalized_path: Optional[str] = normalize_path_maybe_ignore( + path, root, report + ) + if normalized_path is None: + if verbose: + out(f'Skipping invalid source: "{normalized_path}"', fg="red") continue if is_stdin: - p = Path(f"{STDIN_PLACEHOLDER}{str(p)}") + path = Path(f"{STDIN_PLACEHOLDER}{str(path)}") - if p.suffix == ".ipynb" and not jupyter_dependencies_are_installed( - verbose=verbose, quiet=quiet + if path.suffix == ".ipynb" and not jupyter_dependencies_are_installed( + warn=verbose or not quiet ): continue - sources.add(p) - elif p.is_dir(): - p = root / normalize_path_maybe_ignore(p, ctx.obj["root"], report) + if verbose: + out(f'Found input source: "{normalized_path}"', fg="blue") + sources.add(path) + elif path.is_dir(): + path = root / (path.resolve().relative_to(root)) + if verbose: + out(f'Found input source directory: "{path}"', fg="blue") + if using_default_exclude: gitignore = { root: root_gitignore, - p: get_gitignore(p), + path: get_gitignore(path), } sources.update( gen_python_files( - p.iterdir(), - ctx.obj["root"], + path.iterdir(), + root, include, exclude, extend_exclude, @@ -695,9 +755,12 @@ def get_sources( ) ) elif s == "-": - sources.add(p) + if verbose: + out("Found input source stdin", fg="blue") + sources.add(path) else: err(f"invalid path: {s}") + return sources @@ -714,7 +777,13 @@ def path_empty( def reformat_code( - content: str, fast: bool, write_back: WriteBack, mode: Mode, report: Report + content: str, + fast: bool, + write_back: WriteBack, + mode: Mode, + report: Report, + *, + lines: Collection[Tuple[int, int]] = (), ) -> None: """ Reformat and print out `content` without spawning child processes. @@ -727,7 +796,7 @@ def reformat_code( try: changed = Changed.NO if format_stdin_to_stdout( - content=content, fast=fast, write_back=write_back, mode=mode + content=content, fast=fast, write_back=write_back, mode=mode, lines=lines ): changed = Changed.YES report.done(path, changed) @@ -741,7 +810,13 @@ def reformat_code( # not ideal, but this shouldn't cause any issues ... hopefully. ~ichard26 @mypyc_attr(patchable=True) def reformat_one( - src: Path, fast: bool, write_back: WriteBack, mode: Mode, report: "Report" + src: Path, + fast: bool, + write_back: WriteBack, + mode: Mode, + report: "Report", + *, + lines: Collection[Tuple[int, int]] = (), ) -> None: """Reformat a single file under `src` without spawning child processes. @@ -766,24 +841,23 @@ def reformat_one( mode = replace(mode, is_pyi=True) elif src.suffix == ".ipynb": mode = replace(mode, is_ipynb=True) - if format_stdin_to_stdout(fast=fast, write_back=write_back, mode=mode): + if format_stdin_to_stdout( + fast=fast, write_back=write_back, mode=mode, lines=lines + ): changed = Changed.YES else: - cache: Cache = {} + cache = Cache.read(mode) if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF): - cache = read_cache(mode) - res_src = src.resolve() - res_src_s = str(res_src) - if res_src_s in cache and cache[res_src_s] == get_cache_info(res_src): + if not cache.is_changed(src): changed = Changed.CACHED if changed is not Changed.CACHED and format_file_in_place( - src, fast=fast, write_back=write_back, mode=mode + src, fast=fast, write_back=write_back, mode=mode, lines=lines ): changed = Changed.YES if (write_back is WriteBack.YES and changed is not Changed.CACHED) or ( write_back is WriteBack.CHECK and changed is Changed.NO ): - write_cache(cache, [src], mode) + cache.write([src]) report.done(src, changed) except Exception as exc: if report.verbose: @@ -797,6 +871,8 @@ def format_file_in_place( mode: Mode, write_back: WriteBack = WriteBack.NO, lock: Any = None, # multiprocessing.Manager().Lock() is some crazy proxy + *, + lines: Collection[Tuple[int, int]] = (), ) -> bool: """Format file under `src` path. Return True if changed. @@ -809,14 +885,16 @@ def format_file_in_place( elif src.suffix == ".ipynb": mode = replace(mode, is_ipynb=True) - then = datetime.utcfromtimestamp(src.stat().st_mtime) + then = datetime.fromtimestamp(src.stat().st_mtime, timezone.utc) header = b"" with open(src, "rb") as buf: if mode.skip_source_first_line: header = buf.readline() src_contents, encoding, newline = decode_bytes(buf.read()) try: - dst_contents = format_file_contents(src_contents, fast=fast, mode=mode) + dst_contents = format_file_contents( + src_contents, fast=fast, mode=mode, lines=lines + ) except NothingChanged: return False except JSONDecodeError: @@ -830,9 +908,9 @@ def format_file_in_place( with open(src, "w", encoding=encoding, newline=newline) as f: f.write(dst_contents) elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF): - now = datetime.utcnow() - src_name = f"{src}\t{then} +0000" - dst_name = f"{src}\t{now} +0000" + now = datetime.now(timezone.utc) + src_name = f"{src}\t{then}" + dst_name = f"{src}\t{now}" if mode.is_ipynb: diff_contents = ipynb_diff(src_contents, dst_contents, src_name, dst_name) else: @@ -861,6 +939,7 @@ def format_stdin_to_stdout( content: Optional[str] = None, write_back: WriteBack = WriteBack.NO, mode: Mode, + lines: Collection[Tuple[int, int]] = (), ) -> bool: """Format file on stdin. Return True if changed. @@ -870,7 +949,7 @@ def format_stdin_to_stdout( write a diff to stdout. The `mode` argument is passed to :func:`format_file_contents`. """ - then = datetime.utcnow() + then = datetime.now(timezone.utc) if content is None: src, encoding, newline = decode_bytes(sys.stdin.buffer.read()) @@ -879,7 +958,7 @@ def format_stdin_to_stdout( dst = src try: - dst = format_file_contents(src, fast=fast, mode=mode) + dst = format_file_contents(src, fast=fast, mode=mode, lines=lines) return True except NothingChanged: @@ -895,9 +974,9 @@ def format_stdin_to_stdout( dst += "\n" f.write(dst) elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF): - now = datetime.utcnow() - src_name = f"STDIN\t{then} +0000" - dst_name = f"STDOUT\t{now} +0000" + now = datetime.now(timezone.utc) + src_name = f"STDIN\t{then}" + dst_name = f"STDOUT\t{now}" d = diff(src, dst, src_name, dst_name) if write_back == WriteBack.COLOR_DIFF: d = color_diff(d) @@ -907,7 +986,11 @@ def format_stdin_to_stdout( def check_stability_and_equivalence( - src_contents: str, dst_contents: str, *, mode: Mode + src_contents: str, + dst_contents: str, + *, + mode: Mode, + lines: Collection[Tuple[int, int]] = (), ) -> None: """Perform stability and equivalence checks. @@ -916,10 +999,16 @@ def check_stability_and_equivalence( content differently. """ assert_equivalent(src_contents, dst_contents) - assert_stable(src_contents, dst_contents, mode=mode) + assert_stable(src_contents, dst_contents, mode=mode, lines=lines) -def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileContent: +def format_file_contents( + src_contents: str, + *, + fast: bool, + mode: Mode, + lines: Collection[Tuple[int, int]] = (), +) -> FileContent: """Reformat contents of a file and return new contents. If `fast` is False, additionally confirm that the reformatted code is @@ -929,13 +1018,15 @@ def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileCo if mode.is_ipynb: dst_contents = format_ipynb_string(src_contents, fast=fast, mode=mode) else: - dst_contents = format_str(src_contents, mode=mode) + dst_contents = format_str(src_contents, mode=mode, lines=lines) if src_contents == dst_contents: raise NothingChanged if not fast and not mode.is_ipynb: # Jupyter notebooks will already have been checked above. - check_stability_and_equivalence(src_contents, dst_contents, mode=mode) + check_stability_and_equivalence( + src_contents, dst_contents, mode=mode, lines=lines + ) return dst_contents @@ -1046,7 +1137,9 @@ def format_ipynb_string(src_contents: str, *, fast: bool, mode: Mode) -> FileCon raise NothingChanged -def format_str(src_contents: str, *, mode: Mode) -> str: +def format_str( + src_contents: str, *, mode: Mode, lines: Collection[Tuple[int, int]] = () +) -> str: """Reformat a string and return new contents. `mode` determines formatting options, such as how many characters per line are @@ -1076,16 +1169,20 @@ def f( hey """ - dst_contents = _format_str_once(src_contents, mode=mode) + dst_contents = _format_str_once(src_contents, mode=mode, lines=lines) # Forced second pass to work around optional trailing commas (becoming # forced trailing commas on pass 2) interacting differently with optional # parentheses. Admittedly ugly. if src_contents != dst_contents: - return _format_str_once(dst_contents, mode=mode) + if lines: + lines = adjusted_lines(lines, src_contents, dst_contents) + return _format_str_once(dst_contents, mode=mode, lines=lines) return dst_contents -def _format_str_once(src_contents: str, *, mode: Mode) -> str: +def _format_str_once( + src_contents: str, *, mode: Mode, lines: Collection[Tuple[int, int]] = () +) -> str: src_node = lib2to3_parse(src_contents.lstrip(), mode.target_versions) dst_blocks: List[LinesBlock] = [] if mode.target_versions: @@ -1099,8 +1196,12 @@ def _format_str_once(src_contents: str, *, mode: Mode) -> str: for feature in {Feature.PARENTHESIZED_CONTEXT_MANAGERS} if supports_feature(versions, feature) } - normalize_fmt_off(src_node) - lines = LineGenerator(mode=mode, features=context_manager_features) + normalize_fmt_off(src_node, mode, lines) + if lines: + # This should be called after normalize_fmt_off. + convert_unchanged_lines(src_node, lines) + + line_generator = LineGenerator(mode=mode, features=context_manager_features) elt = EmptyLineTracker(mode=mode) split_line_features = { feature @@ -1108,7 +1209,7 @@ def _format_str_once(src_contents: str, *, mode: Mode) -> str: if supports_feature(versions, feature) } block: Optional[LinesBlock] = None - for current_line in lines.visit(src_node): + for current_line in line_generator.visit(src_node): block = elt.maybe_empty_lines(current_line) dst_blocks.append(block) for line in transform_line( @@ -1250,7 +1351,7 @@ def get_features_used( # noqa: C901 if ( len(atom_children) == 3 and atom_children[0].type == token.LPAR - and atom_children[1].type == syms.testlist_gexp + and _contains_asexpr(atom_children[1]) and atom_children[2].type == token.RPAR ): features.add(Feature.PARENTHESIZED_CONTEXT_MANAGERS) @@ -1277,9 +1378,28 @@ def get_features_used( # noqa: C901 ): features.add(Feature.VARIADIC_GENERICS) + elif n.type in (syms.type_stmt, syms.typeparams): + features.add(Feature.TYPE_PARAMS) + return features +def _contains_asexpr(node: Union[Node, Leaf]) -> bool: + """Return True if `node` contains an as-pattern.""" + if node.type == syms.asexpr_test: + return True + elif node.type == syms.atom: + if ( + len(node.children) == 3 + and node.children[0].type == token.LPAR + and node.children[2].type == token.RPAR + ): + return _contains_asexpr(node.children[1]) + elif node.type == syms.testlist_gexp: + return any(_contains_asexpr(child) for child in node.children) + return False + + def detect_target_versions( node: Node, *, future_imports: Optional[Set[str]] = None ) -> Set[TargetVersion]: @@ -1373,12 +1493,21 @@ def assert_equivalent(src: str, dst: str) -> None: ) from None -def assert_stable(src: str, dst: str, mode: Mode) -> None: +def assert_stable( + src: str, dst: str, mode: Mode, *, lines: Collection[Tuple[int, int]] = () +) -> None: """Raise AssertionError if `dst` reformats differently the second time.""" + if lines: + # Formatting specified lines requires `adjusted_lines` to map original lines + # to the formatted lines before re-formatting the previously formatted result. + # Due to less-ideal diff algorithm, some edge cases produce incorrect new line + # ranges. Hence for now, we skip the stable check. + # See https://github.com/psf/black/issues/4033 for context. + return # We shouldn't call format_str() here, because that formats the string # twice and may hide a bug where we bounce back and forth between two # versions. - newdst = _format_str_once(dst, mode=mode) + newdst = _format_str_once(dst, mode=mode, lines=lines) if dst != newdst: log = dump_to_file( str(mode), @@ -1401,40 +1530,6 @@ def nullcontext() -> Iterator[None]: yield -def patch_click() -> None: - """Make Click not crash on Python 3.6 with LANG=C. - - On certain misconfigured environments, Python 3 selects the ASCII encoding as the - default which restricts paths that it can access during the lifetime of the - application. Click refuses to work in this scenario by raising a RuntimeError. - - In case of Black the likelihood that non-ASCII characters are going to be used in - file paths is minimal since it's Python source code. Moreover, this crash was - spurious on Python 3.7 thanks to PEP 538 and PEP 540. - """ - modules: List[Any] = [] - try: - from click import core - except ImportError: - pass - else: - modules.append(core) - try: - # Removed in Click 8.1.0 and newer; we keep this around for users who have - # older versions installed. - from click import _unicodefun # type: ignore - except ImportError: - pass - else: - modules.append(_unicodefun) - - for module in modules: - if hasattr(module, "_verify_python3_env"): - module._verify_python3_env = lambda: None - if hasattr(module, "_verify_python_env"): - module._verify_python_env = lambda: None - - def patched_main() -> None: # PyInstaller patches multiprocessing to need freeze_support() even in non-Windows # environments so just assume we always need to call it if frozen. @@ -1443,7 +1538,6 @@ def patched_main() -> None: freeze_support() - patch_click() main() diff --git a/src/black/_width_table.py b/src/black/_width_table.py new file mode 100644 index 00000000000..f3304e48ed0 --- /dev/null +++ b/src/black/_width_table.py @@ -0,0 +1,478 @@ +# Generated by make_width_table.py +# wcwidth 0.2.6 +# Unicode 15.0.0 +from typing import Final, List, Tuple + +WIDTH_TABLE: Final[List[Tuple[int, int, int]]] = [ + (0, 0, 0), + (1, 31, -1), + (127, 159, -1), + (768, 879, 0), + (1155, 1161, 0), + (1425, 1469, 0), + (1471, 1471, 0), + (1473, 1474, 0), + (1476, 1477, 0), + (1479, 1479, 0), + (1552, 1562, 0), + (1611, 1631, 0), + (1648, 1648, 0), + (1750, 1756, 0), + (1759, 1764, 0), + (1767, 1768, 0), + (1770, 1773, 0), + (1809, 1809, 0), + (1840, 1866, 0), + (1958, 1968, 0), + (2027, 2035, 0), + (2045, 2045, 0), + (2070, 2073, 0), + (2075, 2083, 0), + (2085, 2087, 0), + (2089, 2093, 0), + (2137, 2139, 0), + (2200, 2207, 0), + (2250, 2273, 0), + (2275, 2306, 0), + (2362, 2362, 0), + (2364, 2364, 0), + (2369, 2376, 0), + (2381, 2381, 0), + (2385, 2391, 0), + (2402, 2403, 0), + (2433, 2433, 0), + (2492, 2492, 0), + (2497, 2500, 0), + (2509, 2509, 0), + (2530, 2531, 0), + (2558, 2558, 0), + (2561, 2562, 0), + (2620, 2620, 0), + (2625, 2626, 0), + (2631, 2632, 0), + (2635, 2637, 0), + (2641, 2641, 0), + (2672, 2673, 0), + (2677, 2677, 0), + (2689, 2690, 0), + (2748, 2748, 0), + (2753, 2757, 0), + (2759, 2760, 0), + (2765, 2765, 0), + (2786, 2787, 0), + (2810, 2815, 0), + (2817, 2817, 0), + (2876, 2876, 0), + (2879, 2879, 0), + (2881, 2884, 0), + (2893, 2893, 0), + (2901, 2902, 0), + (2914, 2915, 0), + (2946, 2946, 0), + (3008, 3008, 0), + (3021, 3021, 0), + (3072, 3072, 0), + (3076, 3076, 0), + (3132, 3132, 0), + (3134, 3136, 0), + (3142, 3144, 0), + (3146, 3149, 0), + (3157, 3158, 0), + (3170, 3171, 0), + (3201, 3201, 0), + (3260, 3260, 0), + (3263, 3263, 0), + (3270, 3270, 0), + (3276, 3277, 0), + (3298, 3299, 0), + (3328, 3329, 0), + (3387, 3388, 0), + (3393, 3396, 0), + (3405, 3405, 0), + (3426, 3427, 0), + (3457, 3457, 0), + (3530, 3530, 0), + (3538, 3540, 0), + (3542, 3542, 0), + (3633, 3633, 0), + (3636, 3642, 0), + (3655, 3662, 0), + (3761, 3761, 0), + (3764, 3772, 0), + (3784, 3790, 0), + (3864, 3865, 0), + (3893, 3893, 0), + (3895, 3895, 0), + (3897, 3897, 0), + (3953, 3966, 0), + (3968, 3972, 0), + (3974, 3975, 0), + (3981, 3991, 0), + (3993, 4028, 0), + (4038, 4038, 0), + (4141, 4144, 0), + (4146, 4151, 0), + (4153, 4154, 0), + (4157, 4158, 0), + (4184, 4185, 0), + (4190, 4192, 0), + (4209, 4212, 0), + (4226, 4226, 0), + (4229, 4230, 0), + (4237, 4237, 0), + (4253, 4253, 0), + (4352, 4447, 2), + (4957, 4959, 0), + (5906, 5908, 0), + (5938, 5939, 0), + (5970, 5971, 0), + (6002, 6003, 0), + (6068, 6069, 0), + (6071, 6077, 0), + (6086, 6086, 0), + (6089, 6099, 0), + (6109, 6109, 0), + (6155, 6157, 0), + (6159, 6159, 0), + (6277, 6278, 0), + (6313, 6313, 0), + (6432, 6434, 0), + (6439, 6440, 0), + (6450, 6450, 0), + (6457, 6459, 0), + (6679, 6680, 0), + (6683, 6683, 0), + (6742, 6742, 0), + (6744, 6750, 0), + (6752, 6752, 0), + (6754, 6754, 0), + (6757, 6764, 0), + (6771, 6780, 0), + (6783, 6783, 0), + (6832, 6862, 0), + (6912, 6915, 0), + (6964, 6964, 0), + (6966, 6970, 0), + (6972, 6972, 0), + (6978, 6978, 0), + (7019, 7027, 0), + (7040, 7041, 0), + (7074, 7077, 0), + (7080, 7081, 0), + (7083, 7085, 0), + (7142, 7142, 0), + (7144, 7145, 0), + (7149, 7149, 0), + (7151, 7153, 0), + (7212, 7219, 0), + (7222, 7223, 0), + (7376, 7378, 0), + (7380, 7392, 0), + (7394, 7400, 0), + (7405, 7405, 0), + (7412, 7412, 0), + (7416, 7417, 0), + (7616, 7679, 0), + (8203, 8207, 0), + (8232, 8238, 0), + (8288, 8291, 0), + (8400, 8432, 0), + (8986, 8987, 2), + (9001, 9002, 2), + (9193, 9196, 2), + (9200, 9200, 2), + (9203, 9203, 2), + (9725, 9726, 2), + (9748, 9749, 2), + (9800, 9811, 2), + (9855, 9855, 2), + (9875, 9875, 2), + (9889, 9889, 2), + (9898, 9899, 2), + (9917, 9918, 2), + (9924, 9925, 2), + (9934, 9934, 2), + (9940, 9940, 2), + (9962, 9962, 2), + (9970, 9971, 2), + (9973, 9973, 2), + (9978, 9978, 2), + (9981, 9981, 2), + (9989, 9989, 2), + (9994, 9995, 2), + (10024, 10024, 2), + (10060, 10060, 2), + (10062, 10062, 2), + (10067, 10069, 2), + (10071, 10071, 2), + (10133, 10135, 2), + (10160, 10160, 2), + (10175, 10175, 2), + (11035, 11036, 2), + (11088, 11088, 2), + (11093, 11093, 2), + (11503, 11505, 0), + (11647, 11647, 0), + (11744, 11775, 0), + (11904, 11929, 2), + (11931, 12019, 2), + (12032, 12245, 2), + (12272, 12283, 2), + (12288, 12329, 2), + (12330, 12333, 0), + (12334, 12350, 2), + (12353, 12438, 2), + (12441, 12442, 0), + (12443, 12543, 2), + (12549, 12591, 2), + (12593, 12686, 2), + (12688, 12771, 2), + (12784, 12830, 2), + (12832, 12871, 2), + (12880, 19903, 2), + (19968, 42124, 2), + (42128, 42182, 2), + (42607, 42610, 0), + (42612, 42621, 0), + (42654, 42655, 0), + (42736, 42737, 0), + (43010, 43010, 0), + (43014, 43014, 0), + (43019, 43019, 0), + (43045, 43046, 0), + (43052, 43052, 0), + (43204, 43205, 0), + (43232, 43249, 0), + (43263, 43263, 0), + (43302, 43309, 0), + (43335, 43345, 0), + (43360, 43388, 2), + (43392, 43394, 0), + (43443, 43443, 0), + (43446, 43449, 0), + (43452, 43453, 0), + (43493, 43493, 0), + (43561, 43566, 0), + (43569, 43570, 0), + (43573, 43574, 0), + (43587, 43587, 0), + (43596, 43596, 0), + (43644, 43644, 0), + (43696, 43696, 0), + (43698, 43700, 0), + (43703, 43704, 0), + (43710, 43711, 0), + (43713, 43713, 0), + (43756, 43757, 0), + (43766, 43766, 0), + (44005, 44005, 0), + (44008, 44008, 0), + (44013, 44013, 0), + (44032, 55203, 2), + (63744, 64255, 2), + (64286, 64286, 0), + (65024, 65039, 0), + (65040, 65049, 2), + (65056, 65071, 0), + (65072, 65106, 2), + (65108, 65126, 2), + (65128, 65131, 2), + (65281, 65376, 2), + (65504, 65510, 2), + (66045, 66045, 0), + (66272, 66272, 0), + (66422, 66426, 0), + (68097, 68099, 0), + (68101, 68102, 0), + (68108, 68111, 0), + (68152, 68154, 0), + (68159, 68159, 0), + (68325, 68326, 0), + (68900, 68903, 0), + (69291, 69292, 0), + (69373, 69375, 0), + (69446, 69456, 0), + (69506, 69509, 0), + (69633, 69633, 0), + (69688, 69702, 0), + (69744, 69744, 0), + (69747, 69748, 0), + (69759, 69761, 0), + (69811, 69814, 0), + (69817, 69818, 0), + (69826, 69826, 0), + (69888, 69890, 0), + (69927, 69931, 0), + (69933, 69940, 0), + (70003, 70003, 0), + (70016, 70017, 0), + (70070, 70078, 0), + (70089, 70092, 0), + (70095, 70095, 0), + (70191, 70193, 0), + (70196, 70196, 0), + (70198, 70199, 0), + (70206, 70206, 0), + (70209, 70209, 0), + (70367, 70367, 0), + (70371, 70378, 0), + (70400, 70401, 0), + (70459, 70460, 0), + (70464, 70464, 0), + (70502, 70508, 0), + (70512, 70516, 0), + (70712, 70719, 0), + (70722, 70724, 0), + (70726, 70726, 0), + (70750, 70750, 0), + (70835, 70840, 0), + (70842, 70842, 0), + (70847, 70848, 0), + (70850, 70851, 0), + (71090, 71093, 0), + (71100, 71101, 0), + (71103, 71104, 0), + (71132, 71133, 0), + (71219, 71226, 0), + (71229, 71229, 0), + (71231, 71232, 0), + (71339, 71339, 0), + (71341, 71341, 0), + (71344, 71349, 0), + (71351, 71351, 0), + (71453, 71455, 0), + (71458, 71461, 0), + (71463, 71467, 0), + (71727, 71735, 0), + (71737, 71738, 0), + (71995, 71996, 0), + (71998, 71998, 0), + (72003, 72003, 0), + (72148, 72151, 0), + (72154, 72155, 0), + (72160, 72160, 0), + (72193, 72202, 0), + (72243, 72248, 0), + (72251, 72254, 0), + (72263, 72263, 0), + (72273, 72278, 0), + (72281, 72283, 0), + (72330, 72342, 0), + (72344, 72345, 0), + (72752, 72758, 0), + (72760, 72765, 0), + (72767, 72767, 0), + (72850, 72871, 0), + (72874, 72880, 0), + (72882, 72883, 0), + (72885, 72886, 0), + (73009, 73014, 0), + (73018, 73018, 0), + (73020, 73021, 0), + (73023, 73029, 0), + (73031, 73031, 0), + (73104, 73105, 0), + (73109, 73109, 0), + (73111, 73111, 0), + (73459, 73460, 0), + (73472, 73473, 0), + (73526, 73530, 0), + (73536, 73536, 0), + (73538, 73538, 0), + (78912, 78912, 0), + (78919, 78933, 0), + (92912, 92916, 0), + (92976, 92982, 0), + (94031, 94031, 0), + (94095, 94098, 0), + (94176, 94179, 2), + (94180, 94180, 0), + (94192, 94193, 2), + (94208, 100343, 2), + (100352, 101589, 2), + (101632, 101640, 2), + (110576, 110579, 2), + (110581, 110587, 2), + (110589, 110590, 2), + (110592, 110882, 2), + (110898, 110898, 2), + (110928, 110930, 2), + (110933, 110933, 2), + (110948, 110951, 2), + (110960, 111355, 2), + (113821, 113822, 0), + (118528, 118573, 0), + (118576, 118598, 0), + (119143, 119145, 0), + (119163, 119170, 0), + (119173, 119179, 0), + (119210, 119213, 0), + (119362, 119364, 0), + (121344, 121398, 0), + (121403, 121452, 0), + (121461, 121461, 0), + (121476, 121476, 0), + (121499, 121503, 0), + (121505, 121519, 0), + (122880, 122886, 0), + (122888, 122904, 0), + (122907, 122913, 0), + (122915, 122916, 0), + (122918, 122922, 0), + (123023, 123023, 0), + (123184, 123190, 0), + (123566, 123566, 0), + (123628, 123631, 0), + (124140, 124143, 0), + (125136, 125142, 0), + (125252, 125258, 0), + (126980, 126980, 2), + (127183, 127183, 2), + (127374, 127374, 2), + (127377, 127386, 2), + (127488, 127490, 2), + (127504, 127547, 2), + (127552, 127560, 2), + (127568, 127569, 2), + (127584, 127589, 2), + (127744, 127776, 2), + (127789, 127797, 2), + (127799, 127868, 2), + (127870, 127891, 2), + (127904, 127946, 2), + (127951, 127955, 2), + (127968, 127984, 2), + (127988, 127988, 2), + (127992, 128062, 2), + (128064, 128064, 2), + (128066, 128252, 2), + (128255, 128317, 2), + (128331, 128334, 2), + (128336, 128359, 2), + (128378, 128378, 2), + (128405, 128406, 2), + (128420, 128420, 2), + (128507, 128591, 2), + (128640, 128709, 2), + (128716, 128716, 2), + (128720, 128722, 2), + (128725, 128727, 2), + (128732, 128735, 2), + (128747, 128748, 2), + (128756, 128764, 2), + (128992, 129003, 2), + (129008, 129008, 2), + (129292, 129338, 2), + (129340, 129349, 2), + (129351, 129535, 2), + (129648, 129660, 2), + (129664, 129672, 2), + (129680, 129725, 2), + (129727, 129733, 2), + (129742, 129755, 2), + (129760, 129768, 2), + (129776, 129784, 2), + (131072, 196605, 2), + (196608, 262141, 2), + (917760, 917999, 0), +] diff --git a/src/black/brackets.py b/src/black/brackets.py index 343f0608d50..3020cc0d390 100644 --- a/src/black/brackets.py +++ b/src/black/brackets.py @@ -1,13 +1,7 @@ """Builds on top of nodes.py to track brackets.""" -import sys from dataclasses import dataclass, field -from typing import Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union - -if sys.version_info < (3, 8): - from typing_extensions import Final -else: - from typing import Final +from typing import Dict, Final, Iterable, List, Optional, Sequence, Set, Tuple, Union from black.nodes import ( BRACKET, @@ -133,6 +127,13 @@ def mark(self, leaf: Leaf) -> None: self.maybe_increment_lambda_arguments(leaf) self.maybe_increment_for_loop_variable(leaf) + def any_open_for_or_lambda(self) -> bool: + """Return True if there is an open for or lambda expression on the line. + + See maybe_increment_for_loop_variable and maybe_increment_lambda_arguments + for details.""" + return bool(self._for_loop_depths or self._lambda_argument_depths) + def any_open_brackets(self) -> bool: """Return True if there is an yet unmatched open bracket on the line.""" return bool(self.bracket_match) diff --git a/src/black/cache.py b/src/black/cache.py index 9455ff44772..6baa096baca 100644 --- a/src/black/cache.py +++ b/src/black/cache.py @@ -1,21 +1,29 @@ """Caching of formatted files with feature-based invalidation.""" +import hashlib import os import pickle +import sys import tempfile +from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, Iterable, Set, Tuple +from typing import Dict, Iterable, NamedTuple, Set, Tuple from platformdirs import user_cache_dir from _black_version import version as __version__ from black.mode import Mode -# types -Timestamp = float -FileSize = int -CacheInfo = Tuple[Timestamp, FileSize] -Cache = Dict[str, CacheInfo] +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + + +class FileData(NamedTuple): + st_mtime: float + st_size: int + hash: str def get_cache_dir() -> Path: @@ -29,69 +37,107 @@ def get_cache_dir() -> Path: repeated calls. """ # NOTE: Function mostly exists as a clean way to test getting the cache directory. - default_cache_dir = user_cache_dir("black", version=__version__) + default_cache_dir = user_cache_dir("black") cache_dir = Path(os.environ.get("BLACK_CACHE_DIR", default_cache_dir)) + cache_dir = cache_dir / __version__ return cache_dir CACHE_DIR = get_cache_dir() -def read_cache(mode: Mode) -> Cache: - """Read the cache if it exists and is well formed. - - If it is not well formed, the call to write_cache later should resolve the issue. - """ - cache_file = get_cache_file(mode) - if not cache_file.exists(): - return {} - - with cache_file.open("rb") as fobj: - try: - cache: Cache = pickle.load(fobj) - except (pickle.UnpicklingError, ValueError, IndexError): - return {} - - return cache - - def get_cache_file(mode: Mode) -> Path: return CACHE_DIR / f"cache.{mode.get_cache_key()}.pickle" -def get_cache_info(path: Path) -> CacheInfo: - """Return the information used to check if a file is already formatted or not.""" - stat = path.stat() - return stat.st_mtime, stat.st_size - - -def filter_cached(cache: Cache, sources: Iterable[Path]) -> Tuple[Set[Path], Set[Path]]: - """Split an iterable of paths in `sources` into two sets. - - The first contains paths of files that modified on disk or are not in the - cache. The other contains paths to non-modified files. - """ - todo, done = set(), set() - for src in sources: - res_src = src.resolve() - if cache.get(str(res_src)) != get_cache_info(res_src): - todo.add(src) - else: - done.add(src) - return todo, done - - -def write_cache(cache: Cache, sources: Iterable[Path], mode: Mode) -> None: - """Update the cache file.""" - cache_file = get_cache_file(mode) - try: - CACHE_DIR.mkdir(parents=True, exist_ok=True) - new_cache = { - **cache, - **{str(src.resolve()): get_cache_info(src) for src in sources}, - } - with tempfile.NamedTemporaryFile(dir=str(cache_file.parent), delete=False) as f: - pickle.dump(new_cache, f, protocol=4) - os.replace(f.name, cache_file) - except OSError: - pass +@dataclass +class Cache: + mode: Mode + cache_file: Path + file_data: Dict[str, FileData] = field(default_factory=dict) + + @classmethod + def read(cls, mode: Mode) -> Self: + """Read the cache if it exists and is well formed. + + If it is not well formed, the call to write later should + resolve the issue. + """ + cache_file = get_cache_file(mode) + if not cache_file.exists(): + return cls(mode, cache_file) + + with cache_file.open("rb") as fobj: + try: + data: Dict[str, Tuple[float, int, str]] = pickle.load(fobj) + file_data = {k: FileData(*v) for k, v in data.items()} + except (pickle.UnpicklingError, ValueError, IndexError): + return cls(mode, cache_file) + + return cls(mode, cache_file, file_data) + + @staticmethod + def hash_digest(path: Path) -> str: + """Return hash digest for path.""" + + data = path.read_bytes() + return hashlib.sha256(data).hexdigest() + + @staticmethod + def get_file_data(path: Path) -> FileData: + """Return file data for path.""" + + stat = path.stat() + hash = Cache.hash_digest(path) + return FileData(stat.st_mtime, stat.st_size, hash) + + def is_changed(self, source: Path) -> bool: + """Check if source has changed compared to cached version.""" + res_src = source.resolve() + old = self.file_data.get(str(res_src)) + if old is None: + return True + + st = res_src.stat() + if st.st_size != old.st_size: + return True + if int(st.st_mtime) != int(old.st_mtime): + new_hash = Cache.hash_digest(res_src) + if new_hash != old.hash: + return True + return False + + def filtered_cached(self, sources: Iterable[Path]) -> Tuple[Set[Path], Set[Path]]: + """Split an iterable of paths in `sources` into two sets. + + The first contains paths of files that modified on disk or are not in the + cache. The other contains paths to non-modified files. + """ + changed: Set[Path] = set() + done: Set[Path] = set() + for src in sources: + if self.is_changed(src): + changed.add(src) + else: + done.add(src) + return changed, done + + def write(self, sources: Iterable[Path]) -> None: + """Update the cache file data and write a new cache file.""" + self.file_data.update( + **{str(src.resolve()): Cache.get_file_data(src) for src in sources} + ) + try: + CACHE_DIR.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + dir=str(self.cache_file.parent), delete=False + ) as f: + # We store raw tuples in the cache because pickling NamedTuples + # doesn't work with mypyc on Python 3.8, and because it's faster. + data: Dict[str, Tuple[float, int, str]] = { + k: (*v,) for k, v in self.file_data.items() + } + pickle.dump(data, f, protocol=4) + os.replace(f.name, self.cache_file) + except OSError: + pass diff --git a/src/black/comments.py b/src/black/comments.py index 7cf15bf67b3..25413121199 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -1,20 +1,16 @@ import re -import sys from dataclasses import dataclass from functools import lru_cache -from typing import Iterator, List, Optional, Union - -if sys.version_info >= (3, 8): - from typing import Final -else: - from typing_extensions import Final +from typing import Collection, Final, Iterator, List, Optional, Tuple, Union +from black.mode import Mode, Preview from black.nodes import ( CLOSING_BRACKETS, STANDALONE_COMMENT, WHITESPACE, container_of, first_leaf_of, + make_simple_prefix, preceding_leaf, syms, ) @@ -26,10 +22,11 @@ FMT_OFF: Final = {"# fmt: off", "# fmt:off", "# yapf: disable"} FMT_SKIP: Final = {"# fmt: skip", "# fmt:skip"} -FMT_PASS: Final = {*FMT_OFF, *FMT_SKIP} FMT_ON: Final = {"# fmt: on", "# fmt:on", "# yapf: enable"} COMMENT_EXCEPTIONS = " !:#'" +_COMMENT_PREFIX = "# " +_COMMENT_LIST_SEPARATOR = ";" @dataclass @@ -48,6 +45,7 @@ class ProtoComment: value: str # content of the comment newlines: int # how many newlines before the comment consumed: int # how many characters of the original leaf's prefix did we consume + form_feed: bool # is there a form feed before the comment def generate_comments(leaf: LN) -> Iterator[Leaf]: @@ -69,8 +67,12 @@ def generate_comments(leaf: LN) -> Iterator[Leaf]: Inline comments are emitted as regular token.COMMENT leaves. Standalone are emitted with a fake STANDALONE_COMMENT token identifier. """ + total_consumed = 0 for pc in list_comments(leaf.prefix, is_endmarker=leaf.type == token.ENDMARKER): - yield Leaf(pc.type, pc.value, prefix="\n" * pc.newlines) + total_consumed = pc.consumed + prefix = make_simple_prefix(pc.newlines, pc.form_feed) + yield Leaf(pc.type, pc.value, prefix=prefix) + normalize_trailing_prefix(leaf, total_consumed) @lru_cache(maxsize=4096) @@ -83,11 +85,14 @@ def list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]: consumed = 0 nlines = 0 ignored_lines = 0 - for index, line in enumerate(re.split("\r?\n", prefix)): - consumed += len(line) + 1 # adding the length of the split '\n' - line = line.lstrip() + form_feed = False + for index, full_line in enumerate(re.split("\r?\n", prefix)): + consumed += len(full_line) + 1 # adding the length of the split '\n' + line = full_line.lstrip() if not line: nlines += 1 + if "\f" in full_line: + form_feed = True if not line.startswith("#"): # Escaped newlines outside of a comment are not really newlines at # all. We treat a single-line comment following an escaped newline @@ -103,13 +108,33 @@ def list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]: comment = make_comment(line) result.append( ProtoComment( - type=comment_type, value=comment, newlines=nlines, consumed=consumed + type=comment_type, + value=comment, + newlines=nlines, + consumed=consumed, + form_feed=form_feed, ) ) + form_feed = False nlines = 0 return result +def normalize_trailing_prefix(leaf: LN, total_consumed: int) -> None: + """Normalize the prefix that's left over after generating comments. + + Note: don't use backslashes for formatting or you'll lose your voting rights. + """ + remainder = leaf.prefix[total_consumed:] + if "\\" not in remainder: + nl_count = remainder.count("\n") + form_feed = "\f" in remainder and remainder.endswith("\n") + leaf.prefix = make_simple_prefix(nl_count, form_feed) + return + + leaf.prefix = "" + + def make_comment(content: str) -> str: """Return a consistently formatted comment from the given `content` string. @@ -136,14 +161,18 @@ def make_comment(content: str) -> str: return "#" + content -def normalize_fmt_off(node: Node) -> None: +def normalize_fmt_off( + node: Node, mode: Mode, lines: Collection[Tuple[int, int]] +) -> None: """Convert content between `# fmt: off`/`# fmt: on` into standalone comments.""" try_again = True while try_again: - try_again = convert_one_fmt_off_pair(node) + try_again = convert_one_fmt_off_pair(node, mode, lines) -def convert_one_fmt_off_pair(node: Node) -> bool: +def convert_one_fmt_off_pair( + node: Node, mode: Mode, lines: Collection[Tuple[int, int]] +) -> bool: """Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment. Returns True if a pair was converted. @@ -151,21 +180,27 @@ def convert_one_fmt_off_pair(node: Node) -> bool: for leaf in node.leaves(): previous_consumed = 0 for comment in list_comments(leaf.prefix, is_endmarker=False): - if comment.value not in FMT_PASS: + should_pass_fmt = comment.value in FMT_OFF or _contains_fmt_skip_comment( + comment.value, mode + ) + if not should_pass_fmt: previous_consumed = comment.consumed continue # We only want standalone comments. If there's no previous leaf or # the previous leaf is indentation, it's a standalone comment in # disguise. - if comment.value in FMT_PASS and comment.type != STANDALONE_COMMENT: + if should_pass_fmt and comment.type != STANDALONE_COMMENT: prev = preceding_leaf(leaf) if prev: if comment.value in FMT_OFF and prev.type not in WHITESPACE: continue - if comment.value in FMT_SKIP and prev.type in WHITESPACE: + if ( + _contains_fmt_skip_comment(comment.value, mode) + and prev.type in WHITESPACE + ): continue - ignored_nodes = list(generate_ignored_nodes(leaf, comment)) + ignored_nodes = list(generate_ignored_nodes(leaf, comment, mode)) if not ignored_nodes: continue @@ -174,7 +209,7 @@ def convert_one_fmt_off_pair(node: Node) -> bool: prefix = first.prefix if comment.value in FMT_OFF: first.prefix = prefix[comment.consumed :] - if comment.value in FMT_SKIP: + if _contains_fmt_skip_comment(comment.value, mode): first.prefix = "" standalone_comment_prefix = prefix else: @@ -182,9 +217,20 @@ def convert_one_fmt_off_pair(node: Node) -> bool: prefix[:previous_consumed] + "\n" * comment.newlines ) hidden_value = "".join(str(n) for n in ignored_nodes) + comment_lineno = leaf.lineno - comment.newlines if comment.value in FMT_OFF: + fmt_off_prefix = "" + if len(lines) > 0 and not any( + comment_lineno >= line[0] and comment_lineno <= line[1] + for line in lines + ): + # keeping indentation of comment by preserving original whitespaces. + fmt_off_prefix = prefix.split(comment.value)[0] + if "\n" in fmt_off_prefix: + fmt_off_prefix = fmt_off_prefix.split("\n")[-1] + standalone_comment_prefix += fmt_off_prefix hidden_value = comment.value + "\n" + hidden_value - if comment.value in FMT_SKIP: + if _contains_fmt_skip_comment(comment.value, mode): hidden_value += " " + comment.value if hidden_value.endswith("\n"): # That happens when one of the `ignored_nodes` ended with a NEWLINE @@ -203,6 +249,7 @@ def convert_one_fmt_off_pair(node: Node) -> bool: STANDALONE_COMMENT, hidden_value, prefix=standalone_comment_prefix, + fmt_pass_converted_first_leaf=first_leaf_of(first), ), ) return True @@ -210,13 +257,15 @@ def convert_one_fmt_off_pair(node: Node) -> bool: return False -def generate_ignored_nodes(leaf: Leaf, comment: ProtoComment) -> Iterator[LN]: +def generate_ignored_nodes( + leaf: Leaf, comment: ProtoComment, mode: Mode +) -> Iterator[LN]: """Starting from the container of `leaf`, generate all leaves until `# fmt: on`. If comment is skip, returns leaf only. Stops at the end of the block. """ - if comment.value in FMT_SKIP: + if _contains_fmt_skip_comment(comment.value, mode): yield from _generate_ignored_nodes_from_fmt_skip(leaf, comment) return container: Optional[LN] = container_of(leaf) @@ -332,3 +381,32 @@ def contains_pragma_comment(comment_list: List[Leaf]) -> bool: return True return False + + +def _contains_fmt_skip_comment(comment_line: str, mode: Mode) -> bool: + """ + Checks if the given comment contains FMT_SKIP alone or paired with other comments. + Matching styles: + # fmt:skip <-- single comment + # noqa:XXX # fmt:skip # a nice line <-- multiple comments (Preview) + # pylint:XXX; fmt:skip <-- list of comments (; separated, Preview) + """ + semantic_comment_blocks = ( + [ + comment_line, + *[ + _COMMENT_PREFIX + comment.strip() + for comment in comment_line.split(_COMMENT_PREFIX)[1:] + ], + *[ + _COMMENT_PREFIX + comment.strip() + for comment in comment_line.strip(_COMMENT_PREFIX).split( + _COMMENT_LIST_SEPARATOR + ) + ], + ] + if Preview.single_line_format_skip_with_multiple_comments in mode + else [comment_line] + ) + + return any(comment in FMT_SKIP for comment in semantic_comment_blocks) diff --git a/src/black/concurrency.py b/src/black/concurrency.py index 1598f51e43f..ff0a8f5fd32 100644 --- a/src/black/concurrency.py +++ b/src/black/concurrency.py @@ -9,6 +9,7 @@ import os import signal import sys +import traceback from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor from multiprocessing import Manager from pathlib import Path @@ -17,7 +18,7 @@ from mypy_extensions import mypyc_attr from black import WriteBack, format_file_in_place -from black.cache import Cache, filter_cached, read_cache, write_cache +from black.cache import Cache from black.mode import Mode from black.output import err from black.report import Changed, Report @@ -37,7 +38,7 @@ def maybe_install_uvloop() -> None: pass -def cancel(tasks: Iterable["asyncio.Task[Any]"]) -> None: +def cancel(tasks: Iterable["asyncio.Future[Any]"]) -> None: """asyncio signal handler that cancels all `tasks` and reports to stderr.""" err("Aborted!") for task in tasks: @@ -80,7 +81,8 @@ def reformat_many( executor: Executor if workers is None: - workers = os.cpu_count() or 1 + workers = int(os.environ.get("BLACK_NUM_WORKERS", 0)) + workers = workers or os.cpu_count() or 1 if sys.platform == "win32": # Work around https://bugs.python.org/issue26903 workers = min(workers, 60) @@ -132,10 +134,9 @@ async def schedule_formatting( `write_back`, `fast`, and `mode` options are passed to :func:`format_file_in_place`. """ - cache: Cache = {} + cache = Cache.read(mode) if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF): - cache = read_cache(mode) - sources, cached = filter_cached(cache, sources) + sources, cached = cache.filtered_cached(sources) for src in sorted(cached): report.done(src, Changed.CACHED) if not sources: @@ -170,8 +171,10 @@ async def schedule_formatting( src = tasks.pop(task) if task.cancelled(): cancelled.append(task) - elif task.exception(): - report.failed(src, str(task.exception())) + elif exc := task.exception(): + if report.verbose: + traceback.print_exception(type(exc), exc, exc.__traceback__) + report.failed(src, str(exc)) else: changed = Changed.YES if task.result() else Changed.NO # If the file was written back or was successfully checked as @@ -184,4 +187,4 @@ async def schedule_formatting( if cancelled: await asyncio.gather(*cancelled, return_exceptions=True) if sources_to_cache: - write_cache(cache, sources_to_cache, mode) + cache.write(sources_to_cache) diff --git a/src/black/const.py b/src/black/const.py index 0e13f31517d..ee466679c70 100644 --- a/src/black/const.py +++ b/src/black/const.py @@ -1,4 +1,4 @@ DEFAULT_LINE_LENGTH = 88 -DEFAULT_EXCLUDES = r"/(\.direnv|\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|venv|\.svn|\.ipynb_checkpoints|_build|buck-out|build|dist|__pypackages__)/" # noqa: B950 +DEFAULT_EXCLUDES = r"/(\.direnv|\.eggs|\.git|\.hg|\.ipynb_checkpoints|\.mypy_cache|\.nox|\.pytest_cache|\.ruff_cache|\.tox|\.svn|\.venv|\.vscode|__pypackages__|_build|buck-out|build|dist|venv)/" # noqa: B950 DEFAULT_INCLUDES = r"(\.pyi?|\.ipynb)$" STDIN_PLACEHOLDER = "__BLACK_STDIN_FILENAME__" diff --git a/src/black/debug.py b/src/black/debug.py index 150b44842dd..cebc48765ba 100644 --- a/src/black/debug.py +++ b/src/black/debug.py @@ -1,5 +1,5 @@ -from dataclasses import dataclass -from typing import Iterator, TypeVar, Union +from dataclasses import dataclass, field +from typing import Any, Iterator, List, TypeVar, Union from black.nodes import Visitor from black.output import out @@ -14,26 +14,33 @@ @dataclass class DebugVisitor(Visitor[T]): tree_depth: int = 0 + list_output: List[str] = field(default_factory=list) + print_output: bool = True + + def out(self, message: str, *args: Any, **kwargs: Any) -> None: + self.list_output.append(message) + if self.print_output: + out(message, *args, **kwargs) def visit_default(self, node: LN) -> Iterator[T]: indent = " " * (2 * self.tree_depth) if isinstance(node, Node): _type = type_repr(node.type) - out(f"{indent}{_type}", fg="yellow") + self.out(f"{indent}{_type}", fg="yellow") self.tree_depth += 1 for child in node.children: yield from self.visit(child) self.tree_depth -= 1 - out(f"{indent}/{_type}", fg="yellow", bold=False) + self.out(f"{indent}/{_type}", fg="yellow", bold=False) else: _type = token.tok_name.get(node.type, str(node.type)) - out(f"{indent}{_type}", fg="blue", nl=False) + self.out(f"{indent}{_type}", fg="blue", nl=False) if node.prefix: # We don't have to handle prefixes for `Node` objects since # that delegates to the first child anyway. - out(f" {node.prefix!r}", fg="green", bold=False, nl=False) - out(f" {node.value!r}", fg="blue", bold=False) + self.out(f" {node.prefix!r}", fg="green", bold=False, nl=False) + self.out(f" {node.value!r}", fg="blue", bold=False) @classmethod def show(cls, code: Union[str, Leaf, Node]) -> None: diff --git a/src/black/files.py b/src/black/files.py index 8c0131126b7..858303ca1a3 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -42,7 +42,7 @@ import colorama # noqa: F401 -@lru_cache() +@lru_cache def find_project_root( srcs: Sequence[str], stdin_filename: Optional[str] = None ) -> Tuple[Path, str]: @@ -89,9 +89,11 @@ def find_project_root( return directory, "file system root" -def find_pyproject_toml(path_search_start: Tuple[str, ...]) -> Optional[str]: +def find_pyproject_toml( + path_search_start: Tuple[str, ...], stdin_filename: Optional[str] = None +) -> Optional[str]: """Find the absolute filepath to a pyproject.toml if it exists""" - path_project_root, _ = find_project_root(path_search_start) + path_project_root, _ = find_project_root(path_search_start, stdin_filename) path_pyproject_toml = path_project_root / "pyproject.toml" if path_pyproject_toml.is_file(): return str(path_pyproject_toml) @@ -210,7 +212,7 @@ def strip_specifier_set(specifier_set: SpecifierSet) -> SpecifierSet: return SpecifierSet(",".join(str(s) for s in specifiers)) -@lru_cache() +@lru_cache def find_user_pyproject_toml() -> Path: r"""Return the path to the top-level user configuration for black. @@ -230,7 +232,7 @@ def find_user_pyproject_toml() -> Path: return user_config_path.resolve() -@lru_cache() +@lru_cache def get_gitignore(root: Path) -> PathSpec: """Return a PathSpec matching gitignore content if present.""" gitignore = root / ".gitignore" @@ -274,15 +276,20 @@ def normalize_path_maybe_ignore( return root_relative_path -def path_is_ignored( - path: Path, gitignore_dict: Dict[Path, PathSpec], report: Report +def _path_is_ignored( + root_relative_path: str, + root: Path, + gitignore_dict: Dict[Path, PathSpec], ) -> bool: + path = root / root_relative_path + # Note that this logic is sensitive to the ordering of gitignore_dict. Callers must + # ensure that gitignore_dict is ordered from least specific to most specific. for gitignore_path, pattern in gitignore_dict.items(): - relative_path = normalize_path_maybe_ignore(path, gitignore_path, report) - if relative_path is None: + try: + relative_path = path.relative_to(gitignore_path).as_posix() + except ValueError: break if pattern.match_file(relative_path): - report.path_ignored(path, "matches a .gitignore file content") return True return False @@ -319,33 +326,38 @@ def gen_python_files( assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}" for child in paths: - normalized_path = normalize_path_maybe_ignore(child, root, report) - if normalized_path is None: - continue + root_relative_path = child.absolute().relative_to(root).as_posix() # First ignore files matching .gitignore, if passed - if gitignore_dict and path_is_ignored(child, gitignore_dict, report): + if gitignore_dict and _path_is_ignored( + root_relative_path, root, gitignore_dict + ): + report.path_ignored(child, "matches a .gitignore file content") continue # Then ignore with `--exclude` `--extend-exclude` and `--force-exclude` options. - normalized_path = "/" + normalized_path + root_relative_path = "/" + root_relative_path if child.is_dir(): - normalized_path += "/" + root_relative_path += "/" - if path_is_excluded(normalized_path, exclude): + if path_is_excluded(root_relative_path, exclude): report.path_ignored(child, "matches the --exclude regular expression") continue - if path_is_excluded(normalized_path, extend_exclude): + if path_is_excluded(root_relative_path, extend_exclude): report.path_ignored( child, "matches the --extend-exclude regular expression" ) continue - if path_is_excluded(normalized_path, force_exclude): + if path_is_excluded(root_relative_path, force_exclude): report.path_ignored(child, "matches the --force-exclude regular expression") continue + normalized_path = normalize_path_maybe_ignore(child, root, report) + if normalized_path is None: + continue + if child.is_dir(): # If gitignore is None, gitignore usage is disabled, while a Falsey # gitignore is when the directory doesn't have a .gitignore file. @@ -371,10 +383,10 @@ def gen_python_files( elif child.is_file(): if child.suffix == ".ipynb" and not jupyter_dependencies_are_installed( - verbose=verbose, quiet=quiet + warn=verbose or not quiet ): continue - include_match = include.search(normalized_path) if include else True + include_match = include.search(root_relative_path) if include else True if include_match: yield child diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index 9e1af757c32..5b2847cb0c4 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -6,6 +6,7 @@ import secrets import sys from functools import lru_cache +from importlib.util import find_spec from typing import Dict, List, Optional, Tuple if sys.version_info >= (3, 10): @@ -16,36 +17,30 @@ from black.output import out from black.report import NothingChanged -TRANSFORMED_MAGICS = frozenset( - ( - "get_ipython().run_cell_magic", - "get_ipython().system", - "get_ipython().getoutput", - "get_ipython().run_line_magic", - ) -) -TOKENS_TO_IGNORE = frozenset( - ( - "ENDMARKER", - "NL", - "NEWLINE", - "COMMENT", - "DEDENT", - "UNIMPORTANT_WS", - "ESCAPED_NL", - ) -) -PYTHON_CELL_MAGICS = frozenset( - ( - "capture", - "prun", - "pypy", - "python", - "python3", - "time", - "timeit", - ) -) +TRANSFORMED_MAGICS = frozenset(( + "get_ipython().run_cell_magic", + "get_ipython().system", + "get_ipython().getoutput", + "get_ipython().run_line_magic", +)) +TOKENS_TO_IGNORE = frozenset(( + "ENDMARKER", + "NL", + "NEWLINE", + "COMMENT", + "DEDENT", + "UNIMPORTANT_WS", + "ESCAPED_NL", +)) +PYTHON_CELL_MAGICS = frozenset(( + "capture", + "prun", + "pypy", + "python", + "python3", + "time", + "timeit", +)) TOKEN_HEX = secrets.token_hex @@ -55,21 +50,18 @@ class Replacement: src: str -@lru_cache() -def jupyter_dependencies_are_installed(*, verbose: bool, quiet: bool) -> bool: - try: - import IPython # noqa:F401 - import tokenize_rt # noqa:F401 - except ModuleNotFoundError: - if verbose or not quiet: - msg = ( - "Skipping .ipynb files as Jupyter dependencies are not installed.\n" - 'You can fix this by running ``pip install "black[jupyter]"``' - ) - out(msg) - return False - else: - return True +@lru_cache +def jupyter_dependencies_are_installed(*, warn: bool) -> bool: + installed = ( + find_spec("tokenize_rt") is not None and find_spec("IPython") is not None + ) + if not installed and warn: + msg = ( + "Skipping .ipynb files as Jupyter dependencies are not installed.\n" + 'You can fix this by running ``pip install "black[jupyter]"``' + ) + out(msg) + return installed def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: @@ -330,7 +322,8 @@ class CellMagicFinder(ast.NodeVisitor): For example, - %%time\nfoo() + %%time\n + foo() would have been transformed to diff --git a/src/black/linegen.py b/src/black/linegen.py index 7afb1733939..245be235231 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -1,8 +1,10 @@ """ Generating lines of code. """ + +import re import sys -from dataclasses import dataclass +from dataclasses import replace from enum import Enum, auto from functools import partial, wraps from typing import Collection, Iterator, List, Optional, Set, Union, cast @@ -16,6 +18,7 @@ from black.comments import FMT_OFF, generate_comments, list_comments from black.lines import ( Line, + RHSResult, append_leaves, can_be_split, can_omit_invisible_parens, @@ -35,9 +38,11 @@ Visitor, ensure_visible, is_arith_like, + is_async_stmt_or_funcdef, is_atom_with_invisible_parens, is_docstring, is_empty_tuple, + is_function_or_class, is_lpar_token, is_multiline_string, is_name_token, @@ -47,6 +52,7 @@ is_stub_body, is_stub_suite, is_tuple_containing_walrus, + is_type_ignore_comment_string, is_vararg, is_walrus_assignment, is_yield, @@ -109,6 +115,17 @@ def line(self, indent: int = 0) -> Iterator[Line]: self.current_line.depth += indent return # Line is empty, don't emit. Creating a new one unnecessary. + if ( + Preview.improved_async_statements_handling in self.mode + and len(self.current_line.leaves) == 1 + and is_async_stmt_or_funcdef(self.current_line.leaves[0]) + ): + # Special case for async def/for/with statements. `visit_async_stmt` + # adds an `ASYNC` leaf then visits the child def/for/with statement + # nodes. Line yields from those nodes shouldn't treat the former + # `ASYNC` leaf as a complete line. + return + complete_line = self.current_line self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent) yield complete_line @@ -133,7 +150,8 @@ def visit_default(self, node: LN) -> Iterator[Line]: self.current_line.append(comment) yield from self.line() - normalize_prefix(node, inside_brackets=any_open_brackets) + if any_open_brackets: + node.prefix = "" if self.mode.string_normalization and node.type == token.STRING: node.value = normalize_string_prefix(node.value) node.value = normalize_string_quotes(node.value) @@ -202,6 +220,18 @@ def visit_stmt( yield from self.visit(child) + def visit_typeparams(self, node: Node) -> Iterator[Line]: + yield from self.visit_default(node) + node.children[0].prefix = "" + + def visit_typevartuple(self, node: Node) -> Iterator[Line]: + yield from self.visit_default(node) + node.children[1].prefix = "" + + def visit_paramspec(self, node: Node) -> Iterator[Line]: + yield from self.visit_default(node) + node.children[1].prefix = "" + def visit_dictsetmaker(self, node: Node) -> Iterator[Line]: if Preview.wrap_long_dict_values_in_parens in self.mode: for i, child in enumerate(node.children): @@ -255,7 +285,9 @@ def visit_match_case(self, node: Node) -> Iterator[Line]: def visit_suite(self, node: Node) -> Iterator[Line]: """Visit a suite.""" - if self.mode.is_pyi and is_stub_suite(node): + if ( + self.mode.is_pyi or Preview.dummy_implementations in self.mode + ) and is_stub_suite(node, self.mode): yield from self.visit(node.children[2]) else: yield from self.visit_default(node) @@ -268,9 +300,12 @@ def visit_simple_stmt(self, node: Node) -> Iterator[Line]: wrap_in_parentheses(node, child, visible=False) prev_type = child.type - is_suite_like = node.parent and node.parent.type in STATEMENT - if is_suite_like: - if self.mode.is_pyi and is_stub_body(node): + if node.parent and node.parent.type in STATEMENT: + if Preview.dummy_implementations in self.mode: + condition = is_function_or_class(node.parent) + else: + condition = self.mode.is_pyi + if condition and is_stub_body(node): yield from self.visit_default(node) else: yield from self.line(+1) @@ -279,9 +314,9 @@ def visit_simple_stmt(self, node: Node) -> Iterator[Line]: else: if ( - not self.mode.is_pyi + not (self.mode.is_pyi or Preview.dummy_implementations in self.mode) or not node.parent - or not is_stub_suite(node.parent) + or not is_stub_suite(node.parent, self.mode) ): yield from self.line() yield from self.visit_default(node) @@ -300,8 +335,11 @@ def visit_async_stmt(self, node: Node) -> Iterator[Line]: break internal_stmt = next(children) - for child in internal_stmt.children: - yield from self.visit(child) + if Preview.improved_async_statements_handling in self.mode: + yield from self.visit(internal_stmt) + else: + for child in internal_stmt.children: + yield from self.visit(child) def visit_decorators(self, node: Node) -> Iterator[Line]: """Visit decorators.""" @@ -364,11 +402,29 @@ def visit_factor(self, node: Node) -> Iterator[Line]: node.insert_child(index, Node(syms.atom, [lpar, operand, rpar])) yield from self.visit_default(node) + def visit_tname(self, node: Node) -> Iterator[Line]: + """ + Add potential parentheses around types in function parameter lists to be made + into real parentheses in case the type hint is too long to fit on a line + Examples: + def foo(a: int, b: float = 7): ... + + -> + + def foo(a: (int), b: (float) = 7): ... + """ + if Preview.parenthesize_long_type_hints in self.mode: + assert len(node.children) == 3 + if maybe_make_parens_invisible_in_atom(node.children[2], parent=node): + wrap_in_parentheses(node, node.children[2], visible=False) + + yield from self.visit_default(node) + def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: if Preview.hex_codes_in_unicode_sequences in self.mode: normalize_unicode_escape_sequences(leaf) - if is_docstring(leaf) and "\\\n" not in leaf.value: + if is_docstring(leaf) and not re.search(r"\\\s*\n", leaf.value): # We're ignoring docstrings with backslash newline escapes because changing # indentation of those changes the AST representation of the code. if self.mode.string_normalization: @@ -465,7 +521,14 @@ def __post_init__(self) -> None: self.visit_except_clause = partial(v, keywords={"except"}, parens={"except"}) self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"}) self.visit_classdef = partial(v, keywords={"class"}, parens=Ø) - self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS) + + # When this is moved out of preview, add ":" directly to ASSIGNMENTS in nodes.py + if Preview.parenthesize_long_type_hints in self.mode: + assignments = ASSIGNMENTS | {":"} + else: + assignments = ASSIGNMENTS + self.visit_expr_stmt = partial(v, keywords=Ø, parens=assignments) + self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"}) self.visit_import_from = partial(v, keywords=Ø, parens={"import"}) self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"}) @@ -477,6 +540,17 @@ def __post_init__(self) -> None: self.visit_case_block = self.visit_match_case +def _hugging_power_ops_line_to_string( + line: Line, + features: Collection[Feature], + mode: Mode, +) -> Optional[str]: + try: + return line_to_string(next(hug_power_op(line, features, mode))) + except CannotTransform: + return None + + def transform_line( line: Line, mode: Mode, features: Collection[Feature] = () ) -> Iterator[Line]: @@ -492,6 +566,14 @@ def transform_line( line_str = line_to_string(line) + # We need the line string when power operators are hugging to determine if we should + # split the line. Default to line_str, if no power operator are present on the line. + line_str_hugging_power_ops = ( + (_hugging_power_ops_line_to_string(line, features, mode) or line_str) + if Preview.fix_power_op_line_length in mode + else line_str + ) + ll = mode.line_length sn = mode.string_normalization string_merge = StringMerger(ll, sn) @@ -505,22 +587,23 @@ def transform_line( and not line.should_split_rhs and not line.magic_trailing_comma and ( - is_line_short_enough(line, line_length=mode.line_length, line_str=line_str) + is_line_short_enough(line, mode=mode, line_str=line_str_hugging_power_ops) or line.contains_unsplittable_type_ignore() ) and not (line.inside_brackets and line.contains_standalone_comments()) + and not line.contains_implicit_multiline_string_with_comments() ): # Only apply basic string preprocessing, since lines shouldn't be split here. if Preview.string_processing in mode: transformers = [string_merge, string_paren_strip] else: transformers = [] - elif line.is_def: + elif line.is_def and not should_split_funcdef_with_rhs(line, mode): transformers = [left_hand_split] else: def _rhs( - self: object, line: Line, features: Collection[Feature] + self: object, line: Line, features: Collection[Feature], mode: Mode ) -> Iterator[Line]: """Wraps calls to `right_hand_split`. @@ -529,14 +612,12 @@ def _rhs( bracket pair instead. """ for omit in generate_trailers_to_omit(line, mode.line_length): - lines = list( - right_hand_split(line, mode.line_length, features, omit=omit) - ) + lines = list(right_hand_split(line, mode, features, omit=omit)) # Note: this check is only able to figure out if the first line of the # *current* transformation fits in the line length. This is true only # for simple cases. All others require running more transforms via # `transform_line()`. This check doesn't know if those would succeed. - if is_line_short_enough(lines[0], line_length=mode.line_length): + if is_line_short_enough(lines[0], mode=mode): yield from lines return @@ -544,9 +625,7 @@ def _rhs( # This mostly happens to multiline strings that are by definition # reported as not fitting a single line, as well as lines that contain # trailing commas (those have to be exploded). - yield from right_hand_split( - line, line_length=mode.line_length, features=features - ) + yield from right_hand_split(line, mode, features=features) # HACK: nested functions (like _rhs) compiled by mypyc don't retain their # __name__ attribute which is needed in `run_transformer` further down. @@ -598,13 +677,49 @@ def _rhs( yield line +def should_split_funcdef_with_rhs(line: Line, mode: Mode) -> bool: + """If a funcdef has a magic trailing comma in the return type, then we should first + split the line with rhs to respect the comma. + """ + if Preview.respect_magic_trailing_comma_in_return_type not in mode: + return False + + return_type_leaves: List[Leaf] = [] + in_return_type = False + + for leaf in line.leaves: + if leaf.type == token.COLON: + in_return_type = False + if in_return_type: + return_type_leaves.append(leaf) + if leaf.type == token.RARROW: + in_return_type = True + + # using `bracket_split_build_line` will mess with whitespace, so we duplicate a + # couple lines from it. + result = Line(mode=line.mode, depth=line.depth) + leaves_to_track = get_leaves_inside_matching_brackets(return_type_leaves) + for leaf in return_type_leaves: + result.append( + leaf, + preformatted=True, + track_bracket=id(leaf) in leaves_to_track, + ) + + # we could also return true if the line is too long, and the return type is longer + # than the param list. Or if `should_split_rhs` returns True. + return result.magic_trailing_comma is not None + + class _BracketSplitComponent(Enum): head = auto() body = auto() tail = auto() -def left_hand_split(line: Line, _features: Collection[Feature] = ()) -> Iterator[Line]: +def left_hand_split( + line: Line, _features: Collection[Feature], mode: Mode +) -> Iterator[Line]: """Split line into many lines, starting with the first matching bracket pair. Note: this usually looks weird, only use this for function definitions. @@ -631,7 +746,7 @@ def left_hand_split(line: Line, _features: Collection[Feature] = ()) -> Iterator if leaf.type in OPENING_BRACKETS: matching_bracket = leaf current_leaves = body_leaves - if not matching_bracket: + if not matching_bracket or not tail_leaves: raise CannotSplit("No brackets found") head = bracket_split_build_line( @@ -649,20 +764,9 @@ def left_hand_split(line: Line, _features: Collection[Feature] = ()) -> Iterator yield result -@dataclass -class _RHSResult: - """Intermediate split result from a right hand split.""" - - head: Line - body: Line - tail: Line - opening_bracket: Leaf - closing_bracket: Leaf - - def right_hand_split( line: Line, - line_length: int, + mode: Mode, features: Collection[Feature] = (), omit: Collection[LeafID] = (), ) -> Iterator[Line]: @@ -676,14 +780,14 @@ def right_hand_split( """ rhs_result = _first_right_hand_split(line, omit=omit) yield from _maybe_split_omitting_optional_parens( - rhs_result, line, line_length, features=features, omit=omit + rhs_result, line, mode, features=features, omit=omit ) def _first_right_hand_split( line: Line, omit: Collection[LeafID] = (), -) -> _RHSResult: +) -> RHSResult: """Split the line into head, body, tail starting with the last bracket pair. Note: this function should not have side effects. It's relied upon by @@ -715,23 +819,78 @@ def _first_right_hand_split( tail_leaves.reverse() body_leaves.reverse() head_leaves.reverse() + + body: Optional[Line] = None + if ( + Preview.hug_parens_with_braces_and_square_brackets in line.mode + and tail_leaves[0].value + and tail_leaves[0].opening_bracket is head_leaves[-1] + ): + inner_body_leaves = list(body_leaves) + hugged_opening_leaves: List[Leaf] = [] + hugged_closing_leaves: List[Leaf] = [] + is_unpacking = body_leaves[0].type in [token.STAR, token.DOUBLESTAR] + unpacking_offset: int = 1 if is_unpacking else 0 + while ( + len(inner_body_leaves) >= 2 + unpacking_offset + and inner_body_leaves[-1].type in CLOSING_BRACKETS + and inner_body_leaves[-1].opening_bracket + is inner_body_leaves[unpacking_offset] + ): + if unpacking_offset: + hugged_opening_leaves.append(inner_body_leaves.pop(0)) + unpacking_offset = 0 + hugged_opening_leaves.append(inner_body_leaves.pop(0)) + hugged_closing_leaves.insert(0, inner_body_leaves.pop()) + + if hugged_opening_leaves and inner_body_leaves: + inner_body = bracket_split_build_line( + inner_body_leaves, + line, + hugged_opening_leaves[-1], + component=_BracketSplitComponent.body, + ) + if ( + line.mode.magic_trailing_comma + and inner_body_leaves[-1].type == token.COMMA + ): + should_hug = True + else: + line_length = line.mode.line_length - sum( + len(str(leaf)) + for leaf in hugged_opening_leaves + hugged_closing_leaves + ) + if is_line_short_enough( + inner_body, mode=replace(line.mode, line_length=line_length) + ): + # Do not hug if it fits on a single line. + should_hug = False + else: + should_hug = True + if should_hug: + body_leaves = inner_body_leaves + head_leaves.extend(hugged_opening_leaves) + tail_leaves = hugged_closing_leaves + tail_leaves + body = inner_body # No need to re-calculate the body again later. + head = bracket_split_build_line( head_leaves, line, opening_bracket, component=_BracketSplitComponent.head ) - body = bracket_split_build_line( - body_leaves, line, opening_bracket, component=_BracketSplitComponent.body - ) + if body is None: + body = bracket_split_build_line( + body_leaves, line, opening_bracket, component=_BracketSplitComponent.body + ) tail = bracket_split_build_line( tail_leaves, line, opening_bracket, component=_BracketSplitComponent.tail ) bracket_split_succeeded_or_raise(head, body, tail) - return _RHSResult(head, body, tail, opening_bracket, closing_bracket) + return RHSResult(head, body, tail, opening_bracket, closing_bracket) def _maybe_split_omitting_optional_parens( - rhs: _RHSResult, + rhs: RHSResult, line: Line, - line_length: int, + mode: Mode, features: Collection[Feature] = (), omit: Collection[LeafID] = (), ) -> Iterator[Line]: @@ -746,41 +905,52 @@ def _maybe_split_omitting_optional_parens( # it's not an import (optional parens are the only thing we can split on # in this case; attempting a split without them is a waste of time) and not line.is_import - # there are no standalone comments in the body - and not rhs.body.contains_standalone_comments(0) # and we can actually remove the parens - and can_omit_invisible_parens(rhs.body, line_length) + and can_omit_invisible_parens(rhs, mode.line_length) ): omit = {id(rhs.closing_bracket), *omit} try: - # The _RHSResult Omitting Optional Parens. + # The RHSResult Omitting Optional Parens. rhs_oop = _first_right_hand_split(line, omit=omit) - if not ( + prefer_splitting_rhs_mode = ( Preview.prefer_splitting_right_hand_side_of_assignments in line.mode - # the split is right after `=` - and len(rhs.head.leaves) >= 2 - and rhs.head.leaves[-2].type == token.EQUAL - # the left side of assignement contains brackets - and any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1]) - # the left side of assignment is short enough (the -1 is for the ending - # optional paren) - and is_line_short_enough(rhs.head, line_length=line_length - 1) - # the left side of assignment won't explode further because of magic - # trailing comma - and rhs.head.magic_trailing_comma is None - # the split by omitting optional parens isn't preferred by some other - # reason - and not _prefer_split_rhs_oop(rhs_oop, line_length=line_length) + ) + is_split_right_after_equal = ( + len(rhs.head.leaves) >= 2 and rhs.head.leaves[-2].type == token.EQUAL + ) + rhs_head_contains_brackets = any( + leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1] + ) + # the -1 is for the ending optional paren + rhs_head_short_enough = is_line_short_enough( + rhs.head, mode=replace(mode, line_length=mode.line_length - 1) + ) + rhs_head_explode_blocked_by_magic_trailing_comma = ( + rhs.head.magic_trailing_comma is None + ) + if ( + not ( + prefer_splitting_rhs_mode + and is_split_right_after_equal + and rhs_head_contains_brackets + and rhs_head_short_enough + and rhs_head_explode_blocked_by_magic_trailing_comma + ) + # the omit optional parens split is preferred by some other reason + or _prefer_split_rhs_oop_over_rhs(rhs_oop, rhs, mode) ): yield from _maybe_split_omitting_optional_parens( - rhs_oop, line, line_length, features=features, omit=omit + rhs_oop, line, mode, features=features, omit=omit ) return except CannotSplit as e: - if not ( - can_be_split(rhs.body) - or is_line_short_enough(rhs.body, line_length=line_length) + # For chained assignments we want to use the previous successful split + if line.is_chained_assignment: + pass + + elif not can_be_split(rhs.body) and not is_line_short_enough( + rhs.body, mode=mode ): raise CannotSplit( "Splitting failed, body is still too long and can't be split." @@ -804,10 +974,22 @@ def _maybe_split_omitting_optional_parens( yield result -def _prefer_split_rhs_oop(rhs_oop: _RHSResult, line_length: int) -> bool: +def _prefer_split_rhs_oop_over_rhs( + rhs_oop: RHSResult, rhs: RHSResult, mode: Mode +) -> bool: """ - Returns whether we should prefer the result from a split omitting optional parens. + Returns whether we should prefer the result from a split omitting optional parens + (rhs_oop) over the original (rhs). """ + # If we have multiple targets, we prefer more `=`s on the head vs pushing them to + # the body + rhs_head_equal_count = [leaf.type for leaf in rhs.head.leaves].count(token.EQUAL) + rhs_oop_head_equal_count = [leaf.type for leaf in rhs_oop.head.leaves].count( + token.EQUAL + ) + if rhs_head_equal_count > 1 and rhs_head_equal_count > rhs_oop_head_equal_count: + return False + has_closing_bracket_after_assign = False for leaf in reversed(rhs_oop.head.leaves): if leaf.type == token.EQUAL: @@ -824,7 +1006,7 @@ def _prefer_split_rhs_oop(rhs_oop: _RHSResult, line_length: int) -> bool: # the first line still contains the `=`) any(leaf.type == token.EQUAL for leaf in rhs_oop.head.leaves) # the first line is short enough - and is_line_short_enough(rhs_oop.head, line_length=line_length) + and is_line_short_enough(rhs_oop.head, mode=mode) ) # contains unsplittable type ignore or rhs_oop.head.contains_unsplittable_type_ignore() @@ -880,8 +1062,6 @@ def bracket_split_build_line( result.inside_brackets = True result.depth += 1 if leaves: - # Since body is a new indent level, remove spurious leading whitespace. - normalize_prefix(leaves[0], inside_brackets=True) # Ensure a trailing comma for imports and standalone function arguments, but # be careful not to add one after any comments or within type annotations. no_commas = ( @@ -902,6 +1082,13 @@ def bracket_split_build_line( ) if isinstance(node, Node) and isinstance(node.prev_sibling, Leaf) ) + # Except the false negatives above for PEP 604 unions where we + # can't add the comma. + and not ( + leaves[0].parent + and leaves[0].parent.next_sibling + and leaves[0].parent.next_sibling.type == token.VBAR + ) ) if original.is_import or no_commas: @@ -940,16 +1127,39 @@ def dont_increase_indentation(split_func: Transformer) -> Transformer: """ @wraps(split_func) - def split_wrapper(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]: - for split_line in split_func(line, features): - normalize_prefix(split_line.leaves[0], inside_brackets=True) + def split_wrapper( + line: Line, features: Collection[Feature], mode: Mode + ) -> Iterator[Line]: + for split_line in split_func(line, features, mode): + split_line.leaves[0].prefix = "" yield split_line return split_wrapper +def _get_last_non_comment_leaf(line: Line) -> Optional[int]: + for leaf_idx in range(len(line.leaves) - 1, 0, -1): + if line.leaves[leaf_idx].type != STANDALONE_COMMENT: + return leaf_idx + return None + + +def _safe_add_trailing_comma(safe: bool, delimiter_priority: int, line: Line) -> Line: + if ( + safe + and delimiter_priority == COMMA_PRIORITY + and line.leaves[-1].type != token.COMMA + and line.leaves[-1].type != STANDALONE_COMMENT + ): + new_comma = Leaf(token.COMMA, ",") + line.append(new_comma) + return line + + @dont_increase_indentation -def delimiter_split(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]: +def delimiter_split( + line: Line, features: Collection[Feature], mode: Mode +) -> Iterator[Line]: """Split according to delimiters of the highest priority. If the appropriate Features are given, the split will add trailing commas @@ -989,7 +1199,8 @@ def append_to_line(leaf: Leaf) -> Iterator[Line]: ) current_line.append(leaf) - for leaf in line.leaves: + last_non_comment_leaf = _get_last_non_comment_leaf(line) + for leaf_idx, leaf in enumerate(line.leaves): yield from append_to_line(leaf) for comment_after in line.comments_after(leaf): @@ -1006,6 +1217,15 @@ def append_to_line(leaf: Leaf) -> Iterator[Line]: trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features ) + if ( + Preview.add_trailing_comma_consistently in mode + and last_leaf.type == STANDALONE_COMMENT + and leaf_idx == last_non_comment_leaf + ): + current_line = _safe_add_trailing_comma( + trailing_comma_safe, delimiter_priority, current_line + ) + leaf_priority = bt.delimiters.get(id(leaf)) if leaf_priority == delimiter_priority: yield current_line @@ -1014,23 +1234,18 @@ def append_to_line(leaf: Leaf) -> Iterator[Line]: mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets ) if current_line: - if ( - trailing_comma_safe - and delimiter_priority == COMMA_PRIORITY - and current_line.leaves[-1].type != token.COMMA - and current_line.leaves[-1].type != STANDALONE_COMMENT - ): - new_comma = Leaf(token.COMMA, ",") - current_line.append(new_comma) + current_line = _safe_add_trailing_comma( + trailing_comma_safe, delimiter_priority, current_line + ) yield current_line @dont_increase_indentation def standalone_comment_split( - line: Line, features: Collection[Feature] = () + line: Line, features: Collection[Feature], mode: Mode ) -> Iterator[Line]: """Split standalone comments from the rest of the line.""" - if not line.contains_standalone_comments(0): + if not line.contains_standalone_comments(): raise CannotSplit("Line does not have any standalone comments") current_line = Line( @@ -1060,25 +1275,7 @@ def append_to_line(leaf: Leaf) -> Iterator[Line]: yield current_line -def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None: - """Leave existing extra newlines if not `inside_brackets`. Remove everything - else. - - Note: don't use backslashes for formatting or you'll lose your voting rights. - """ - if not inside_brackets: - spl = leaf.prefix.split("#") - if "\\" not in spl[0]: - nl_count = spl[-1].count("\n") - if len(spl) > 1: - nl_count -= 1 - leaf.prefix = "\n" * nl_count - return - - leaf.prefix = "" - - -def normalize_invisible_parens( +def normalize_invisible_parens( # noqa: C901 node: Node, parens_after: Set[str], *, mode: Mode, features: Collection[Feature] ) -> None: """Make existing optional parentheses invisible or create new ones. @@ -1109,6 +1306,17 @@ def normalize_invisible_parens( child, parens_after=parens_after, mode=mode, features=features ) + # Fixes a bug where invisible parens are not properly wrapped around + # case blocks. + if ( + isinstance(child, Node) + and child.type == syms.case_block + and Preview.long_case_block_line_splitting in mode + ): + normalize_invisible_parens( + child, parens_after={"case"}, mode=mode, features=features + ) + # Add parentheses around long tuple unpacking in assignments. if ( index == 0 @@ -1154,6 +1362,17 @@ def normalize_invisible_parens( # invisible parentheses to work more precisely. continue + elif ( + isinstance(child, Leaf) + and child.next_sibling is not None + and child.next_sibling.type == token.COLON + and child.value == "case" + and Preview.long_case_block_line_splitting in mode + ): + # A special patch for "case case:" scenario, the second occurrence + # of case will be not parsed as a Python keyword. + break + elif not (isinstance(child, Leaf) and is_multiline_string(child)): wrap_in_parentheses(node, child, visible=False) @@ -1201,18 +1420,16 @@ def remove_await_parens(node: Node) -> None: opening_bracket = cast(Leaf, node.children[1].children[0]) closing_bracket = cast(Leaf, node.children[1].children[-1]) bracket_contents = node.children[1].children[1] - if isinstance(bracket_contents, Node): - if bracket_contents.type != syms.power: - ensure_visible(opening_bracket) - ensure_visible(closing_bracket) - elif ( - bracket_contents.type == syms.power - and bracket_contents.children[0].type == token.AWAIT - ): - ensure_visible(opening_bracket) - ensure_visible(closing_bracket) - # If we are in a nested await then recurse down. - remove_await_parens(bracket_contents) + if isinstance(bracket_contents, Node) and ( + bracket_contents.type != syms.power + or bracket_contents.children[0].type == token.AWAIT + or any( + isinstance(child, Leaf) and child.type == token.DOUBLESTAR + for child in bracket_contents.children + ) + ): + ensure_visible(opening_bracket) + ensure_visible(closing_bracket) def _maybe_wrap_cms_in_parens( @@ -1312,7 +1529,7 @@ def maybe_make_parens_invisible_in_atom( Returns whether the node should itself be wrapped in invisible parentheses. """ if ( - node.type != syms.atom + node.type not in (syms.atom, syms.expr) or is_empty_tuple(node) or is_one_tuple(node) or (is_yield(node) and parent.type != syms.expr_stmt) @@ -1336,6 +1553,7 @@ def maybe_make_parens_invisible_in_atom( syms.except_clause, syms.funcdef, syms.with_stmt, + syms.tname, # these ones aren't useful to end users, but they do please fuzzers syms.for_stmt, syms.del_stmt, @@ -1348,8 +1566,13 @@ def maybe_make_parens_invisible_in_atom( if is_lpar_token(first) and is_rpar_token(last): middle = node.children[1] # make parentheses invisible - first.value = "" - last.value = "" + if ( + # If the prefix of `middle` includes a type comment with + # ignore annotation, then we do not remove the parentheses + not is_type_ignore_comment_string(middle.prefix.strip()) + ): + first.value = "" + last.value = "" maybe_make_parens_invisible_in_atom( middle, parent=parent, @@ -1480,7 +1703,7 @@ def run_transformer( if not line_str: line_str = line_to_string(line) result: List[Line] = [] - for transformed_line in transform(line, features): + for transformed_line in transform(line, features, mode): if str(transformed_line).strip("\n") == line_str: raise CannotTransform("Line transformer returned an unchanged result") @@ -1495,7 +1718,7 @@ def run_transformer( or line.contains_multiline_strings() or result[0].contains_uncollapsable_type_comments() or result[0].contains_unsplittable_type_ignore() - or is_line_short_enough(result[0], line_length=mode.line_length) + or is_line_short_enough(result[0], mode=mode) # If any leaves have no parents (which _can_ occur since # `transform(line)` potentially destroys the line's underlying node # structure), then we can't proceed. Doing so would cause the below @@ -1510,8 +1733,6 @@ def run_transformer( second_opinion = run_transformer( line_copy, transform, mode, features_fop, line_str=line_str ) - if all( - is_line_short_enough(ln, line_length=mode.line_length) for ln in second_opinion - ): + if all(is_line_short_enough(ln, mode=mode) for ln in second_opinion): result = second_opinion return result diff --git a/src/black/lines.py b/src/black/lines.py index ec6ef5d9522..2a41db173d4 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -1,5 +1,5 @@ import itertools -import sys +import math from dataclasses import dataclass, field from typing import ( Callable, @@ -10,11 +10,12 @@ Sequence, Tuple, TypeVar, + Union, cast, ) -from black.brackets import DOT_PRIORITY, BracketTracker -from black.mode import Mode +from black.brackets import COMMA_PRIORITY, DOT_PRIORITY, BracketTracker +from black.mode import Mode, Preview from black.nodes import ( BRACKETS, CLOSING_BRACKETS, @@ -22,14 +23,20 @@ STANDALONE_COMMENT, TEST_DESCENDANTS, child_towards, + is_docstring, + is_funcdef, is_import, is_multiline_string, is_one_sequence_between, is_type_comment, + is_type_ignore_comment, + is_with_or_async_with_stmt, + make_simple_prefix, replace_child, syms, whitespace, ) +from black.strings import str_width from blib2to3.pgen2 import token from blib2to3.pytree import Leaf, Node @@ -37,13 +44,14 @@ T = TypeVar("T") Index = int LeafID = int +LN = Union[Leaf, Node] @dataclass class Line: """Holds leaves and comments. Can be printed with `str(line)`.""" - mode: Mode + mode: Mode = field(repr=False) depth: int = 0 leaves: List[Leaf] = field(default_factory=list) # keys ordered like `leaves` @@ -75,7 +83,9 @@ def append( # Note: at this point leaf.prefix should be empty except for # imports, for which we only preserve newlines. leaf.prefix += whitespace( - leaf, complex_subscript=self.is_complex_subscript(leaf) + leaf, + complex_subscript=self.is_complex_subscript(leaf), + mode=self.mode, ) if self.inside_brackets or not preformatted or track_bracket: self.bracket_tracker.mark(leaf) @@ -93,7 +103,10 @@ def append_safe(self, leaf: Leaf, preformatted: bool = False) -> None: Raises ValueError when any `leaf` is appended after a standalone comment or when a standalone comment is not the first leaf on the line. """ - if self.bracket_tracker.depth == 0: + if ( + self.bracket_tracker.depth == 0 + or self.bracket_tracker.any_open_for_or_lambda() + ): if self.is_comment: raise ValueError("cannot append to standalone comments") @@ -119,6 +132,11 @@ def is_import(self) -> bool: """Is this an import line?""" return bool(self) and is_import(self.leaves[0]) + @property + def is_with_or_async_with_stmt(self) -> bool: + """Is this a with_stmt line?""" + return bool(self) and is_with_or_async_with_stmt(self.leaves[0]) + @property def is_class(self) -> bool: """Is this line a class definition?""" @@ -154,6 +172,13 @@ def is_def(self) -> bool: and second_leaf.value == "def" ) + @property + def is_stub_def(self) -> bool: + """Is this line a function definition with a body consisting only of "..."?""" + return self.is_def and self.leaves[-4:] == [Leaf(token.COLON, ":")] + [ + Leaf(token.DOT, ".") for _ in range(3) + ] + @property def is_class_paren_empty(self) -> bool: """Is this a class with no base classes but using parentheses? @@ -173,11 +198,21 @@ def is_class_paren_empty(self) -> bool: @property def is_triple_quoted_string(self) -> bool: """Is the line a triple quoted string?""" - return ( - bool(self) - and self.leaves[0].type == token.STRING - and self.leaves[0].value.startswith(('"""', "'''")) - ) + if not self or self.leaves[0].type != token.STRING: + return False + value = self.leaves[0].value + if value.startswith(('"""', "'''")): + return True + if Preview.accept_raw_docstrings in self.mode and value.startswith( + ("r'''", 'r"""', "R'''", 'R"""') + ): + return True + return False + + @property + def is_chained_assignment(self) -> bool: + """Is the line a chained assignment""" + return [leaf.type for leaf in self.leaves].count(token.EQUAL) > 1 @property def opens_block(self) -> bool: @@ -186,14 +221,49 @@ def opens_block(self) -> bool: return False return self.leaves[-1].type == token.COLON - def contains_standalone_comments(self, depth_limit: int = sys.maxsize) -> bool: + def is_fmt_pass_converted( + self, *, first_leaf_matches: Optional[Callable[[Leaf], bool]] = None + ) -> bool: + """Is this line converted from fmt off/skip code? + + If first_leaf_matches is not None, it only returns True if the first + leaf of converted code matches. + """ + if len(self.leaves) != 1: + return False + leaf = self.leaves[0] + if ( + leaf.type != STANDALONE_COMMENT + or leaf.fmt_pass_converted_first_leaf is None + ): + return False + return first_leaf_matches is None or first_leaf_matches( + leaf.fmt_pass_converted_first_leaf + ) + + def contains_standalone_comments(self) -> bool: """If so, needs to be split before emitting.""" for leaf in self.leaves: - if leaf.type == STANDALONE_COMMENT and leaf.bracket_depth <= depth_limit: + if leaf.type == STANDALONE_COMMENT: return True return False + def contains_implicit_multiline_string_with_comments(self) -> bool: + """Chck if we have an implicit multiline string with comments on the line""" + for leaf_type, leaf_group_iterator in itertools.groupby( + self.leaves, lambda leaf: leaf.type + ): + if leaf_type != token.STRING: + continue + leaf_list = list(leaf_group_iterator) + if len(leaf_list) == 1: + continue + for leaf in leaf_list: + if self.comments_after(leaf): + return True + return False + def contains_uncollapsable_type_comments(self) -> bool: ignored_ids = set() try: @@ -221,7 +291,7 @@ def contains_uncollapsable_type_comments(self) -> bool: for comment in comments: if is_type_comment(comment): if comment_seen or ( - not is_type_comment(comment, " ignore") + not is_type_ignore_comment(comment) and leaf_id not in ignored_ids ): return True @@ -258,7 +328,7 @@ def contains_unsplittable_type_ignore(self) -> bool: # line. for node in self.leaves[-2:]: for comment in self.comments.get(id(node), []): - if is_type_comment(comment, " ignore"): + if is_type_ignore_comment(comment): return True return False @@ -289,9 +359,9 @@ def has_magic_trailing_comma( if closing.type == token.RSQB: if ( - closing.parent + closing.parent is not None and closing.parent.type == syms.trailer - and closing.opening_bracket + and closing.opening_bracket is not None and is_one_sequence_between( closing.opening_bracket, closing, @@ -301,22 +371,7 @@ def has_magic_trailing_comma( ): return False - if not ensure_removable: - return True - - comma = self.leaves[-1] - if comma.parent is None: - return False - return ( - comma.parent.type != syms.subscriptlist - or closing.opening_bracket is None - or not is_one_sequence_between( - closing.opening_bracket, - closing, - self.leaves, - brackets=(token.LSQB, token.RSQB), - ) - ) + return True if self.is_import: return True @@ -446,6 +501,17 @@ def __bool__(self) -> bool: return bool(self.leaves or self.comments) +@dataclass +class RHSResult: + """Intermediate split result from a right hand split.""" + + head: Line + body: Line + tail: Line + opening_bracket: Leaf + closing_bracket: Leaf + + @dataclass class LinesBlock: """Class that holds information about a block of formatted lines. @@ -460,12 +526,12 @@ class LinesBlock: before: int = 0 content_lines: List[str] = field(default_factory=list) after: int = 0 + form_feed: bool = False def all_lines(self) -> List[str]: empty_line = str(Line(mode=self.mode)) - return ( - [empty_line * self.before] + self.content_lines + [empty_line * self.after] - ) + prefix = make_simple_prefix(self.before, self.form_feed, empty_line) + return [prefix] + self.content_lines + [empty_line * self.after] @dataclass @@ -481,7 +547,7 @@ class EmptyLineTracker: mode: Mode previous_line: Optional[Line] = None previous_block: Optional[LinesBlock] = None - previous_defs: List[int] = field(default_factory=list) + previous_defs: List[Line] = field(default_factory=list) semantic_leading_comment: Optional[LinesBlock] = None def maybe_empty_lines(self, current_line: Line) -> LinesBlock: @@ -490,6 +556,12 @@ def maybe_empty_lines(self, current_line: Line) -> LinesBlock: This is for separating `def`, `async def` and `class` with extra empty lines (two on module-level). """ + form_feed = ( + Preview.allow_form_feeds in self.mode + and current_line.depth == 0 + and bool(current_line.leaves) + and "\f\n" in current_line.leaves[0].prefix + ) before, after = self._maybe_empty_lines(current_line) previous_after = self.previous_block.after if self.previous_block else 0 before = ( @@ -499,12 +571,23 @@ def maybe_empty_lines(self, current_line: Line) -> LinesBlock: if self.previous_line is None else before - previous_after ) + if ( + Preview.module_docstring_newlines in current_line.mode + and self.previous_block + and self.previous_block.previous_block is None + and len(self.previous_block.original_line.leaves) == 1 + and self.previous_block.original_line.is_triple_quoted_string + and not (current_line.is_class or current_line.is_def) + ): + before = 1 + block = LinesBlock( mode=self.mode, previous_block=self.previous_block, original_line=current_line, before=before, after=after, + form_feed=form_feed, ) # Maintain the semantic_leading_comment state. @@ -536,13 +619,26 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: first_leaf.prefix = "" else: before = 0 + + user_had_newline = bool(before) depth = current_line.depth - while self.previous_defs and self.previous_defs[-1] >= depth: + + previous_def = None + while self.previous_defs and self.previous_defs[-1].depth >= depth: + previous_def = self.previous_defs.pop() + + if previous_def is not None: + assert self.previous_line is not None if self.mode.is_pyi: - assert self.previous_line is not None if depth and not current_line.is_def and self.previous_line.is_def: # Empty lines between attributes and methods should be preserved. - before = min(1, before) + before = 1 if user_had_newline else 0 + elif ( + Preview.blank_line_after_nested_stub_class in self.mode + and previous_def.is_class + and not previous_def.is_stub_class + ): + before = 1 elif depth: before = 0 else: @@ -552,7 +648,7 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: before = 1 elif ( not depth - and self.previous_defs[-1] + and previous_def.depth and current_line.leaves[-1].type == token.COLON and ( current_line.leaves[0].value @@ -569,14 +665,17 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: before = 1 else: before = 2 - self.previous_defs.pop() + if current_line.is_decorator or current_line.is_def or current_line.is_class: - return self._maybe_empty_lines_for_class_or_def(current_line, before) + return self._maybe_empty_lines_for_class_or_def( + current_line, before, user_had_newline + ) if ( self.previous_line and self.previous_line.is_import and not current_line.is_import + and not current_line.is_fmt_pass_converted(first_leaf_matches=is_import) and depth == self.previous_line.depth ): return (before or 1), 0 @@ -586,17 +685,38 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: and self.previous_line.is_class and current_line.is_triple_quoted_string ): + if Preview.no_blank_line_before_class_docstring in current_line.mode: + return 0, 1 return before, 1 - if self.previous_line and self.previous_line.opens_block: + # In preview mode, always allow blank lines, except right before a function + # docstring + is_empty_first_line_ok = ( + Preview.allow_empty_first_line_in_block in current_line.mode + and ( + not is_docstring(current_line.leaves[0]) + or ( + self.previous_line + and self.previous_line.leaves[0] + and self.previous_line.leaves[0].parent + and not is_funcdef(self.previous_line.leaves[0].parent) + ) + ) + ) + + if ( + self.previous_line + and self.previous_line.opens_block + and not is_empty_first_line_ok + ): return 0, 0 return before, 0 - def _maybe_empty_lines_for_class_or_def( - self, current_line: Line, before: int + def _maybe_empty_lines_for_class_or_def( # noqa: C901 + self, current_line: Line, before: int, user_had_newline: bool ) -> Tuple[int, int]: if not current_line.is_decorator: - self.previous_defs.append(current_line.depth) + self.previous_defs.append(current_line) if self.previous_line is None: # Don't insert empty lines before the first line in the file. return 0, 0 @@ -642,6 +762,17 @@ def _maybe_empty_lines_for_class_or_def( newlines = 0 else: newlines = 1 + # Remove case `self.previous_line.depth > current_line.depth` below when + # this becomes stable. + # + # Don't inspect the previous line if it's part of the body of the previous + # statement in the same level, we always want a blank line if there's + # something with a body preceding. + elif ( + Preview.blank_line_between_nested_and_def_stub_file in current_line.mode + and self.previous_line.depth > current_line.depth + ): + newlines = 1 elif ( current_line.is_def or current_line.is_decorator ) and not self.previous_line.is_def: @@ -659,6 +790,14 @@ def _maybe_empty_lines_for_class_or_def( newlines = 0 else: newlines = 1 if current_line.depth else 2 + # If a user has left no space after a dummy implementation, don't insert + # new lines. This is useful for instance for @overload or Protocols. + if ( + Preview.dummy_implementations in self.mode + and self.previous_line.is_stub_def + and not user_had_newline + ): + newlines = 0 if comment_to_add_newlines is not None: previous_block = comment_to_add_newlines.previous_block if previous_block is not None: @@ -701,18 +840,95 @@ def append_leaves( new_line.append(comment_leaf, preformatted=True) -def is_line_short_enough(line: Line, *, line_length: int, line_str: str = "") -> bool: - """Return True if `line` is no longer than `line_length`. - +def is_line_short_enough( # noqa: C901 + line: Line, *, mode: Mode, line_str: str = "" +) -> bool: + """For non-multiline strings, return True if `line` is no longer than `line_length`. + For multiline strings, looks at the context around `line` to determine + if it should be inlined or split up. Uses the provided `line_str` rendering, if any, otherwise computes a new one. """ if not line_str: line_str = line_to_string(line) - return ( - len(line_str) <= line_length - and "\n" not in line_str # multiline strings - and not line.contains_standalone_comments() - ) + + width = str_width if Preview.respect_east_asian_width in mode else len + + if Preview.multiline_string_handling not in mode: + return ( + width(line_str) <= mode.line_length + and "\n" not in line_str # multiline strings + and not line.contains_standalone_comments() + ) + + if line.contains_standalone_comments(): + return False + if "\n" not in line_str: + # No multiline strings (MLS) present + return width(line_str) <= mode.line_length + + first, *_, last = line_str.split("\n") + if width(first) > mode.line_length or width(last) > mode.line_length: + return False + + # Traverse the AST to examine the context of the multiline string (MLS), + # tracking aspects such as depth and comma existence, + # to determine whether to split the MLS or keep it together. + # Depth (which is based on the existing bracket_depth concept) + # is needed to determine nesting level of the MLS. + # Includes special case for trailing commas. + commas: List[int] = [] # tracks number of commas per depth level + multiline_string: Optional[Leaf] = None + # store the leaves that contain parts of the MLS + multiline_string_contexts: List[LN] = [] + + max_level_to_update: Union[int, float] = math.inf # track the depth of the MLS + for i, leaf in enumerate(line.leaves): + if max_level_to_update == math.inf: + had_comma: Optional[int] = None + if leaf.bracket_depth + 1 > len(commas): + commas.append(0) + elif leaf.bracket_depth + 1 < len(commas): + had_comma = commas.pop() + if ( + had_comma is not None + and multiline_string is not None + and multiline_string.bracket_depth == leaf.bracket_depth + 1 + ): + # Have left the level with the MLS, stop tracking commas + max_level_to_update = leaf.bracket_depth + if had_comma > 0: + # MLS was in parens with at least one comma - force split + return False + + if leaf.bracket_depth <= max_level_to_update and leaf.type == token.COMMA: + # Ignore non-nested trailing comma + # directly after MLS/MLS-containing expression + ignore_ctxs: List[Optional[LN]] = [None] + ignore_ctxs += multiline_string_contexts + if not (leaf.prev_sibling in ignore_ctxs and i == len(line.leaves) - 1): + commas[leaf.bracket_depth] += 1 + if max_level_to_update != math.inf: + max_level_to_update = min(max_level_to_update, leaf.bracket_depth) + + if is_multiline_string(leaf): + if len(multiline_string_contexts) > 0: + # >1 multiline string cannot fit on a single line - force split + return False + multiline_string = leaf + ctx: LN = leaf + # fetch the leaf components of the MLS in the AST + while str(ctx) in line_str: + multiline_string_contexts.append(ctx) + if ctx.parent is None: + break + ctx = ctx.parent + + # May not have a triple-quoted multiline string at all, + # in case of a regular string with embedded newlines and line continuations + if len(multiline_string_contexts) == 0: + return True + + return all(val == 0 for val in commas) def can_be_split(line: Line) -> bool: @@ -752,25 +968,59 @@ def can_be_split(line: Line) -> bool: def can_omit_invisible_parens( - line: Line, + rhs: RHSResult, line_length: int, ) -> bool: - """Does `line` have a shape safe to reformat without optional parens around it? + """Does `rhs.body` have a shape safe to reformat without optional parens around it? Returns True for only a subset of potentially nice looking formattings but the point is to not return false positives that end up producing lines that are too long. """ + line = rhs.body + + # We need optional parens in order to split standalone comments to their own lines + # if there are no nested parens around the standalone comments + closing_bracket: Optional[Leaf] = None + for leaf in reversed(line.leaves): + if closing_bracket and leaf is closing_bracket.opening_bracket: + closing_bracket = None + if leaf.type == STANDALONE_COMMENT and not closing_bracket: + return False + if ( + not closing_bracket + and leaf.type in CLOSING_BRACKETS + and leaf.opening_bracket in line.leaves + and leaf.value + ): + closing_bracket = leaf + bt = line.bracket_tracker if not bt.delimiters: # Without delimiters the optional parentheses are useless. return True max_priority = bt.max_delimiter_priority() - if bt.delimiter_count_with_priority(max_priority) > 1: + delimiter_count = bt.delimiter_count_with_priority(max_priority) + if delimiter_count > 1: # With more than one delimiter of a kind the optional parentheses read better. return False + if delimiter_count == 1: + if ( + Preview.wrap_multiple_context_managers_in_parens in line.mode + and max_priority == COMMA_PRIORITY + and rhs.head.is_with_or_async_with_stmt + ): + # For two context manager with statements, the optional parentheses read + # better. In this case, `rhs.body` is the context managers part of + # the with statement. `rhs.head` is the `with (` part on the previous + # line. + return False + # Otherwise it may also read better, but we don't do it today and requires + # careful considerations for all possible cases. See + # https://github.com/psf/black/issues/2156. + if max_priority == DOT_PRIORITY: # A single stranded method call doesn't require optional parentheses. return True diff --git a/src/black/mode.py b/src/black/mode.py index 1af16070073..38b861e39ca 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -4,19 +4,13 @@ chosen by the user. """ -import sys from dataclasses import dataclass, field from enum import Enum, auto from hashlib import sha256 from operator import attrgetter -from typing import Dict, Set +from typing import Dict, Final, Set from warnings import warn -if sys.version_info < (3, 8): - from typing_extensions import Final -else: - from typing import Final - from black.const import DEFAULT_LINE_LENGTH @@ -30,6 +24,7 @@ class TargetVersion(Enum): PY39 = 9 PY310 = 10 PY311 = 11 + PY312 = 12 class Feature(Enum): @@ -51,6 +46,7 @@ class Feature(Enum): VARIADIC_GENERICS = 15 DEBUG_F_STRINGS = 16 PARENTHESIZED_CONTEXT_MANAGERS = 17 + TYPE_PARAMS = 18 FORCE_OPTIONAL_PARENTHESES = 50 # __future__ flags @@ -143,6 +139,25 @@ class Feature(Enum): Feature.EXCEPT_STAR, Feature.VARIADIC_GENERICS, }, + TargetVersion.PY312: { + Feature.F_STRINGS, + Feature.DEBUG_F_STRINGS, + Feature.NUMERIC_UNDERSCORES, + Feature.TRAILING_COMMA_IN_CALL, + Feature.TRAILING_COMMA_IN_DEF, + Feature.ASYNC_KEYWORDS, + Feature.FUTURE_ANNOTATIONS, + Feature.ASSIGNMENT_EXPRESSIONS, + Feature.RELAXED_DECORATORS, + Feature.POS_ONLY_ARGUMENTS, + Feature.UNPACKING_ON_FLOW, + Feature.ANN_ASSIGN_EXTENDED_RHS, + Feature.PARENTHESIZED_CONTEXT_MANAGERS, + Feature.PATTERN_MATCHING, + Feature.EXCEPT_STAR, + Feature.VARIADIC_GENERICS, + Feature.TYPE_PARAMS, + }, } @@ -153,15 +168,34 @@ def supports_feature(target_versions: Set[TargetVersion], feature: Feature) -> b class Preview(Enum): """Individual preview style features.""" + add_trailing_comma_consistently = auto() + blank_line_after_nested_stub_class = auto() + blank_line_between_nested_and_def_stub_file = auto() hex_codes_in_unicode_sequences = auto() + improved_async_statements_handling = auto() + multiline_string_handling = auto() + no_blank_line_before_class_docstring = auto() prefer_splitting_right_hand_side_of_assignments = auto() # NOTE: string_processing requires wrap_long_dict_values_in_parens # for https://github.com/psf/black/issues/3117 to be fixed. string_processing = auto() parenthesize_conditional_expressions = auto() + parenthesize_long_type_hints = auto() + respect_magic_trailing_comma_in_return_type = auto() skip_magic_trailing_comma_in_subscript = auto() wrap_long_dict_values_in_parens = auto() wrap_multiple_context_managers_in_parens = auto() + dummy_implementations = auto() + walrus_subscript = auto() + module_docstring_newlines = auto() + accept_raw_docstrings = auto() + fix_power_op_line_length = auto() + hug_parens_with_braces_and_square_brackets = auto() + allow_empty_first_line_in_block = auto() + single_line_format_skip_with_multiple_comments = auto() + long_case_block_line_splitting = auto() + allow_form_feeds = auto() + respect_east_asian_width = auto() class Deprecated(UserWarning): @@ -184,10 +218,8 @@ class Mode: def __post_init__(self) -> None: if self.experimental_string_processing: warn( - ( - "`experimental string processing` has been included in `preview`" - " and deprecated. Use `preview` instead." - ), + "`experimental string processing` has been included in `preview`" + " and deprecated. Use `preview` instead.", Deprecated, ) diff --git a/src/black/nodes.py b/src/black/nodes.py index a588077f4de..a4f555b4032 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -3,12 +3,8 @@ """ import sys -from typing import Generic, Iterator, List, Optional, Set, Tuple, TypeVar, Union +from typing import Final, Generic, Iterator, List, Optional, Set, Tuple, TypeVar, Union -if sys.version_info >= (3, 8): - from typing import Final -else: - from typing_extensions import Final if sys.version_info >= (3, 10): from typing import TypeGuard else: @@ -17,7 +13,8 @@ from mypy_extensions import mypyc_attr from black.cache import CACHE_DIR -from black.strings import has_triple_quotes +from black.mode import Mode, Preview +from black.strings import get_string_prefix, has_triple_quotes from blib2to3 import pygram from blib2to3.pgen2 import token from blib2to3.pytree import NL, Leaf, Node, type_repr @@ -175,15 +172,15 @@ def visit_default(self, node: LN) -> Iterator[T]: yield from self.visit(child) -def whitespace(leaf: Leaf, *, complex_subscript: bool) -> str: # noqa: C901 +def whitespace(leaf: Leaf, *, complex_subscript: bool, mode: Mode) -> str: # noqa: C901 """Return whitespace prefix if needed for the given `leaf`. `complex_subscript` signals whether the given leaf is part of a subscription which has non-trivial arguments, like arithmetic expressions or function calls. """ - NO: Final = "" - SPACE: Final = " " - DOUBLESPACE: Final = " " + NO: Final[str] = "" + SPACE: Final[str] = " " + DOUBLESPACE: Final[str] = " " t = leaf.type p = leaf.parent v = leaf.value @@ -349,6 +346,11 @@ def whitespace(leaf: Leaf, *, complex_subscript: bool) -> str: # noqa: C901 return NO + elif Preview.walrus_subscript in mode and ( + t == token.COLONEQUAL or prev.type == token.COLONEQUAL + ): + return SPACE + elif not complex_subscript: return NO @@ -405,6 +407,13 @@ def whitespace(leaf: Leaf, *, complex_subscript: bool) -> str: # noqa: C901 return SPACE +def make_simple_prefix(nl_count: int, form_feed: bool, empty_line: str = "\n") -> str: + """Generate a normalized prefix string.""" + if form_feed: + return (empty_line * (nl_count - 1)) + "\f" + empty_line + return empty_line * nl_count + + def preceding_leaf(node: Optional[LN]) -> Optional[Leaf]: """Return the first leaf that precedes `node`, if any.""" while node: @@ -523,6 +532,13 @@ def is_arith_like(node: LN) -> bool: def is_docstring(leaf: Leaf) -> bool: + if leaf.type != token.STRING: + return False + + prefix = get_string_prefix(leaf.value) + if set(prefix).intersection("bBfF"): + return False + if prev_siblings_are( leaf.parent, [None, token.NEWLINE, token.INDENT, syms.simple_stmt] ): @@ -716,8 +732,27 @@ def is_multiline_string(leaf: Leaf) -> bool: return has_triple_quotes(leaf.value) and "\n" in leaf.value -def is_stub_suite(node: Node) -> bool: +def is_funcdef(node: Node) -> bool: + return node.type == syms.funcdef + + +def is_function_or_class(node: Node) -> bool: + return node.type in {syms.funcdef, syms.classdef, syms.async_funcdef} + + +def is_stub_suite(node: Node, mode: Mode) -> bool: """Return True if `node` is a suite with a stub body.""" + if ( + node.parent is not None + and Preview.dummy_implementations in mode + and not is_function_or_class(node.parent) + ): + return False + + # If there is a comment, we want to keep it. + if node.prefix.strip(): + return False + if ( len(node.children) != 4 or node.children[0].type != token.NEWLINE @@ -726,6 +761,9 @@ def is_stub_suite(node: Node) -> bool: ): return False + if node.children[3].prefix.strip(): + return False + return is_stub_body(node.children[2]) @@ -739,7 +777,8 @@ def is_stub_body(node: LN) -> bool: child = node.children[0] return ( - child.type == syms.atom + not child.prefix.strip() + and child.type == syms.atom and len(child.children) == 3 and all(leaf == Leaf(token.DOT, ".") for leaf in child.children) ) @@ -789,12 +828,54 @@ def is_import(leaf: Leaf) -> bool: ) -def is_type_comment(leaf: Leaf, suffix: str = "") -> bool: - """Return True if the given leaf is a special comment. - Only returns true for type comments for now.""" +def is_with_or_async_with_stmt(leaf: Leaf) -> bool: + """Return True if the given leaf starts a with or async with statement.""" + return bool( + leaf.type == token.NAME + and leaf.value == "with" + and leaf.parent + and leaf.parent.type == syms.with_stmt + ) or bool( + leaf.type == token.ASYNC + and leaf.next_sibling + and leaf.next_sibling.type == syms.with_stmt + ) + + +def is_async_stmt_or_funcdef(leaf: Leaf) -> bool: + """Return True if the given leaf starts an async def/for/with statement. + + Note that `async def` can be either an `async_stmt` or `async_funcdef`, + the latter is used when it has decorators. + """ + return bool( + leaf.type == token.ASYNC + and leaf.parent + and leaf.parent.type in {syms.async_stmt, syms.async_funcdef} + ) + + +def is_type_comment(leaf: Leaf) -> bool: + """Return True if the given leaf is a type comment. This function should only + be used for general type comments (excluding ignore annotations, which should + use `is_type_ignore_comment`). Note that general type comments are no longer + used in modern version of Python, this function may be deprecated in the future.""" + t = leaf.type + v = leaf.value + return t in {token.COMMENT, STANDALONE_COMMENT} and v.startswith("# type:") + + +def is_type_ignore_comment(leaf: Leaf) -> bool: + """Return True if the given leaf is a type comment with ignore annotation.""" t = leaf.type v = leaf.value - return t in {token.COMMENT, STANDALONE_COMMENT} and v.startswith("# type:" + suffix) + return t in {token.COMMENT, STANDALONE_COMMENT} and is_type_ignore_comment_string(v) + + +def is_type_ignore_comment_string(value: str) -> bool: + """Return True if the given string match with type comment with + ignore annotation.""" + return value.startswith("# type: ignore") def wrap_in_parentheses(parent: Node, child: LN, *, visible: bool = True) -> None: @@ -871,3 +952,31 @@ def is_part_of_annotation(leaf: Leaf) -> bool: return True ancestor = ancestor.parent return False + + +def first_leaf(node: LN) -> Optional[Leaf]: + """Returns the first leaf of the ancestor node.""" + if isinstance(node, Leaf): + return node + elif not node.children: + return None + else: + return first_leaf(node.children[0]) + + +def last_leaf(node: LN) -> Optional[Leaf]: + """Returns the last leaf of the ancestor node.""" + if isinstance(node, Leaf): + return node + elif not node.children: + return None + else: + return last_leaf(node.children[-1]) + + +def furthest_ancestor_with_last_leaf(leaf: Leaf) -> LN: + """Returns the furthest ancestor that has this leaf node as the last leaf.""" + node: LN = leaf + while node.parent and node.parent.children and node is node.parent.children[-1]: + node = node.parent + return node diff --git a/src/black/numerics.py b/src/black/numerics.py index 879e5b2cf36..3040de06fde 100644 --- a/src/black/numerics.py +++ b/src/black/numerics.py @@ -1,6 +1,7 @@ """ Formatting numeric literals. """ + from blib2to3.pytree import Leaf @@ -13,7 +14,7 @@ def format_hex(text: str) -> str: def format_scientific_notation(text: str) -> str: - """Formats a numeric string utilizing scentific notation""" + """Formats a numeric string utilizing scientific notation""" before, after = text.split("e") sign = "" if after.startswith("-"): diff --git a/src/black/output.py b/src/black/output.py index f4c17f28ea4..7c7dd0fe14e 100644 --- a/src/black/output.py +++ b/src/black/output.py @@ -4,8 +4,9 @@ """ import json +import re import tempfile -from typing import Any, Optional +from typing import Any, List, Optional from click import echo, style from mypy_extensions import mypyc_attr @@ -55,12 +56,28 @@ def ipynb_diff(a: str, b: str, a_name: str, b_name: str) -> str: return "".join(diff_lines) +_line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))") + + +def _splitlines_no_ff(source: str) -> List[str]: + """Split a string into lines ignoring form feed and other chars. + + This mimics how the Python parser splits source code. + + A simplified version of the function with the same name in Lib/ast.py + """ + result = [match[0] for match in _line_pattern.finditer(source)] + if result[-1] == "": + result.pop(-1) + return result + + def diff(a: str, b: str, a_name: str, b_name: str) -> str: """Return a unified diff string between strings `a` and `b`.""" import difflib - a_lines = a.splitlines(keepends=True) - b_lines = b.splitlines(keepends=True) + a_lines = _splitlines_no_ff(a) + b_lines = _splitlines_no_ff(b) diff_lines = [] for line in difflib.unified_diff( a_lines, b_lines, fromfile=a_name, tofile=b_name, n=5 diff --git a/src/black/parsing.py b/src/black/parsing.py index ba474c5e047..178a7ef10e2 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -1,15 +1,10 @@ """ Parse Python code and perform AST validation. """ + import ast -import platform import sys -from typing import Any, Iterable, Iterator, List, Set, Tuple, Type, Union - -if sys.version_info < (3, 8): - from typing_extensions import Final -else: - from typing import Final +from typing import Iterable, Iterator, List, Set, Tuple from black.mode import VERSION_TO_FEATURES, Feature, TargetVersion, supports_feature from black.nodes import syms @@ -20,29 +15,6 @@ from blib2to3.pgen2.tokenize import TokenError from blib2to3.pytree import Leaf, Node -ast3: Any - -_IS_PYPY = platform.python_implementation() == "PyPy" - -try: - from typed_ast import ast3 -except ImportError: - if sys.version_info < (3, 8) and not _IS_PYPY: - print( - ( - "The typed_ast package is required but not installed.\n" - "You can upgrade to Python 3.8+ or install typed_ast with\n" - "`python3 -m pip install typed-ast`." - ), - file=sys.stderr, - ) - sys.exit(1) - else: - ast3 = ast - - -PY2_HINT: Final = "Python 2 support was removed in version 22.0." - class InvalidInput(ValueError): """Raised when input source code fails all parse attempts.""" @@ -53,9 +25,9 @@ def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]: # No target_version specified, so try all grammars. return [ # Python 3.7-3.9 - pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords, + pygram.python_grammar_async_keywords, # Python 3.0-3.6 - pygram.python_grammar_no_print_statement_no_exec_statement, + pygram.python_grammar, # Python 3.10+ pygram.python_grammar_soft_keywords, ] @@ -66,12 +38,10 @@ def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]: target_versions, Feature.ASYNC_IDENTIFIERS ) and not supports_feature(target_versions, Feature.PATTERN_MATCHING): # Python 3.7-3.9 - grammars.append( - pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords - ) + grammars.append(pygram.python_grammar_async_keywords) if not supports_feature(target_versions, Feature.ASYNC_KEYWORDS): # Python 3.0-3.6 - grammars.append(pygram.python_grammar_no_print_statement_no_exec_statement) + grammars.append(pygram.python_grammar) if any(Feature.PATTERN_MATCHING in VERSION_TO_FEATURES[v] for v in target_versions): # Python 3.10+ grammars.append(pygram.python_grammar_soft_keywords) @@ -116,14 +86,6 @@ def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) - # Choose the latest version when raising the actual parsing error. assert len(errors) >= 1 exc = errors[max(errors)] - - if matches_grammar(src_txt, pygram.python_grammar) or matches_grammar( - src_txt, pygram.python_grammar_no_print_statement - ): - original_msg = exc.args[0] - msg = f"{original_msg}\n{PY2_HINT}" - raise InvalidInput(msg) from None - raise exc from None if isinstance(result, Leaf): @@ -148,42 +110,34 @@ def lib2to3_unparse(node: Node) -> str: def parse_single_version( - src: str, version: Tuple[int, int] -) -> Union[ast.AST, ast3.AST]: + src: str, version: Tuple[int, int], *, type_comments: bool +) -> ast.AST: filename = "" - # typed-ast is needed because of feature version limitations in the builtin ast 3.8> - if sys.version_info >= (3, 8) and version >= (3,): - return ast.parse(src, filename, feature_version=version, type_comments=True) - - if _IS_PYPY: - # PyPy 3.7 doesn't support type comment tracking which is not ideal, but there's - # not much we can do as typed-ast won't work either. - if sys.version_info >= (3, 8): - return ast3.parse(src, filename, type_comments=True) - else: - return ast3.parse(src, filename) - else: - # Typed-ast is guaranteed to be used here and automatically tracks type - # comments separately. - return ast3.parse(src, filename, feature_version=version[1]) + return ast.parse( + src, filename, feature_version=version, type_comments=type_comments + ) -def parse_ast(src: str) -> Union[ast.AST, ast3.AST]: +def parse_ast(src: str) -> ast.AST: # TODO: support Python 4+ ;) versions = [(3, minor) for minor in range(3, sys.version_info[1] + 1)] first_error = "" for version in sorted(versions, reverse=True): try: - return parse_single_version(src, version) + return parse_single_version(src, version, type_comments=True) except SyntaxError as e: if not first_error: first_error = str(e) - raise SyntaxError(first_error) - + # Try to parse without type comments + for version in sorted(versions, reverse=True): + try: + return parse_single_version(src, version, type_comments=False) + except SyntaxError: + pass -ast3_AST: Final[Type[ast3.AST]] = ast3.AST + raise SyntaxError(first_error) def _normalize(lineend: str, value: str) -> str: @@ -196,30 +150,32 @@ def _normalize(lineend: str, value: str) -> str: return normalized.strip() -def stringify_ast(node: Union[ast.AST, ast3.AST], depth: int = 0) -> Iterator[str]: +def stringify_ast(node: ast.AST, depth: int = 0) -> Iterator[str]: """Simple visitor generating strings to compare ASTs by content.""" - node = fixup_ast_constants(node) + if ( + isinstance(node, ast.Constant) + and isinstance(node.value, str) + and node.kind == "u" + ): + # It's a quirk of history that we strip the u prefix over here. We used to + # rewrite the AST nodes for Python version compatibility and we never copied + # over the kind + node.kind = None yield f"{' ' * depth}{node.__class__.__name__}(" - type_ignore_classes: Tuple[Type[Any], ...] for field in sorted(node._fields): # noqa: F402 - # TypeIgnore will not be present using pypy < 3.8, so need for this - if not (_IS_PYPY and sys.version_info < (3, 8)): - # TypeIgnore has only one field 'lineno' which breaks this comparison - type_ignore_classes = (ast3.TypeIgnore,) - if sys.version_info >= (3, 8): - type_ignore_classes += (ast.TypeIgnore,) - if isinstance(node, type_ignore_classes): - break + # TypeIgnore has only one field 'lineno' which breaks this comparison + if isinstance(node, ast.TypeIgnore): + break try: value: object = getattr(node, field) except AttributeError: continue - yield f"{' ' * (depth+1)}{field}=" + yield f"{' ' * (depth + 1)}{field}=" if isinstance(value, list): for item in value: @@ -227,51 +183,34 @@ def stringify_ast(node: Union[ast.AST, ast3.AST], depth: int = 0) -> Iterator[st # parentheses and they change the AST. if ( field == "targets" - and isinstance(node, (ast.Delete, ast3.Delete)) - and isinstance(item, (ast.Tuple, ast3.Tuple)) + and isinstance(node, ast.Delete) + and isinstance(item, ast.Tuple) ): for elt in item.elts: yield from stringify_ast(elt, depth + 2) - elif isinstance(item, (ast.AST, ast3.AST)): + elif isinstance(item, ast.AST): yield from stringify_ast(item, depth + 2) - # Note that we are referencing the typed-ast ASTs via global variables and not - # direct module attribute accesses because that breaks mypyc. It's probably - # something to do with the ast3 variables being marked as Any leading - # mypy to think this branch is always taken, leaving the rest of the code - # unanalyzed. Tighting up the types for the typed-ast AST types avoids the - # mypyc crash. - elif isinstance(value, (ast.AST, ast3_AST)): + elif isinstance(value, ast.AST): yield from stringify_ast(value, depth + 2) else: normalized: object - # Constant strings may be indented across newlines, if they are - # docstrings; fold spaces after newlines when comparing. Similarly, - # trailing and leading space may be removed. if ( isinstance(node, ast.Constant) and field == "value" and isinstance(value, str) ): + # Constant strings may be indented across newlines, if they are + # docstrings; fold spaces after newlines when comparing. Similarly, + # trailing and leading space may be removed. normalized = _normalize("\n", value) + elif field == "type_comment" and isinstance(value, str): + # Trailing whitespace in type comments is removed. + normalized = value.rstrip() else: normalized = value - yield f"{' ' * (depth+2)}{normalized!r}, # {value.__class__.__name__}" + yield f"{' ' * (depth + 2)}{normalized!r}, # {value.__class__.__name__}" yield f"{' ' * depth}) # /{node.__class__.__name__}" - - -def fixup_ast_constants(node: Union[ast.AST, ast3.AST]) -> Union[ast.AST, ast3.AST]: - """Map ast nodes deprecated in 3.8 to Constant.""" - if isinstance(node, (ast.Str, ast3.Str, ast.Bytes, ast3.Bytes)): - return ast.Constant(value=node.s) - - if isinstance(node, (ast.Num, ast3.Num)): - return ast.Constant(value=node.n) - - if isinstance(node, (ast.NameConstant, ast3.NameConstant)): - return ast.Constant(value=node.value) - - return node diff --git a/src/black/ranges.py b/src/black/ranges.py new file mode 100644 index 00000000000..59e19242d47 --- /dev/null +++ b/src/black/ranges.py @@ -0,0 +1,496 @@ +"""Functions related to Black's formatting by line ranges feature.""" + +import difflib +from dataclasses import dataclass +from typing import Collection, Iterator, List, Sequence, Set, Tuple, Union + +from black.nodes import ( + LN, + STANDALONE_COMMENT, + Leaf, + Node, + Visitor, + first_leaf, + furthest_ancestor_with_last_leaf, + last_leaf, + syms, +) +from blib2to3.pgen2.token import ASYNC, NEWLINE + + +def parse_line_ranges(line_ranges: Sequence[str]) -> List[Tuple[int, int]]: + lines: List[Tuple[int, int]] = [] + for lines_str in line_ranges: + parts = lines_str.split("-") + if len(parts) != 2: + raise ValueError( + "Incorrect --line-ranges format, expect 'START-END', found" + f" {lines_str!r}" + ) + try: + start = int(parts[0]) + end = int(parts[1]) + except ValueError: + raise ValueError( + "Incorrect --line-ranges value, expect integer ranges, found" + f" {lines_str!r}" + ) from None + else: + lines.append((start, end)) + return lines + + +def is_valid_line_range(lines: Tuple[int, int]) -> bool: + """Returns whether the line range is valid.""" + return not lines or lines[0] <= lines[1] + + +def adjusted_lines( + lines: Collection[Tuple[int, int]], + original_source: str, + modified_source: str, +) -> List[Tuple[int, int]]: + """Returns the adjusted line ranges based on edits from the original code. + + This computes the new line ranges by diffing original_source and + modified_source, and adjust each range based on how the range overlaps with + the diffs. + + Note the diff can contain lines outside of the original line ranges. This can + happen when the formatting has to be done in adjacent to maintain consistent + local results. For example: + + 1. def my_func(arg1, arg2, + 2. arg3,): + 3. pass + + If it restricts to line 2-2, it can't simply reformat line 2, it also has + to reformat line 1: + + 1. def my_func( + 2. arg1, + 3. arg2, + 4. arg3, + 5. ): + 6. pass + + In this case, we will expand the line ranges to also include the whole diff + block. + + Args: + lines: a collection of line ranges. + original_source: the original source. + modified_source: the modified source. + """ + lines_mappings = _calculate_lines_mappings(original_source, modified_source) + + new_lines = [] + # Keep an index of the current search. Since the lines and lines_mappings are + # sorted, this makes the search complexity linear. + current_mapping_index = 0 + for start, end in sorted(lines): + start_mapping_index = _find_lines_mapping_index( + start, + lines_mappings, + current_mapping_index, + ) + end_mapping_index = _find_lines_mapping_index( + end, + lines_mappings, + start_mapping_index, + ) + current_mapping_index = start_mapping_index + if start_mapping_index >= len(lines_mappings) or end_mapping_index >= len( + lines_mappings + ): + # Protect against invalid inputs. + continue + start_mapping = lines_mappings[start_mapping_index] + end_mapping = lines_mappings[end_mapping_index] + if start_mapping.is_changed_block: + # When the line falls into a changed block, expands to the whole block. + new_start = start_mapping.modified_start + else: + new_start = ( + start - start_mapping.original_start + start_mapping.modified_start + ) + if end_mapping.is_changed_block: + # When the line falls into a changed block, expands to the whole block. + new_end = end_mapping.modified_end + else: + new_end = end - end_mapping.original_start + end_mapping.modified_start + new_range = (new_start, new_end) + if is_valid_line_range(new_range): + new_lines.append(new_range) + return new_lines + + +def convert_unchanged_lines(src_node: Node, lines: Collection[Tuple[int, int]]) -> None: + """Converts unchanged lines to STANDALONE_COMMENT. + + The idea is similar to how `# fmt: on/off` is implemented. It also converts the + nodes between those markers as a single `STANDALONE_COMMENT` leaf node with + the unformatted code as its value. `STANDALONE_COMMENT` is a "fake" token + that will be formatted as-is with its prefix normalized. + + Here we perform two passes: + + 1. Visit the top-level statements, and convert them to a single + `STANDALONE_COMMENT` when unchanged. This speeds up formatting when some + of the top-level statements aren't changed. + 2. Convert unchanged "unwrapped lines" to `STANDALONE_COMMENT` nodes line by + line. "unwrapped lines" are divided by the `NEWLINE` token. e.g. a + multi-line statement is *one* "unwrapped line" that ends with `NEWLINE`, + even though this statement itself can span multiple lines, and the + tokenizer only sees the last '\n' as the `NEWLINE` token. + + NOTE: During pass (2), comment prefixes and indentations are ALWAYS + normalized even when the lines aren't changed. This is fixable by moving + more formatting to pass (1). However, it's hard to get it correct when + incorrect indentations are used. So we defer this to future optimizations. + """ + lines_set: Set[int] = set() + for start, end in lines: + lines_set.update(range(start, end + 1)) + visitor = _TopLevelStatementsVisitor(lines_set) + _ = list(visitor.visit(src_node)) # Consume all results. + _convert_unchanged_line_by_line(src_node, lines_set) + + +def _contains_standalone_comment(node: LN) -> bool: + if isinstance(node, Leaf): + return node.type == STANDALONE_COMMENT + else: + for child in node.children: + if _contains_standalone_comment(child): + return True + return False + + +class _TopLevelStatementsVisitor(Visitor[None]): + """ + A node visitor that converts unchanged top-level statements to + STANDALONE_COMMENT. + + This is used in addition to _convert_unchanged_line_by_line, to + speed up formatting when there are unchanged top-level + classes/functions/statements. + """ + + def __init__(self, lines_set: Set[int]): + self._lines_set = lines_set + + def visit_simple_stmt(self, node: Node) -> Iterator[None]: + # This is only called for top-level statements, since `visit_suite` + # won't visit its children nodes. + yield from [] + newline_leaf = last_leaf(node) + if not newline_leaf: + return + assert ( + newline_leaf.type == NEWLINE + ), f"Unexpectedly found leaf.type={newline_leaf.type}" + # We need to find the furthest ancestor with the NEWLINE as the last + # leaf, since a `suite` can simply be a `simple_stmt` when it puts + # its body on the same line. Example: `if cond: pass`. + ancestor = furthest_ancestor_with_last_leaf(newline_leaf) + if not _get_line_range(ancestor).intersection(self._lines_set): + _convert_node_to_standalone_comment(ancestor) + + def visit_suite(self, node: Node) -> Iterator[None]: + yield from [] + # If there is a STANDALONE_COMMENT node, it means parts of the node tree + # have fmt on/off/skip markers. Those STANDALONE_COMMENT nodes can't + # be simply converted by calling str(node). So we just don't convert + # here. + if _contains_standalone_comment(node): + return + # Find the semantic parent of this suite. For `async_stmt` and + # `async_funcdef`, the ASYNC token is defined on a separate level by the + # grammar. + semantic_parent = node.parent + if semantic_parent is not None: + if ( + semantic_parent.prev_sibling is not None + and semantic_parent.prev_sibling.type == ASYNC + ): + semantic_parent = semantic_parent.parent + if semantic_parent is not None and not _get_line_range( + semantic_parent + ).intersection(self._lines_set): + _convert_node_to_standalone_comment(semantic_parent) + + +def _convert_unchanged_line_by_line(node: Node, lines_set: Set[int]) -> None: + """Converts unchanged to STANDALONE_COMMENT line by line.""" + for leaf in node.leaves(): + if leaf.type != NEWLINE: + # We only consider "unwrapped lines", which are divided by the NEWLINE + # token. + continue + if leaf.parent and leaf.parent.type == syms.match_stmt: + # The `suite` node is defined as: + # match_stmt: "match" subject_expr ':' NEWLINE INDENT case_block+ DEDENT + # Here we need to check `subject_expr`. The `case_block+` will be + # checked by their own NEWLINEs. + nodes_to_ignore: List[LN] = [] + prev_sibling = leaf.prev_sibling + while prev_sibling: + nodes_to_ignore.insert(0, prev_sibling) + prev_sibling = prev_sibling.prev_sibling + if not _get_line_range(nodes_to_ignore).intersection(lines_set): + _convert_nodes_to_standalone_comment(nodes_to_ignore, newline=leaf) + elif leaf.parent and leaf.parent.type == syms.suite: + # The `suite` node is defined as: + # suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT + # We will check `simple_stmt` and `stmt+` separately against the lines set + parent_sibling = leaf.parent.prev_sibling + nodes_to_ignore = [] + while parent_sibling and not parent_sibling.type == syms.suite: + # NOTE: Multiple suite nodes can exist as siblings in e.g. `if_stmt`. + nodes_to_ignore.insert(0, parent_sibling) + parent_sibling = parent_sibling.prev_sibling + # Special case for `async_stmt` and `async_funcdef` where the ASYNC + # token is on the grandparent node. + grandparent = leaf.parent.parent + if ( + grandparent is not None + and grandparent.prev_sibling is not None + and grandparent.prev_sibling.type == ASYNC + ): + nodes_to_ignore.insert(0, grandparent.prev_sibling) + if not _get_line_range(nodes_to_ignore).intersection(lines_set): + _convert_nodes_to_standalone_comment(nodes_to_ignore, newline=leaf) + else: + ancestor = furthest_ancestor_with_last_leaf(leaf) + # Consider multiple decorators as a whole block, as their + # newlines have different behaviors than the rest of the grammar. + if ( + ancestor.type == syms.decorator + and ancestor.parent + and ancestor.parent.type == syms.decorators + ): + ancestor = ancestor.parent + if not _get_line_range(ancestor).intersection(lines_set): + _convert_node_to_standalone_comment(ancestor) + + +def _convert_node_to_standalone_comment(node: LN) -> None: + """Convert node to STANDALONE_COMMENT by modifying the tree inline.""" + parent = node.parent + if not parent: + return + first = first_leaf(node) + last = last_leaf(node) + if not first or not last: + return + if first is last: + # This can happen on the following edge cases: + # 1. A block of `# fmt: off/on` code except the `# fmt: on` is placed + # on the end of the last line instead of on a new line. + # 2. A single backslash on its own line followed by a comment line. + # Ideally we don't want to format them when not requested, but fixing + # isn't easy. These cases are also badly formatted code, so it isn't + # too bad we reformat them. + return + # The prefix contains comments and indentation whitespaces. They are + # reformatted accordingly to the correct indentation level. + # This also means the indentation will be changed on the unchanged lines, and + # this is actually required to not break incremental reformatting. + prefix = first.prefix + first.prefix = "" + index = node.remove() + if index is not None: + # Remove the '\n', as STANDALONE_COMMENT will have '\n' appended when + # generating the formatted code. + value = str(node)[:-1] + parent.insert_child( + index, + Leaf( + STANDALONE_COMMENT, + value, + prefix=prefix, + fmt_pass_converted_first_leaf=first, + ), + ) + + +def _convert_nodes_to_standalone_comment(nodes: Sequence[LN], *, newline: Leaf) -> None: + """Convert nodes to STANDALONE_COMMENT by modifying the tree inline.""" + if not nodes: + return + parent = nodes[0].parent + first = first_leaf(nodes[0]) + if not parent or not first: + return + prefix = first.prefix + first.prefix = "" + value = "".join(str(node) for node in nodes) + # The prefix comment on the NEWLINE leaf is the trailing comment of the statement. + if newline.prefix: + value += newline.prefix + newline.prefix = "" + index = nodes[0].remove() + for node in nodes[1:]: + node.remove() + if index is not None: + parent.insert_child( + index, + Leaf( + STANDALONE_COMMENT, + value, + prefix=prefix, + fmt_pass_converted_first_leaf=first, + ), + ) + + +def _leaf_line_end(leaf: Leaf) -> int: + """Returns the line number of the leaf node's last line.""" + if leaf.type == NEWLINE: + return leaf.lineno + else: + # Leaf nodes like multiline strings can occupy multiple lines. + return leaf.lineno + str(leaf).count("\n") + + +def _get_line_range(node_or_nodes: Union[LN, List[LN]]) -> Set[int]: + """Returns the line range of this node or list of nodes.""" + if isinstance(node_or_nodes, list): + nodes = node_or_nodes + if not nodes: + return set() + first = first_leaf(nodes[0]) + last = last_leaf(nodes[-1]) + if first and last: + line_start = first.lineno + line_end = _leaf_line_end(last) + return set(range(line_start, line_end + 1)) + else: + return set() + else: + node = node_or_nodes + if isinstance(node, Leaf): + return set(range(node.lineno, _leaf_line_end(node) + 1)) + else: + first = first_leaf(node) + last = last_leaf(node) + if first and last: + return set(range(first.lineno, _leaf_line_end(last) + 1)) + else: + return set() + + +@dataclass +class _LinesMapping: + """1-based lines mapping from original source to modified source. + + Lines [original_start, original_end] from original source + are mapped to [modified_start, modified_end]. + + The ranges are inclusive on both ends. + """ + + original_start: int + original_end: int + modified_start: int + modified_end: int + # Whether this range corresponds to a changed block, or an unchanged block. + is_changed_block: bool + + +def _calculate_lines_mappings( + original_source: str, + modified_source: str, +) -> Sequence[_LinesMapping]: + """Returns a sequence of _LinesMapping by diffing the sources. + + For example, given the following diff: + import re + - def func(arg1, + - arg2, arg3): + + def func(arg1, arg2, arg3): + pass + It returns the following mappings: + original -> modified + (1, 1) -> (1, 1), is_changed_block=False (the "import re" line) + (2, 3) -> (2, 2), is_changed_block=True (the diff) + (4, 4) -> (3, 3), is_changed_block=False (the "pass" line) + + You can think of this visually as if it brings up a side-by-side diff, and tries + to map the line ranges from the left side to the right side: + + (1, 1)->(1, 1) 1. import re 1. import re + (2, 3)->(2, 2) 2. def func(arg1, 2. def func(arg1, arg2, arg3): + 3. arg2, arg3): + (4, 4)->(3, 3) 4. pass 3. pass + + Args: + original_source: the original source. + modified_source: the modified source. + """ + matcher = difflib.SequenceMatcher( + None, + original_source.splitlines(keepends=True), + modified_source.splitlines(keepends=True), + ) + matching_blocks = matcher.get_matching_blocks() + lines_mappings: List[_LinesMapping] = [] + # matching_blocks is a sequence of "same block of code ranges", see + # https://docs.python.org/3/library/difflib.html#difflib.SequenceMatcher.get_matching_blocks + # Each block corresponds to a _LinesMapping with is_changed_block=False, + # and the ranges between two blocks corresponds to a _LinesMapping with + # is_changed_block=True, + # NOTE: matching_blocks is 0-based, but _LinesMapping is 1-based. + for i, block in enumerate(matching_blocks): + if i == 0: + if block.a != 0 or block.b != 0: + lines_mappings.append( + _LinesMapping( + original_start=1, + original_end=block.a, + modified_start=1, + modified_end=block.b, + is_changed_block=False, + ) + ) + else: + previous_block = matching_blocks[i - 1] + lines_mappings.append( + _LinesMapping( + original_start=previous_block.a + previous_block.size + 1, + original_end=block.a, + modified_start=previous_block.b + previous_block.size + 1, + modified_end=block.b, + is_changed_block=True, + ) + ) + if i < len(matching_blocks) - 1: + lines_mappings.append( + _LinesMapping( + original_start=block.a + 1, + original_end=block.a + block.size, + modified_start=block.b + 1, + modified_end=block.b + block.size, + is_changed_block=False, + ) + ) + return lines_mappings + + +def _find_lines_mapping_index( + original_line: int, + lines_mappings: Sequence[_LinesMapping], + start_index: int, +) -> int: + """Returns the original index of the lines mappings for the original line.""" + index = start_index + while index < len(lines_mappings): + mapping = lines_mappings[index] + if ( + mapping.original_start <= original_line + and original_line <= mapping.original_end + ): + return index + index += 1 + return index diff --git a/src/black/report.py b/src/black/report.py index a507671e4c0..89899f2f389 100644 --- a/src/black/report.py +++ b/src/black/report.py @@ -1,6 +1,7 @@ """ Summarize Black runs to users. """ + from dataclasses import dataclass from enum import Enum from pathlib import Path diff --git a/src/black/rusty.py b/src/black/rusty.py index 84a80b5a2c2..ebd4c052d1f 100644 --- a/src/black/rusty.py +++ b/src/black/rusty.py @@ -2,6 +2,7 @@ See https://doc.rust-lang.org/book/ch09-00-error-handling.html. """ + from typing import Generic, TypeVar, Union T = TypeVar("T") diff --git a/src/black/strings.py b/src/black/strings.py index 3e3bc12fe72..0d30f09ed11 100644 --- a/src/black/strings.py +++ b/src/black/strings.py @@ -5,16 +5,11 @@ import re import sys from functools import lru_cache -from typing import List, Match, Pattern +from typing import Final, List, Match, Pattern +from black._width_table import WIDTH_TABLE from blib2to3.pytree import Leaf -if sys.version_info < (3, 8): - from typing_extensions import Final -else: - from typing import Final - - STRING_PREFIX_CHARS: Final = "furbFURB" # All possible string prefix characters. STRING_PREFIX_RE: Final = re.compile( r"^([" + STRING_PREFIX_CHARS + r"]*)(.*)$", re.DOTALL @@ -278,3 +273,57 @@ def replace(m: Match[str]) -> str: return back_slashes + "N{" + groups["N"].upper() + "}" leaf.value = re.sub(UNICODE_ESCAPE_RE, replace, text) + + +@lru_cache(maxsize=4096) +def char_width(char: str) -> int: + """Return the width of a single character as it would be displayed in a + terminal or editor (which respects Unicode East Asian Width). + + Full width characters are counted as 2, while half width characters are + counted as 1. Also control characters are counted as 0. + """ + table = WIDTH_TABLE + codepoint = ord(char) + highest = len(table) - 1 + lowest = 0 + idx = highest // 2 + while True: + start_codepoint, end_codepoint, width = table[idx] + if codepoint < start_codepoint: + highest = idx - 1 + elif codepoint > end_codepoint: + lowest = idx + 1 + else: + return 0 if width < 0 else width + if highest < lowest: + break + idx = (highest + lowest) // 2 + return 1 + + +def str_width(line_str: str) -> int: + """Return the width of `line_str` as it would be displayed in a terminal + or editor (which respects Unicode East Asian Width). + + You could utilize this function to determine, for example, if a string + is too wide to display in a terminal or editor. + """ + if line_str.isascii(): + # Fast path for a line consisting of only ASCII characters + return len(line_str) + return sum(map(char_width, line_str)) + + +def count_chars_in_width(line_str: str, max_width: int) -> int: + """Count the number of characters in `line_str` that would fit in a + terminal or editor of `max_width` (which respects Unicode East Asian + Width). + """ + total_width = 0 + for i, char in enumerate(line_str): + width = char_width(char) + if width + total_width > max_width: + return i + total_width += width + return len(line_str) diff --git a/src/black/trans.py b/src/black/trans.py index 2360c13f06a..ab3197fa6df 100644 --- a/src/black/trans.py +++ b/src/black/trans.py @@ -1,8 +1,8 @@ """ String transformers that can split and merge strings. """ + import re -import sys from abc import ABC, abstractmethod from collections import defaultdict from dataclasses import dataclass @@ -12,9 +12,11 @@ ClassVar, Collection, Dict, + Final, Iterable, Iterator, List, + Literal, Optional, Sequence, Set, @@ -23,16 +25,11 @@ Union, ) -if sys.version_info < (3, 8): - from typing_extensions import Final, Literal -else: - from typing import Literal, Final - from mypy_extensions import trait from black.comments import contains_pragma_comment from black.lines import Line, append_leaves -from black.mode import Feature +from black.mode import Feature, Mode from black.nodes import ( CLOSING_BRACKETS, OPENING_BRACKETS, @@ -48,9 +45,11 @@ from black.rusty import Err, Ok, Result from black.strings import ( assert_is_leaf_string, + count_chars_in_width, get_string_prefix, has_triple_quotes, normalize_string_quotes, + str_width, ) from blib2to3.pgen2 import token from blib2to3.pytree import Leaf, Node @@ -63,7 +62,7 @@ class CannotTransform(Exception): # types T = TypeVar("T") LN = Union[Leaf, Node] -Transformer = Callable[[Line, Collection[Feature]], Iterator[Line]] +Transformer = Callable[[Line, Collection[Feature], Mode], Iterator[Line]] Index = int NodeType = int ParserState = int @@ -71,6 +70,8 @@ class CannotTransform(Exception): TResult = Result[T, CannotTransform] # (T)ransform Result TMatchResult = TResult[List[Index]] +SPLIT_SAFE_CHARS = frozenset(["\u3001", "\u3002", "\uff0c"]) # East Asian stops + def TErr(err_msg: str) -> Err[CannotTransform]: """(T)ransform Err @@ -81,7 +82,9 @@ def TErr(err_msg: str) -> Err[CannotTransform]: return Err(cant_transform) -def hug_power_op(line: Line, features: Collection[Feature]) -> Iterator[Line]: +def hug_power_op( + line: Line, features: Collection[Feature], mode: Mode +) -> Iterator[Line]: """A transformer which normalizes spacing around power operators.""" # Performance optimization to avoid unnecessary Leaf clones and other ops. @@ -199,11 +202,11 @@ def do_match(self, line: Line) -> TMatchResult: """ Returns: * Ok(string_indices) such that for each index, `line.leaves[index]` - is our target string if a match was able to be made. For - transformers that don't result in more lines (e.g. StringMerger, - StringParenStripper), multiple matches and transforms are done at - once to reduce the complexity. - OR + is our target string if a match was able to be made. For + transformers that don't result in more lines (e.g. StringMerger, + StringParenStripper), multiple matches and transforms are done at + once to reduce the complexity. + OR * Err(CannotTransform), if no match could be made. """ @@ -214,12 +217,12 @@ def do_transform( """ Yields: * Ok(new_line) where new_line is the new transformed line. - OR + OR * Err(CannotTransform) if the transformation failed for some reason. The - `do_match(...)` template method should usually be used to reject - the form of the given Line, but in some cases it is difficult to - know whether or not a Line meets the StringTransformer's - requirements until the transformation is already midway. + `do_match(...)` template method should usually be used to reject + the form of the given Line, but in some cases it is difficult to + know whether or not a Line meets the StringTransformer's + requirements until the transformation is already midway. Side Effects: This method should NOT mutate @line directly, but it MAY mutate the @@ -228,7 +231,9 @@ def do_transform( yield an CannotTransform after that point.) """ - def __call__(self, line: Line, _features: Collection[Feature]) -> Iterator[Line]: + def __call__( + self, line: Line, _features: Collection[Feature], _mode: Mode + ) -> Iterator[Line]: """ StringTransformer instances have a call signature that mirrors that of the Transformer type. @@ -327,8 +332,8 @@ def pop_custom_splits(self, string: str) -> List[CustomSplit]: Returns: * A list of the custom splits that are mapped to @string, if any - exist. - OR + exist. + OR * [], otherwise. Side Effects: @@ -357,14 +362,14 @@ class StringMerger(StringTransformer, CustomSplitMapMixin): Requirements: (A) The line contains adjacent strings such that ALL of the validation checks listed in StringMerger._validate_msg(...)'s docstring pass. - OR + OR (B) The line contains a string which uses line continuation backslashes. Transformations: Depending on which of the two requirements above where met, either: (A) The string group associated with the target string is merged. - OR + OR (B) All line-continuation backslashes are removed from the target string. Collaborations: @@ -385,7 +390,19 @@ def do_match(self, line: Line) -> TMatchResult: and is_valid_index(idx + 1) and LL[idx + 1].type == token.STRING ): - if not is_part_of_annotation(leaf): + # Let's check if the string group contains an inline comment + # If we have a comment inline, we don't merge the strings + contains_comment = False + i = idx + while is_valid_index(i): + if LL[i].type != token.STRING: + break + if line.comments_after(LL[i]): + contains_comment = True + break + i += 1 + + if not is_part_of_annotation(leaf) and not contains_comment: string_indices.append(idx) # Advance to the next non-STRING leaf. @@ -573,11 +590,22 @@ def make_naked(string: str, string_prefix: str) -> str: """ assert_is_leaf_string(string) if "f" in string_prefix: - string = _toggle_fexpr_quotes(string, QUOTE) - # After quotes toggling, quotes in expressions won't be escaped - # because quotes can't be reused in f-strings. So we can simply - # let the escaping logic below run without knowing f-string - # expressions. + f_expressions = ( + string[span[0] + 1 : span[1] - 1] # +-1 to get rid of curly braces + for span in iter_fexpr_spans(string) + ) + debug_expressions_contain_visible_quotes = any( + re.search(r".*[\'\"].*(? TResult[None]: Returns: * Ok(None), if ALL of the requirements are met. - OR + OR * Err(CannotTransform), if ANY of the requirements are NOT met. """ LL = line.leaves @@ -1160,7 +1194,7 @@ def _get_max_string_length(self, line: Line, string_idx: int) -> int: # WMA4 the length of the inline comment. offset += len(comment_leaf.value) - max_string_length = self.line_length - offset + max_string_length = count_chars_in_width(str(line), self.line_length - offset) return max_string_length @staticmethod @@ -1178,19 +1212,33 @@ def _prefer_paren_wrap_match(LL: List[Leaf]) -> Optional[int]: if LL[0].type != token.STRING: return None - # If the string is surrounded by commas (or is the first/last child)... - prev_sibling = LL[0].prev_sibling - next_sibling = LL[0].next_sibling - if not prev_sibling and not next_sibling and parent_type(LL[0]) == syms.atom: - # If it's an atom string, we need to check the parent atom's siblings. - parent = LL[0].parent - assert parent is not None # For type checkers. - prev_sibling = parent.prev_sibling - next_sibling = parent.next_sibling - if (not prev_sibling or prev_sibling.type == token.COMMA) and ( - not next_sibling or next_sibling.type == token.COMMA + matching_nodes = [ + syms.listmaker, + syms.dictsetmaker, + syms.testlist_gexp, + ] + # If the string is an immediate child of a list/set/tuple literal... + if ( + parent_type(LL[0]) in matching_nodes + or parent_type(LL[0].parent) in matching_nodes ): - return 0 + # And the string is surrounded by commas (or is the first/last child)... + prev_sibling = LL[0].prev_sibling + next_sibling = LL[0].next_sibling + if ( + not prev_sibling + and not next_sibling + and parent_type(LL[0]) == syms.atom + ): + # If it's an atom string, we need to check the parent atom's siblings. + parent = LL[0].parent + assert parent is not None # For type checkers. + prev_sibling = parent.prev_sibling + next_sibling = parent.next_sibling + if (not prev_sibling or prev_sibling.type == token.COMMA) and ( + not next_sibling or next_sibling.type == token.COMMA + ): + return 0 return None @@ -1277,9 +1325,9 @@ class StringSplitter(BaseStringSplitter, CustomSplitMapMixin): Requirements: * The line consists ONLY of a single string (possibly prefixed by a - string operator [e.g. '+' or '==']), MAYBE a string trailer, and MAYBE - a trailing comma. - AND + string operator [e.g. '+' or '==']), MAYBE a string trailer, and MAYBE + a trailing comma. + AND * All of the requirements listed in BaseStringSplitter's docstring. Transformations: @@ -1415,11 +1463,13 @@ def maybe_append_string_operators(new_line: Line) -> None: is_valid_index(string_idx + 1) and LL[string_idx + 1].type == token.COMMA ) - def max_last_string() -> int: + def max_last_string_column() -> int: """ Returns: - The max allowed length of the string value used for the last - line we will construct. + The max allowed width of the string value used for the last + line we will construct. Note that this value means the width + rather than the number of characters (e.g., many East Asian + characters expand to two columns). """ result = self.line_length result -= line.depth * 4 @@ -1427,14 +1477,14 @@ def max_last_string() -> int: result -= string_op_leaves_length return result - # --- Calculate Max Break Index (for string value) + # --- Calculate Max Break Width (for string value) # We start with the line length limit - max_break_idx = self.line_length + max_break_width = self.line_length # The last index of a string of length N is N-1. - max_break_idx -= 1 + max_break_width -= 1 # Leading whitespace is not present in the string value (e.g. Leaf.value). - max_break_idx -= line.depth * 4 - if max_break_idx < 0: + max_break_width -= line.depth * 4 + if max_break_width < 0: yield TErr( f"Unable to split {LL[string_idx].value} at such high of a line depth:" f" {line.depth}" @@ -1447,7 +1497,7 @@ def max_last_string() -> int: # line limit. use_custom_breakpoints = bool( custom_splits - and all(csplit.break_idx <= max_break_idx for csplit in custom_splits) + and all(csplit.break_idx <= max_break_width for csplit in custom_splits) ) # Temporary storage for the remaining chunk of the string line that @@ -1463,7 +1513,7 @@ def more_splits_should_be_made() -> bool: if use_custom_breakpoints: return len(custom_splits) > 1 else: - return len(rest_value) > max_last_string() + return str_width(rest_value) > max_last_string_column() string_line_results: List[Ok[Line]] = [] while more_splits_should_be_made(): @@ -1473,7 +1523,10 @@ def more_splits_should_be_made() -> bool: break_idx = csplit.break_idx else: # Algorithmic Split (automatic) - max_bidx = max_break_idx - string_op_leaves_length + max_bidx = ( + count_chars_in_width(rest_value, max_break_width) + - string_op_leaves_length + ) maybe_break_idx = self._get_break_idx(rest_value, max_bidx) if maybe_break_idx is None: # If we are unable to algorithmically determine a good split @@ -1570,7 +1623,7 @@ def more_splits_should_be_made() -> bool: # Try to fit them all on the same line with the last substring... if ( - len(temp_value) <= max_last_string() + str_width(temp_value) <= max_last_string_column() or LL[string_idx + 1].type == token.COMMA ): last_line.append(rest_leaf) @@ -1690,6 +1743,7 @@ def passes_all_checks(i: Index) -> bool: section of this classes' docstring would be be met by returning @i. """ is_space = string[i] == " " + is_split_safe = is_valid_index(i - 1) and string[i - 1] in SPLIT_SAFE_CHARS is_not_escaped = True j = i - 1 @@ -1702,7 +1756,7 @@ def passes_all_checks(i: Index) -> bool: and len(string[:i]) >= self.MIN_SUBSTR_SIZE ) return ( - is_space + (is_space or is_split_safe) and is_not_escaped and is_big_enough and not breaks_unsplittable_expression(i) @@ -1780,25 +1834,26 @@ class StringParenWrapper(BaseStringSplitter, CustomSplitMapMixin): addition to the requirements listed below: * The line is a return/yield statement, which returns/yields a string. - OR + OR * The line is part of a ternary expression (e.g. `x = y if cond else - z`) such that the line starts with `else `, where is - some string. - OR + z`) such that the line starts with `else `, where is + some string. + OR * The line is an assert statement, which ends with a string. - OR + OR * The line is an assignment statement (e.g. `x = ` or `x += - `) such that the variable is being assigned the value of some - string. - OR + `) such that the variable is being assigned the value of some + string. + OR * The line is a dictionary key assignment where some valid key is being - assigned the value of some string. - OR + assigned the value of some string. + OR * The line is an lambda expression and the value is a string. - OR + OR * The line starts with an "atom" string that prefers to be wrapped in - parens. It's preferred to be wrapped when the string is surrounded by - commas (or is the first/last child). + parens. It's preferred to be wrapped when it's is an immediate child of + a list/set/tuple literal, AND the string is surrounded by commas (or is + the first/last child). Transformations: The chosen string is wrapped in parentheses and then split at the LPAR. @@ -1847,11 +1902,13 @@ def do_splitter_match(self, line: Line) -> TMatchResult: if string_idx is not None: string_value = line.leaves[string_idx].value - # If the string has no spaces... - if " " not in string_value: + # If the string has neither spaces nor East Asian stops... + if not any( + char == " " or char in SPLIT_SAFE_CHARS for char in string_value + ): # And will still violate the line length limit when split... - max_string_length = self.line_length - ((line.depth + 1) * 4) - if len(string_value) > max_string_length: + max_string_width = self.line_length - ((line.depth + 1) * 4) + if str_width(string_value) > max_string_width: # And has no associated custom splits... if not self.has_custom_splits(string_value): # Then we should NOT put this string on its own line. @@ -2242,7 +2299,7 @@ def parse(self, leaves: List[Leaf], string_idx: int) -> int: Returns: The index directly after the last leaf which is apart of the string trailer, if a "trailer" exists. - OR + OR @string_idx + 1, if no string "trailer" exists. """ assert leaves[string_idx].type == token.STRING @@ -2256,11 +2313,11 @@ def _next_state(self, leaf: Leaf) -> bool: """ Pre-conditions: * On the first call to this function, @leaf MUST be the leaf that - was directly after the string leaf in question (e.g. if our target - string is `line.leaves[i]` then the first call to this method must - be `line.leaves[i + 1]`). + was directly after the string leaf in question (e.g. if our target + string is `line.leaves[i]` then the first call to this method must + be `line.leaves[i + 1]`). * On the next call to this function, the leaf parameter passed in - MUST be the leaf directly following @leaf. + MUST be the leaf directly following @leaf. Returns: True iff @leaf is apart of the string's trailer. diff --git a/src/blackd/__init__.py b/src/blackd/__init__.py index ba4750b8298..6b0f3d33295 100644 --- a/src/blackd/__init__.py +++ b/src/blackd/__init__.py @@ -1,7 +1,7 @@ import asyncio import logging from concurrent.futures import Executor, ProcessPoolExecutor -from datetime import datetime +from datetime import datetime, timezone from functools import partial from multiprocessing import freeze_support from typing import Set, Tuple @@ -59,9 +59,15 @@ class InvalidVariantHeader(Exception): @click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.option( - "--bind-host", type=str, help="Address to bind the server to.", default="localhost" + "--bind-host", + type=str, + help="Address to bind the server to.", + default="localhost", + show_default=True, +) +@click.option( + "--bind-port", type=int, help="Port to listen on", default=45484, show_default=True ) -@click.option("--bind-port", type=int, help="Port to listen on", default=45484) @click.version_option(version=black.__version__) def main(bind_host: str, bind_port: int) -> None: logging.basicConfig(level=logging.INFO) @@ -132,7 +138,7 @@ async def handle(request: web.Request, executor: Executor) -> web.Response: req_bytes = await request.content.read() charset = request.charset if request.charset is not None else "utf8" req_str = req_bytes.decode(charset) - then = datetime.utcnow() + then = datetime.now(timezone.utc) header = "" if skip_source_first_line: @@ -146,7 +152,8 @@ async def handle(request: web.Request, executor: Executor) -> web.Response: ) # Preserve CRLF line endings - if req_str[req_str.find("\n") - 1] == "\r": + nl = req_str.find("\n") + if nl > 0 and req_str[nl - 1] == "\r": formatted_str = formatted_str.replace("\n", "\r\n") # If, after swapping line endings, nothing changed, then say so if formatted_str == req_str: @@ -159,9 +166,9 @@ async def handle(request: web.Request, executor: Executor) -> web.Response: # Only output the diff in the HTTP response only_diff = bool(request.headers.get(DIFF_HEADER, False)) if only_diff: - now = datetime.utcnow() - src_name = f"In\t{then} +0000" - dst_name = f"Out\t{now} +0000" + now = datetime.now(timezone.utc) + src_name = f"In\t{then}" + dst_name = f"Out\t{now}" loop = asyncio.get_event_loop() formatted_str = await loop.run_in_executor( executor, @@ -219,7 +226,6 @@ def parse_python_variant_header(value: str) -> Tuple[bool, Set[black.TargetVersi def patched_main() -> None: maybe_install_uvloop() freeze_support() - black.patch_click() main() diff --git a/src/blib2to3/Grammar.txt b/src/blib2to3/Grammar.txt index bd8a452a386..5db78723cec 100644 --- a/src/blib2to3/Grammar.txt +++ b/src/blib2to3/Grammar.txt @@ -12,11 +12,17 @@ file_input: (NEWLINE | stmt)* ENDMARKER single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE eval_input: testlist NEWLINE* ENDMARKER +typevar: NAME [':' expr] +paramspec: '**' NAME +typevartuple: '*' NAME +typeparam: typevar | paramspec | typevartuple +typeparams: '[' typeparam (',' typeparam)* [','] ']' + decorator: '@' namedexpr_test NEWLINE decorators: decorator+ decorated: decorators (classdef | funcdef | async_funcdef) async_funcdef: ASYNC funcdef -funcdef: 'def' NAME parameters ['->' test] ':' suite +funcdef: 'def' NAME [typeparams] parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' # The following definition for typedarglist is equivalent to this set of rules: @@ -74,8 +80,8 @@ vfplist: vfpdef (',' vfpdef)* [','] stmt: simple_stmt | compound_stmt simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE -small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | - import_stmt | global_stmt | exec_stmt | assert_stmt) +small_stmt: (type_stmt | expr_stmt | del_stmt | pass_stmt | flow_stmt | + import_stmt | global_stmt | assert_stmt) expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) | ('=' (yield_expr|testlist_star_expr))*) annassign: ':' test ['=' (yield_expr|testlist_star_expr)] @@ -83,8 +89,6 @@ testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//=') # For normal and annotated assignments, additional restrictions enforced by the interpreter -print_stmt: 'print' ( [ test (',' test)* [','] ] | - '>>' test [ (',' test)+ [','] ] ) del_stmt: 'del' exprlist pass_stmt: 'pass' flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt @@ -103,8 +107,8 @@ import_as_names: import_as_name (',' import_as_name)* [','] dotted_as_names: dotted_as_name (',' dotted_as_name)* dotted_name: NAME ('.' NAME)* global_stmt: ('global' | 'nonlocal') NAME (',' NAME)* -exec_stmt: 'exec' expr ['in' test [',' test]] assert_stmt: 'assert' test [',' test] +type_stmt: "type" NAME [typeparams] '=' test compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt | match_stmt async_stmt: ASYNC (funcdef | with_stmt | for_stmt) @@ -174,7 +178,7 @@ dictsetmaker: ( ((test ':' asexpr_test | '**' expr) ((test [':=' test] | star_expr) (comp_for | (',' (test [':=' test] | star_expr))* [','])) ) -classdef: 'class' NAME ['(' [arglist] ')'] ':' suite +classdef: 'class' NAME [typeparams] ['(' [arglist] ')'] ':' suite arglist: argument (',' argument)* [','] diff --git a/src/blib2to3/README b/src/blib2to3/README index 0d3c607c9c7..38b04158ddb 100644 --- a/src/blib2to3/README +++ b/src/blib2to3/README @@ -1,18 +1,19 @@ -A subset of lib2to3 taken from Python 3.7.0b2. -Commit hash: 9c17e3a1987004b8bcfbe423953aad84493a7984 +A subset of lib2to3 taken from Python 3.7.0b2. Commit hash: +9c17e3a1987004b8bcfbe423953aad84493a7984 Reasons for forking: + - consistent handling of f-strings for users of Python < 3.6.2 -- backport of BPO-33064 that fixes parsing files with trailing commas after - *args and **kwargs -- backport of GH-6143 that restores the ability to reformat legacy usage of - `async` +- backport of BPO-33064 that fixes parsing files with trailing commas after \*args and + \*\*kwargs +- backport of GH-6143 that restores the ability to reformat legacy usage of `async` - support all types of string literals - better ability to debug (better reprs) - INDENT and DEDENT don't hold whitespace and comment prefixes - ability to Cythonize Change Log: + - Changes default logger used by Driver - Backported the following upstream parser changes: - "bpo-42381: Allow walrus in set literals and set comprehensions (GH-23332)" diff --git a/src/blib2to3/pgen2/conv.py b/src/blib2to3/pgen2/conv.py index fa9825e54d6..04eccfa1d4b 100644 --- a/src/blib2to3/pgen2/conv.py +++ b/src/blib2to3/pgen2/conv.py @@ -63,7 +63,7 @@ def parse_graminit_h(self, filename): try: f = open(filename) except OSError as err: - print("Can't open %s: %s" % (filename, err)) + print(f"Can't open {filename}: {err}") return False self.symbol2number = {} self.number2symbol = {} @@ -72,7 +72,7 @@ def parse_graminit_h(self, filename): lineno += 1 mo = re.match(r"^#define\s+(\w+)\s+(\d+)$", line) if not mo and line.strip(): - print("%s(%s): can't parse %s" % (filename, lineno, line.strip())) + print(f"{filename}({lineno}): can't parse {line.strip()}") else: symbol, number = mo.groups() number = int(number) @@ -113,7 +113,7 @@ def parse_graminit_c(self, filename): try: f = open(filename) except OSError as err: - print("Can't open %s: %s" % (filename, err)) + print(f"Can't open {filename}: {err}") return False # The code below essentially uses f's iterator-ness! lineno = 0 diff --git a/src/blib2to3/pgen2/driver.py b/src/blib2to3/pgen2/driver.py index daf271dfa9a..be3984437a8 100644 --- a/src/blib2to3/pgen2/driver.py +++ b/src/blib2to3/pgen2/driver.py @@ -17,33 +17,21 @@ # Python imports import io -import os import logging +import os import pkgutil import sys -from typing import ( - Any, - cast, - IO, - Iterable, - List, - Optional, - Text, - Iterator, - Tuple, - TypeVar, - Generic, - Union, -) from contextlib import contextmanager from dataclasses import dataclass, field - -# Pgen imports -from . import grammar, parse, token, tokenize, pgen from logging import Logger -from blib2to3.pytree import NL +from typing import IO, Any, Iterable, Iterator, List, Optional, Tuple, Union, cast + from blib2to3.pgen2.grammar import Grammar from blib2to3.pgen2.tokenize import GoodTokenInfo +from blib2to3.pytree import NL + +# Pgen imports +from . import grammar, parse, pgen, token, tokenize Path = Union[str, "os.PathLike[str]"] @@ -107,7 +95,7 @@ def __next__(self) -> Any: def can_advance(self, to: int) -> bool: # Try to eat, fail if it can't. The eat operation is cached - # so there wont be any additional cost of eating here + # so there won't be any additional cost of eating here try: self.eat(to) except StopIteration: @@ -116,7 +104,7 @@ def can_advance(self, to: int) -> bool: return True -class Driver(object): +class Driver: def __init__(self, grammar: Grammar, logger: Optional[Logger] = None) -> None: self.grammar = grammar if logger is None: @@ -189,30 +177,30 @@ def parse_tokens(self, tokens: Iterable[GoodTokenInfo], debug: bool = False) -> assert p.rootnode is not None return p.rootnode - def parse_stream_raw(self, stream: IO[Text], debug: bool = False) -> NL: + def parse_stream_raw(self, stream: IO[str], debug: bool = False) -> NL: """Parse a stream and return the syntax tree.""" tokens = tokenize.generate_tokens(stream.readline, grammar=self.grammar) return self.parse_tokens(tokens, debug) - def parse_stream(self, stream: IO[Text], debug: bool = False) -> NL: + def parse_stream(self, stream: IO[str], debug: bool = False) -> NL: """Parse a stream and return the syntax tree.""" return self.parse_stream_raw(stream, debug) def parse_file( - self, filename: Path, encoding: Optional[Text] = None, debug: bool = False + self, filename: Path, encoding: Optional[str] = None, debug: bool = False ) -> NL: """Parse a file and return the syntax tree.""" - with io.open(filename, "r", encoding=encoding) as stream: + with open(filename, encoding=encoding) as stream: return self.parse_stream(stream, debug) - def parse_string(self, text: Text, debug: bool = False) -> NL: + def parse_string(self, text: str, debug: bool = False) -> NL: """Parse a string and return the syntax tree.""" tokens = tokenize.generate_tokens( io.StringIO(text).readline, grammar=self.grammar ) return self.parse_tokens(tokens, debug) - def _partially_consume_prefix(self, prefix: Text, column: int) -> Tuple[Text, Text]: + def _partially_consume_prefix(self, prefix: str, column: int) -> Tuple[str, str]: lines: List[str] = [] current_line = "" current_column = 0 @@ -234,13 +222,15 @@ def _partially_consume_prefix(self, prefix: Text, column: int) -> Tuple[Text, Te elif char == "\n": # unexpected empty line current_column = 0 + elif char == "\f": + current_column = 0 else: # indent is finished wait_for_nl = True return "".join(lines), current_line -def _generate_pickle_name(gt: Path, cache_dir: Optional[Path] = None) -> Text: +def _generate_pickle_name(gt: Path, cache_dir: Optional[Path] = None) -> str: head, tail = os.path.splitext(gt) if tail == ".txt": tail = "" @@ -252,8 +242,8 @@ def _generate_pickle_name(gt: Path, cache_dir: Optional[Path] = None) -> Text: def load_grammar( - gt: Text = "Grammar.txt", - gp: Optional[Text] = None, + gt: str = "Grammar.txt", + gp: Optional[str] = None, save: bool = True, force: bool = False, logger: Optional[Logger] = None, @@ -276,7 +266,7 @@ def load_grammar( return g -def _newer(a: Text, b: Text) -> bool: +def _newer(a: str, b: str) -> bool: """Inquire whether file a was written since file b.""" if not os.path.exists(a): return False @@ -286,7 +276,7 @@ def _newer(a: Text, b: Text) -> bool: def load_packaged_grammar( - package: str, grammar_source: Text, cache_dir: Optional[Path] = None + package: str, grammar_source: str, cache_dir: Optional[Path] = None ) -> grammar.Grammar: """Normally, loads a pickled grammar by doing pkgutil.get_data(package, pickled_grammar) @@ -309,7 +299,7 @@ def load_packaged_grammar( return g -def main(*args: Text) -> bool: +def main(*args: str) -> bool: """Main program, when run as a script: produce grammar pickle files. Calls load_grammar for each argument, a path to a grammar text file. diff --git a/src/blib2to3/pgen2/grammar.py b/src/blib2to3/pgen2/grammar.py index 337a64f1726..1f3fdc55b97 100644 --- a/src/blib2to3/pgen2/grammar.py +++ b/src/blib2to3/pgen2/grammar.py @@ -16,19 +16,19 @@ import os import pickle import tempfile -from typing import Any, Dict, List, Optional, Text, Tuple, TypeVar, Union +from typing import Any, Dict, List, Optional, Tuple, TypeVar, Union # Local imports from . import token _P = TypeVar("_P", bound="Grammar") -Label = Tuple[int, Optional[Text]] +Label = Tuple[int, Optional[str]] DFA = List[List[Tuple[int, int]]] DFAS = Tuple[DFA, Dict[int, int]] Path = Union[str, "os.PathLike[str]"] -class Grammar(object): +class Grammar: """Pgen parsing tables conversion class. Once initialized, this class supplies the grammar tables for the diff --git a/src/blib2to3/pgen2/literals.py b/src/blib2to3/pgen2/literals.py index b5fe4285114..53c0b8ac2bb 100644 --- a/src/blib2to3/pgen2/literals.py +++ b/src/blib2to3/pgen2/literals.py @@ -4,11 +4,9 @@ """Safely evaluate Python string literals without using eval().""" import re +from typing import Dict, Match -from typing import Dict, Match, Text - - -simple_escapes: Dict[Text, Text] = { +simple_escapes: Dict[str, str] = { "a": "\a", "b": "\b", "f": "\f", @@ -22,7 +20,7 @@ } -def escape(m: Match[Text]) -> Text: +def escape(m: Match[str]) -> str: all, tail = m.group(0, 1) assert all.startswith("\\") esc = simple_escapes.get(tail) @@ -44,7 +42,7 @@ def escape(m: Match[Text]) -> Text: return chr(i) -def evalString(s: Text) -> Text: +def evalString(s: str) -> str: assert s.startswith("'") or s.startswith('"'), repr(s[:1]) q = s[0] if s[:3] == q * 3: diff --git a/src/blib2to3/pgen2/parse.py b/src/blib2to3/pgen2/parse.py index d6deaac6964..ad51a3dad08 100644 --- a/src/blib2to3/pgen2/parse.py +++ b/src/blib2to3/pgen2/parse.py @@ -9,33 +9,32 @@ how this parsing engine works. """ -import copy from contextlib import contextmanager - -# Local imports -from . import grammar, token, tokenize from typing import ( - cast, + TYPE_CHECKING, Any, - Optional, - Text, - Union, - Tuple, + Callable, Dict, - List, Iterator, - Callable, + List, + Optional, Set, - TYPE_CHECKING, + Tuple, + Union, + cast, ) + from blib2to3.pgen2.grammar import Grammar -from blib2to3.pytree import convert, NL, Context, RawNode, Leaf, Node +from blib2to3.pytree import NL, Context, Leaf, Node, RawNode, convert + +# Local imports +from . import grammar, token, tokenize if TYPE_CHECKING: - from blib2to3.driver import TokenProxy + from blib2to3.pgen2.driver import TokenProxy -Results = Dict[Text, NL] +Results = Dict[str, NL] Convert = Callable[[Grammar, RawNode], Union[Node, Leaf]] DFA = List[List[Tuple[int, int]]] DFAS = Tuple[DFA, Dict[int, int]] @@ -100,7 +99,7 @@ def backtrack(self) -> Iterator[None]: finally: self.parser.is_backtracking = is_backtracking - def add_token(self, tok_type: int, tok_val: Text, raw: bool = False) -> None: + def add_token(self, tok_type: int, tok_val: str, raw: bool = False) -> None: func: Callable[..., Any] if raw: func = self.parser._addtoken @@ -114,7 +113,9 @@ def add_token(self, tok_type: int, tok_val: Text, raw: bool = False) -> None: args.insert(0, ilabel) func(*args) - def determine_route(self, value: Optional[Text] = None, force: bool = False) -> Optional[int]: + def determine_route( + self, value: Optional[str] = None, force: bool = False + ) -> Optional[int]: alive_ilabels = self.ilabels if len(alive_ilabels) == 0: *_, most_successful_ilabel = self._dead_ilabels @@ -131,10 +132,10 @@ class ParseError(Exception): """Exception to signal the parser is stuck.""" def __init__( - self, msg: Text, type: Optional[int], value: Optional[Text], context: Context + self, msg: str, type: Optional[int], value: Optional[str], context: Context ) -> None: Exception.__init__( - self, "%s: type=%r, value=%r, context=%r" % (msg, type, value, context) + self, f"{msg}: type={type!r}, value={value!r}, context={context!r}" ) self.msg = msg self.type = type @@ -142,7 +143,7 @@ def __init__( self.context = context -class Parser(object): +class Parser: """Parser engine. The proper usage sequence is: @@ -210,6 +211,7 @@ def __init__(self, grammar: Grammar, convert: Optional[Convert] = None) -> None: # See note in docstring above. TL;DR this is ignored. self.convert = convert or lam_sub self.is_backtracking = False + self.last_token: Optional[int] = None def setup(self, proxy: "TokenProxy", start: Optional[int] = None) -> None: """Prepare for parsing. @@ -235,8 +237,9 @@ def setup(self, proxy: "TokenProxy", start: Optional[int] = None) -> None: self.rootnode: Optional[NL] = None self.used_names: Set[str] = set() self.proxy = proxy + self.last_token = None - def addtoken(self, type: int, value: Text, context: Context) -> bool: + def addtoken(self, type: int, value: str, context: Context) -> bool: """Add a token; return True iff this is the end of the program.""" # Map from token to label ilabels = self.classify(type, value, context) @@ -284,7 +287,7 @@ def addtoken(self, type: int, value: Text, context: Context) -> bool: return self._addtoken(ilabel, type, value, context) - def _addtoken(self, ilabel: int, type: int, value: Text, context: Context) -> bool: + def _addtoken(self, ilabel: int, type: int, value: str, context: Context) -> bool: # Loop until the token is shifted; may raise exceptions while True: dfa, state, node = self.stack[-1] @@ -316,6 +319,7 @@ def _addtoken(self, ilabel: int, type: int, value: Text, context: Context) -> bo dfa, state, node = self.stack[-1] states, first = dfa # Done with this token + self.last_token = type return False else: @@ -329,7 +333,7 @@ def _addtoken(self, ilabel: int, type: int, value: Text, context: Context) -> bo # No success finding a transition raise ParseError("bad input", type, value, context) - def classify(self, type: int, value: Text, context: Context) -> List[int]: + def classify(self, type: int, value: str, context: Context) -> List[int]: """Turn a token into a label. (Internal) Depending on whether the value is a soft-keyword or not, @@ -342,9 +346,23 @@ def classify(self, type: int, value: Text, context: Context) -> List[int]: return [self.grammar.keywords[value]] elif value in self.grammar.soft_keywords: assert type in self.grammar.tokens + # Current soft keywords (match, case, type) can only appear at the + # beginning of a statement. So as a shortcut, don't try to treat them + # like keywords in any other context. + # ('_' is also a soft keyword in the real grammar, but for our grammar + # it's just an expression, so we don't need to treat it specially.) + if self.last_token not in ( + None, + token.INDENT, + token.DEDENT, + token.NEWLINE, + token.SEMI, + token.COLON, + ): + return [self.grammar.tokens[type]] return [ - self.grammar.soft_keywords[value], self.grammar.tokens[type], + self.grammar.soft_keywords[value], ] ilabel = self.grammar.tokens.get(type) @@ -352,7 +370,7 @@ def classify(self, type: int, value: Text, context: Context) -> List[int]: raise ParseError("bad token", type, value, context) return [ilabel] - def shift(self, type: int, value: Text, newstate: int, context: Context) -> None: + def shift(self, type: int, value: str, newstate: int, context: Context) -> None: """Shift a token. (Internal)""" if self.is_backtracking: dfa, state, _ = self.stack[-1] diff --git a/src/blib2to3/pgen2/pgen.py b/src/blib2to3/pgen2/pgen.py index 631682a77c9..3ece9bb41ed 100644 --- a/src/blib2to3/pgen2/pgen.py +++ b/src/blib2to3/pgen2/pgen.py @@ -1,26 +1,22 @@ # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. -# Pgen imports -from . import grammar, token, tokenize - +import os from typing import ( + IO, Any, Dict, - IO, Iterator, List, + NoReturn, Optional, - Text, + Sequence, Tuple, Union, - Sequence, - NoReturn, ) -from blib2to3.pgen2 import grammar -from blib2to3.pgen2.tokenize import GoodTokenInfo -import os +from blib2to3.pgen2 import grammar, token, tokenize +from blib2to3.pgen2.tokenize import GoodTokenInfo Path = Union[str, "os.PathLike[str]"] @@ -29,17 +25,16 @@ class PgenGrammar(grammar.Grammar): pass -class ParserGenerator(object): - +class ParserGenerator: filename: Path - stream: IO[Text] + stream: IO[str] generator: Iterator[GoodTokenInfo] - first: Dict[Text, Optional[Dict[Text, int]]] + first: Dict[str, Optional[Dict[str, int]]] - def __init__(self, filename: Path, stream: Optional[IO[Text]] = None) -> None: + def __init__(self, filename: Path, stream: Optional[IO[str]] = None) -> None: close_stream = None if stream is None: - stream = open(filename) + stream = open(filename, encoding="utf-8") close_stream = stream.close self.filename = filename self.stream = stream @@ -76,7 +71,7 @@ def make_grammar(self) -> PgenGrammar: c.start = c.symbol2number[self.startsymbol] return c - def make_first(self, c: PgenGrammar, name: Text) -> Dict[int, int]: + def make_first(self, c: PgenGrammar, name: str) -> Dict[int, int]: rawfirst = self.first[name] assert rawfirst is not None first = {} @@ -86,7 +81,7 @@ def make_first(self, c: PgenGrammar, name: Text) -> Dict[int, int]: first[ilabel] = 1 return first - def make_label(self, c: PgenGrammar, label: Text) -> int: + def make_label(self, c: PgenGrammar, label: str) -> int: # XXX Maybe this should be a method on a subclass of converter? ilabel = len(c.labels) if label[0].isalpha(): @@ -145,13 +140,13 @@ def addfirstsets(self) -> None: self.calcfirst(name) # print name, self.first[name].keys() - def calcfirst(self, name: Text) -> None: + def calcfirst(self, name: str) -> None: dfa = self.dfas[name] self.first[name] = None # dummy to detect left recursion state = dfa[0] totalset: Dict[str, int] = {} overlapcheck = {} - for label, next in state.arcs.items(): + for label in state.arcs: if label in self.dfas: if label in self.first: fset = self.first[label] @@ -177,7 +172,7 @@ def calcfirst(self, name: Text) -> None: inverse[symbol] = label self.first[name] = totalset - def parse(self) -> Tuple[Dict[Text, List["DFAState"]], Text]: + def parse(self) -> Tuple[Dict[str, List["DFAState"]], str]: dfas = {} startsymbol: Optional[str] = None # MSTART: (NEWLINE | RULE)* ENDMARKER @@ -192,9 +187,9 @@ def parse(self) -> Tuple[Dict[Text, List["DFAState"]], Text]: # self.dump_nfa(name, a, z) dfa = self.make_dfa(a, z) # self.dump_dfa(name, dfa) - oldlen = len(dfa) + # oldlen = len(dfa) self.simplify_dfa(dfa) - newlen = len(dfa) + # newlen = len(dfa) dfas[name] = dfa # print name, oldlen, newlen if startsymbol is None: @@ -241,7 +236,7 @@ def addclosure(state: NFAState, base: Dict[NFAState, int]) -> None: state.addarc(st, label) return states # List of DFAState instances; first one is start - def dump_nfa(self, name: Text, start: "NFAState", finish: "NFAState") -> None: + def dump_nfa(self, name: str, start: "NFAState", finish: "NFAState") -> None: print("Dump of NFA for", name) todo = [start] for i, state in enumerate(todo): @@ -257,7 +252,7 @@ def dump_nfa(self, name: Text, start: "NFAState", finish: "NFAState") -> None: else: print(" %s -> %d" % (label, j)) - def dump_dfa(self, name: Text, dfa: Sequence["DFAState"]) -> None: + def dump_dfa(self, name: str, dfa: Sequence["DFAState"]) -> None: print("Dump of DFA for", name) for i, state in enumerate(dfa): print(" State", i, state.isfinal and "(final)" or "") @@ -348,9 +343,9 @@ def parse_atom(self) -> Tuple["NFAState", "NFAState"]: self.raise_error( "expected (...) or NAME or STRING, got %s/%s", self.type, self.value ) - assert False + raise AssertionError - def expect(self, type: int, value: Optional[Any] = None) -> Text: + def expect(self, type: int, value: Optional[Any] = None) -> str: if self.type != type or (value is not None and self.value != value): self.raise_error( "expected %s/%s, got %s/%s", type, value, self.type, self.value @@ -370,27 +365,27 @@ def raise_error(self, msg: str, *args: Any) -> NoReturn: if args: try: msg = msg % args - except: + except Exception: msg = " ".join([msg] + list(map(str, args))) raise SyntaxError(msg, (self.filename, self.end[0], self.end[1], self.line)) -class NFAState(object): - arcs: List[Tuple[Optional[Text], "NFAState"]] +class NFAState: + arcs: List[Tuple[Optional[str], "NFAState"]] def __init__(self) -> None: self.arcs = [] # list of (label, NFAState) pairs - def addarc(self, next: "NFAState", label: Optional[Text] = None) -> None: + def addarc(self, next: "NFAState", label: Optional[str] = None) -> None: assert label is None or isinstance(label, str) assert isinstance(next, NFAState) self.arcs.append((label, next)) -class DFAState(object): +class DFAState: nfaset: Dict[NFAState, Any] isfinal: bool - arcs: Dict[Text, "DFAState"] + arcs: Dict[str, "DFAState"] def __init__(self, nfaset: Dict[NFAState, Any], final: NFAState) -> None: assert isinstance(nfaset, dict) @@ -400,7 +395,7 @@ def __init__(self, nfaset: Dict[NFAState, Any], final: NFAState) -> None: self.isfinal = final in nfaset self.arcs = {} # map from label to DFAState - def addarc(self, next: "DFAState", label: Text) -> None: + def addarc(self, next: "DFAState", label: str) -> None: assert isinstance(label, str) assert label not in self.arcs assert isinstance(next, DFAState) diff --git a/src/blib2to3/pgen2/token.py b/src/blib2to3/pgen2/token.py index 1e0dec9c714..ed2fc4e85fc 100644 --- a/src/blib2to3/pgen2/token.py +++ b/src/blib2to3/pgen2/token.py @@ -1,12 +1,6 @@ """Token constants (from "token.h").""" -import sys -from typing import Dict - -if sys.version_info < (3, 8): - from typing_extensions import Final -else: - from typing import Final +from typing import Dict, Final # Taken from Python (r53757) and modified to include some tokens # originally monkeypatched in by pgen2.tokenize @@ -78,7 +72,7 @@ tok_name: Final[Dict[int, str]] = {} for _name, _value in list(globals().items()): - if type(_value) is type(0): + if type(_value) is int: tok_name[_value] = _name diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py index 257dbef4a19..b04b18ba870 100644 --- a/src/blib2to3/pgen2/tokenize.py +++ b/src/blib2to3/pgen2/tokenize.py @@ -30,31 +30,40 @@ import sys from typing import ( Callable, + Final, Iterable, Iterator, List, Optional, - Text, - Tuple, Pattern, + Set, + Tuple, Union, - cast, ) -if sys.version_info >= (3, 8): - from typing import Final -else: - from typing_extensions import Final - -from blib2to3.pgen2.token import * from blib2to3.pgen2.grammar import Grammar +from blib2to3.pgen2.token import ( + ASYNC, + AWAIT, + COMMENT, + DEDENT, + ENDMARKER, + ERRORTOKEN, + INDENT, + NAME, + NEWLINE, + NL, + NUMBER, + OP, + STRING, + tok_name, +) __author__ = "Ka-Ping Yee " __credits__ = "GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro" import re from codecs import BOM_UTF8, lookup -from blib2to3.pgen2.token import * from . import token @@ -66,20 +75,20 @@ del token -def group(*choices): +def group(*choices: str) -> str: return "(" + "|".join(choices) + ")" -def any(*choices): +def any(*choices: str) -> str: return group(*choices) + "*" -def maybe(*choices): +def maybe(*choices: str) -> str: return group(*choices) + "?" -def _combinations(*l): - return set(x + y for x in l for y in l + ("",) if x.casefold() != y.casefold()) +def _combinations(*l: str) -> Set[str]: + return {x + y for x in l for y in l + ("",) if x.casefold() != y.casefold()} Whitespace = r"[ \f\t]*" @@ -163,7 +172,6 @@ def _combinations(*l): '"""': double3prog, **{f"{prefix}'''": single3prog for prefix in _strprefixes}, **{f'{prefix}"""': double3prog for prefix in _strprefixes}, - **{prefix: None for prefix in _strprefixes}, } triple_quoted: Final = ( @@ -188,19 +196,23 @@ class StopTokenizing(Exception): pass -def printtoken(type, token, xxx_todo_changeme, xxx_todo_changeme1, line): # for testing - (srow, scol) = xxx_todo_changeme - (erow, ecol) = xxx_todo_changeme1 +Coord = Tuple[int, int] + + +def printtoken( + type: int, token: str, srow_col: Coord, erow_col: Coord, line: str +) -> None: # for testing + (srow, scol) = srow_col + (erow, ecol) = erow_col print( "%d,%d-%d,%d:\t%s\t%s" % (srow, scol, erow, ecol, tok_name[type], repr(token)) ) -Coord = Tuple[int, int] -TokenEater = Callable[[int, Text, Coord, Coord, Text], None] +TokenEater = Callable[[int, str, Coord, Coord, str], None] -def tokenize(readline: Callable[[], Text], tokeneater: TokenEater = printtoken) -> None: +def tokenize(readline: Callable[[], str], tokeneater: TokenEater = printtoken) -> None: """ The tokenize() function accepts two parameters: one representing the input stream, and one providing an output mechanism for tokenize(). @@ -220,18 +232,17 @@ def tokenize(readline: Callable[[], Text], tokeneater: TokenEater = printtoken) # backwards compatible interface -def tokenize_loop(readline, tokeneater): +def tokenize_loop(readline: Callable[[], str], tokeneater: TokenEater) -> None: for token_info in generate_tokens(readline): tokeneater(*token_info) -GoodTokenInfo = Tuple[int, Text, Coord, Coord, Text] +GoodTokenInfo = Tuple[int, str, Coord, Coord, str] TokenInfo = Union[Tuple[int, str], GoodTokenInfo] class Untokenizer: - - tokens: List[Text] + tokens: List[str] prev_row: int prev_col: int @@ -247,14 +258,12 @@ def add_whitespace(self, start: Coord) -> None: if col_offset: self.tokens.append(" " * col_offset) - def untokenize(self, iterable: Iterable[TokenInfo]) -> Text: + def untokenize(self, iterable: Iterable[TokenInfo]) -> str: for t in iterable: if len(t) == 2: - self.compat(cast(Tuple[int, str], t), iterable) + self.compat(t, iterable) break - tok_type, token, start, end, line = cast( - Tuple[int, Text, Coord, Coord, Text], t - ) + tok_type, token, start, end, line = t self.add_whitespace(start) self.tokens.append(token) self.prev_row, self.prev_col = end @@ -263,7 +272,7 @@ def untokenize(self, iterable: Iterable[TokenInfo]) -> Text: self.prev_col = 0 return "".join(self.tokens) - def compat(self, token: Tuple[int, Text], iterable: Iterable[TokenInfo]) -> None: + def compat(self, token: Tuple[int, str], iterable: Iterable[TokenInfo]) -> None: startline = False indents = [] toks_append = self.tokens.append @@ -335,7 +344,7 @@ def read_or_stop() -> bytes: try: return readline() except StopIteration: - return bytes() + return b"" def find_cookie(line: bytes) -> Optional[str]: try: @@ -384,7 +393,7 @@ def find_cookie(line: bytes) -> Optional[str]: return default, [first, second] -def untokenize(iterable: Iterable[TokenInfo]) -> Text: +def untokenize(iterable: Iterable[TokenInfo]) -> str: """Transform tokens back into Python source code. Each element returned by the iterable must be a token sequence @@ -407,7 +416,7 @@ def untokenize(iterable: Iterable[TokenInfo]) -> Text: def generate_tokens( - readline: Callable[[], Text], grammar: Optional[Grammar] = None + readline: Callable[[], str], grammar: Optional[Grammar] = None ) -> Iterator[GoodTokenInfo]: """ The generate_tokens() generator requires one argument, readline, which @@ -425,7 +434,7 @@ def generate_tokens( logical line; continuation lines are included. """ lnum = parenlev = continued = 0 - numchars: Final = "0123456789" + numchars: Final[str] = "0123456789" contstr, needcont = "", 0 contline: Optional[str] = None indents = [0] @@ -599,11 +608,15 @@ def generate_tokens( ): if token[-1] == "\n": # continued string strstart = (lnum, start) - endprog = ( - endprogs[initial] - or endprogs[token[1]] - or endprogs[token[2]] + maybe_endprog = ( + endprogs.get(initial) + or endprogs.get(token[1]) + or endprogs.get(token[2]) ) + assert ( + maybe_endprog is not None + ), f"endprog not found for {token}" + endprog = maybe_endprog contstr, needcont = line[start:], 1 contline = line break @@ -631,7 +644,6 @@ def generate_tokens( if token in ("def", "for"): if stashed and stashed[0] == NAME and stashed[1] == "async": - if token == "def": async_def = True async_def_indent = indents[-1] @@ -674,14 +686,12 @@ def generate_tokens( yield stashed stashed = None - for indent in indents[1:]: # pop remaining indent levels + for _indent in indents[1:]: # pop remaining indent levels yield (DEDENT, "", (lnum, 0), (lnum, 0), "") yield (ENDMARKER, "", (lnum, 0), (lnum, 0), "") if __name__ == "__main__": # testing - import sys - if len(sys.argv) > 1: tokenize(open(sys.argv[1]).readline) else: diff --git a/src/blib2to3/pygram.py b/src/blib2to3/pygram.py index 99012cdd9cb..2b43b4c112b 100644 --- a/src/blib2to3/pygram.py +++ b/src/blib2to3/pygram.py @@ -5,13 +5,10 @@ # Python imports import os - from typing import Union # Local imports -from .pgen2 import token from .pgen2 import driver - from .pgen2.grammar import Grammar # Moved into initialize because mypyc can't handle __file__ (XXX bug) @@ -21,7 +18,7 @@ # "PatternGrammar.txt") -class Symbols(object): +class Symbols: def __init__(self, grammar: Grammar) -> None: """Initializer. @@ -66,7 +63,6 @@ class _python_symbols(Symbols): encoding_decl: int eval_input: int except_clause: int - exec_stmt: int expr: int expr_stmt: int exprlist: int @@ -95,11 +91,11 @@ class _python_symbols(Symbols): old_test: int or_test: int parameters: int + paramspec: int pass_stmt: int pattern: int patterns: int power: int - print_stmt: int raise_stmt: int return_stmt: int shift_expr: int @@ -126,7 +122,12 @@ class _python_symbols(Symbols): tname_star: int trailer: int try_stmt: int + type_stmt: int typedargslist: int + typeparam: int + typeparams: int + typevar: int + typevartuple: int varargslist: int vfpdef: int vfplist: int @@ -150,22 +151,16 @@ class _pattern_symbols(Symbols): python_grammar: Grammar -python_grammar_no_print_statement: Grammar -python_grammar_no_print_statement_no_exec_statement: Grammar -python_grammar_no_print_statement_no_exec_statement_async_keywords: Grammar -python_grammar_no_exec_statement: Grammar -pattern_grammar: Grammar +python_grammar_async_keywords: Grammar python_grammar_soft_keywords: Grammar - +pattern_grammar: Grammar python_symbols: _python_symbols pattern_symbols: _pattern_symbols def initialize(cache_dir: Union[str, "os.PathLike[str]", None] = None) -> None: global python_grammar - global python_grammar_no_print_statement - global python_grammar_no_print_statement_no_exec_statement - global python_grammar_no_print_statement_no_exec_statement_async_keywords + global python_grammar_async_keywords global python_grammar_soft_keywords global python_symbols global pattern_grammar @@ -177,38 +172,25 @@ def initialize(cache_dir: Union[str, "os.PathLike[str]", None] = None) -> None: os.path.dirname(__file__), "PatternGrammar.txt" ) - # Python 2 python_grammar = driver.load_packaged_grammar("blib2to3", _GRAMMAR_FILE, cache_dir) - python_grammar.version = (2, 0) + assert "print" not in python_grammar.keywords + assert "exec" not in python_grammar.keywords soft_keywords = python_grammar.soft_keywords.copy() python_grammar.soft_keywords.clear() python_symbols = _python_symbols(python_grammar) - # Python 2 + from __future__ import print_function - python_grammar_no_print_statement = python_grammar.copy() - del python_grammar_no_print_statement.keywords["print"] - # Python 3.0-3.6 - python_grammar_no_print_statement_no_exec_statement = python_grammar.copy() - del python_grammar_no_print_statement_no_exec_statement.keywords["print"] - del python_grammar_no_print_statement_no_exec_statement.keywords["exec"] - python_grammar_no_print_statement_no_exec_statement.version = (3, 0) + python_grammar.version = (3, 0) # Python 3.7+ - python_grammar_no_print_statement_no_exec_statement_async_keywords = ( - python_grammar_no_print_statement_no_exec_statement.copy() - ) - python_grammar_no_print_statement_no_exec_statement_async_keywords.async_keywords = ( - True - ) - python_grammar_no_print_statement_no_exec_statement_async_keywords.version = (3, 7) + python_grammar_async_keywords = python_grammar.copy() + python_grammar_async_keywords.async_keywords = True + python_grammar_async_keywords.version = (3, 7) # Python 3.10+ - python_grammar_soft_keywords = ( - python_grammar_no_print_statement_no_exec_statement_async_keywords.copy() - ) + python_grammar_soft_keywords = python_grammar_async_keywords.copy() python_grammar_soft_keywords.soft_keywords = soft_keywords python_grammar_soft_keywords.version = (3, 10) diff --git a/src/blib2to3/pytree.py b/src/blib2to3/pytree.py index 15a1420ef7d..2a0cd6d196a 100644 --- a/src/blib2to3/pytree.py +++ b/src/blib2to3/pytree.py @@ -15,16 +15,16 @@ from typing import ( Any, Dict, + Iterable, Iterator, List, Optional, - Text, + Set, Tuple, TypeVar, Union, - Set, - Iterable, ) + from blib2to3.pgen2.grammar import Grammar __author__ = "Guido van Rossum " @@ -34,10 +34,10 @@ HUGE: int = 0x7FFFFFFF # maximum repeat count, default max -_type_reprs: Dict[int, Union[Text, int]] = {} +_type_reprs: Dict[int, Union[str, int]] = {} -def type_repr(type_num: int) -> Union[Text, int]: +def type_repr(type_num: int) -> Union[str, int]: global _type_reprs if not _type_reprs: from .pygram import python_symbols @@ -54,12 +54,11 @@ def type_repr(type_num: int) -> Union[Text, int]: _P = TypeVar("_P", bound="Base") NL = Union["Node", "Leaf"] -Context = Tuple[Text, Tuple[int, int]] -RawNode = Tuple[int, Optional[Text], Optional[Context], Optional[List[NL]]] - +Context = Tuple[str, Tuple[int, int]] +RawNode = Tuple[int, Optional[str], Optional[Context], Optional[List[NL]]] -class Base(object): +class Base: """ Abstract base class for Node and Leaf. @@ -92,7 +91,7 @@ def __eq__(self, other: Any) -> bool: return self._eq(other) @property - def prefix(self) -> Text: + def prefix(self) -> str: raise NotImplementedError def _eq(self: _P, other: _P) -> bool: @@ -225,7 +224,7 @@ def depth(self) -> int: return 0 return 1 + self.parent.depth() - def get_suffix(self) -> Text: + def get_suffix(self) -> str: """ Return the string immediately following the invocant node. This is effectively equivalent to node.next_sibling.prefix @@ -238,18 +237,17 @@ def get_suffix(self) -> Text: class Node(Base): - """Concrete implementation for interior nodes.""" fixers_applied: Optional[List[Any]] - used_names: Optional[Set[Text]] + used_names: Optional[Set[str]] def __init__( self, type: int, children: List[NL], context: Optional[Any] = None, - prefix: Optional[Text] = None, + prefix: Optional[str] = None, fixers_applied: Optional[List[Any]] = None, ) -> None: """ @@ -274,16 +272,16 @@ def __init__( else: self.fixers_applied = None - def __repr__(self) -> Text: + def __repr__(self) -> str: """Return a canonical string representation.""" assert self.type is not None - return "%s(%s, %r)" % ( + return "{}({}, {!r})".format( self.__class__.__name__, type_repr(self.type), self.children, ) - def __str__(self) -> Text: + def __str__(self) -> str: """ Return a pretty string representation. @@ -317,7 +315,7 @@ def pre_order(self) -> Iterator[NL]: yield from child.pre_order() @property - def prefix(self) -> Text: + def prefix(self) -> str: """ The whitespace and comments preceding this node in the input. """ @@ -326,7 +324,7 @@ def prefix(self) -> Text: return self.children[0].prefix @prefix.setter - def prefix(self, prefix: Text) -> None: + def prefix(self, prefix: str) -> None: if self.children: self.children[0].prefix = prefix @@ -379,28 +377,32 @@ def update_sibling_maps(self) -> None: class Leaf(Base): - """Concrete implementation for leaf nodes.""" # Default values for instance variables - value: Text + value: str fixers_applied: List[Any] bracket_depth: int # Changed later in brackets.py opening_bracket: Optional["Leaf"] = None - used_names: Optional[Set[Text]] + used_names: Optional[Set[str]] _prefix = "" # Whitespace and comments preceding this token in the input lineno: int = 0 # Line where this token starts in the input column: int = 0 # Column where this token starts in the input + # If not None, this Leaf is created by converting a block of fmt off/skip + # code, and `fmt_pass_converted_first_leaf` points to the first Leaf in the + # converted code. + fmt_pass_converted_first_leaf: Optional["Leaf"] = None def __init__( self, type: int, - value: Text, + value: str, context: Optional[Context] = None, - prefix: Optional[Text] = None, + prefix: Optional[str] = None, fixers_applied: List[Any] = [], opening_bracket: Optional["Leaf"] = None, + fmt_pass_converted_first_leaf: Optional["Leaf"] = None, ) -> None: """ Initializer. @@ -419,19 +421,20 @@ def __init__( self.fixers_applied: Optional[List[Any]] = fixers_applied[:] self.children = [] self.opening_bracket = opening_bracket + self.fmt_pass_converted_first_leaf = fmt_pass_converted_first_leaf def __repr__(self) -> str: """Return a canonical string representation.""" from .pgen2.token import tok_name assert self.type is not None - return "%s(%s, %r)" % ( + return "{}({}, {!r})".format( self.__class__.__name__, tok_name.get(self.type, self.type), self.value, ) - def __str__(self) -> Text: + def __str__(self) -> str: """ Return a pretty string representation. @@ -465,14 +468,14 @@ def pre_order(self) -> Iterator["Leaf"]: yield self @property - def prefix(self) -> Text: + def prefix(self) -> str: """ The whitespace and comments preceding this token in the input. """ return self._prefix @prefix.setter - def prefix(self, prefix: Text) -> None: + def prefix(self, prefix: str) -> None: self.changed() self._prefix = prefix @@ -497,11 +500,10 @@ def convert(gr: Grammar, raw_node: RawNode) -> NL: return Leaf(type, value or "", context=context) -_Results = Dict[Text, NL] - +_Results = Dict[str, NL] -class BasePattern(object): +class BasePattern: """ A pattern is a tree matching pattern. @@ -520,19 +522,19 @@ class BasePattern(object): type: Optional[int] type = None # Node type (token if < 256, symbol if >= 256) content: Any = None # Optional content matching pattern - name: Optional[Text] = None # Optional name used to store match in results dict + name: Optional[str] = None # Optional name used to store match in results dict def __new__(cls, *args, **kwds): """Constructor that prevents BasePattern from being instantiated.""" assert cls is not BasePattern, "Cannot instantiate BasePattern" return object.__new__(cls) - def __repr__(self) -> Text: + def __repr__(self) -> str: assert self.type is not None args = [type_repr(self.type), self.content, self.name] while args and args[-1] is None: del args[-1] - return "%s(%s)" % (self.__class__.__name__, ", ".join(map(repr, args))) + return "{}({})".format(self.__class__.__name__, ", ".join(map(repr, args))) def _submatch(self, node, results=None) -> bool: raise NotImplementedError @@ -596,8 +598,8 @@ class LeafPattern(BasePattern): def __init__( self, type: Optional[int] = None, - content: Optional[Text] = None, - name: Optional[Text] = None, + content: Optional[str] = None, + name: Optional[str] = None, ) -> None: """ Initializer. Takes optional type, content, and name. @@ -641,14 +643,13 @@ def _submatch(self, node, results=None): class NodePattern(BasePattern): - wildcards: bool = False def __init__( self, type: Optional[int] = None, - content: Optional[Iterable[Text]] = None, - name: Optional[Text] = None, + content: Optional[Iterable[str]] = None, + name: Optional[str] = None, ) -> None: """ Initializer. Takes optional type, content, and name. @@ -710,7 +711,6 @@ def _submatch(self, node, results=None) -> bool: class WildcardPattern(BasePattern): - """ A wildcard pattern can match zero or more nodes. @@ -728,10 +728,10 @@ class WildcardPattern(BasePattern): def __init__( self, - content: Optional[Text] = None, + content: Optional[str] = None, min: int = 0, max: int = HUGE, - name: Optional[Text] = None, + name: Optional[str] = None, ) -> None: """ Initializer. diff --git a/test_requirements.txt b/test_requirements.txt index ef61a1210ee..a3d262bc53d 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,6 +1,6 @@ coverage >= 5.3 pre-commit pytest >= 6.1.1 -pytest-xdist >= 2.2.1, < 3.0.2 -pytest-cov >= 2.11.1 +pytest-xdist >= 3.0.2 +pytest-cov >= 4.1.0 tox diff --git a/tests/conftest.py b/tests/conftest.py index 67517268d1b..1a0dd747d8e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1 +1,28 @@ +import pytest + pytest_plugins = ["tests.optional"] + +PRINT_FULL_TREE: bool = False +PRINT_TREE_DIFF: bool = True + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--print-full-tree", + action="store_true", + default=False, + help="print full syntax trees on failed tests", + ) + parser.addoption( + "--print-tree-diff", + action="store_true", + default=True, + help="print diff of syntax trees on failed tests", + ) + + +def pytest_configure(config: pytest.Config) -> None: + global PRINT_FULL_TREE + global PRINT_TREE_DIFF + PRINT_FULL_TREE = config.getoption("--print-full-tree") + PRINT_TREE_DIFF = config.getoption("--print-tree-diff") diff --git a/tests/data/simple_cases/attribute_access_on_number_literals.py b/tests/data/cases/attribute_access_on_number_literals.py similarity index 100% rename from tests/data/simple_cases/attribute_access_on_number_literals.py rename to tests/data/cases/attribute_access_on_number_literals.py diff --git a/tests/data/simple_cases/beginning_backslash.py b/tests/data/cases/beginning_backslash.py similarity index 100% rename from tests/data/simple_cases/beginning_backslash.py rename to tests/data/cases/beginning_backslash.py diff --git a/tests/data/simple_cases/bracketmatch.py b/tests/data/cases/bracketmatch.py similarity index 100% rename from tests/data/simple_cases/bracketmatch.py rename to tests/data/cases/bracketmatch.py diff --git a/tests/data/cases/bytes_docstring.py b/tests/data/cases/bytes_docstring.py new file mode 100644 index 00000000000..2326e95293a --- /dev/null +++ b/tests/data/cases/bytes_docstring.py @@ -0,0 +1,34 @@ +def bitey(): + b" not a docstring" + +def bitey2(): + b' also not a docstring' + +def triple_quoted_bytes(): + b""" not a docstring""" + +def triple_quoted_bytes2(): + b''' also not a docstring''' + +def capitalized_bytes(): + B" NOT A DOCSTRING" + +# output +def bitey(): + b" not a docstring" + + +def bitey2(): + b" also not a docstring" + + +def triple_quoted_bytes(): + b""" not a docstring""" + + +def triple_quoted_bytes2(): + b""" also not a docstring""" + + +def capitalized_bytes(): + b" NOT A DOCSTRING" \ No newline at end of file diff --git a/tests/data/simple_cases/class_blank_parentheses.py b/tests/data/cases/class_blank_parentheses.py similarity index 100% rename from tests/data/simple_cases/class_blank_parentheses.py rename to tests/data/cases/class_blank_parentheses.py diff --git a/tests/data/simple_cases/class_methods_new_line.py b/tests/data/cases/class_methods_new_line.py similarity index 100% rename from tests/data/simple_cases/class_methods_new_line.py rename to tests/data/cases/class_methods_new_line.py diff --git a/tests/data/simple_cases/collections.py b/tests/data/cases/collections.py similarity index 100% rename from tests/data/simple_cases/collections.py rename to tests/data/cases/collections.py diff --git a/tests/data/simple_cases/comment_after_escaped_newline.py b/tests/data/cases/comment_after_escaped_newline.py similarity index 100% rename from tests/data/simple_cases/comment_after_escaped_newline.py rename to tests/data/cases/comment_after_escaped_newline.py diff --git a/tests/data/cases/comment_type_hint.py b/tests/data/cases/comment_type_hint.py new file mode 100644 index 00000000000..2992da88d90 --- /dev/null +++ b/tests/data/cases/comment_type_hint.py @@ -0,0 +1,3 @@ +# flags: --no-preview-line-length-1 +# split out from comments2 as it does not work with line-length=1, losing the comment +a = "type comment with trailing space" # type: str diff --git a/tests/data/simple_cases/comments.py b/tests/data/cases/comments.py similarity index 100% rename from tests/data/simple_cases/comments.py rename to tests/data/cases/comments.py diff --git a/tests/data/simple_cases/comments2.py b/tests/data/cases/comments2.py similarity index 99% rename from tests/data/simple_cases/comments2.py rename to tests/data/cases/comments2.py index 37e185abf4f..261c5e9f0a0 100644 --- a/tests/data/simple_cases/comments2.py +++ b/tests/data/cases/comments2.py @@ -154,6 +154,7 @@ def _init_host(self, parsed) -> None: not parsed.hostname.strip()): pass + ####################### ### SECTION COMMENT ### ####################### diff --git a/tests/data/simple_cases/comments3.py b/tests/data/cases/comments3.py similarity index 100% rename from tests/data/simple_cases/comments3.py rename to tests/data/cases/comments3.py diff --git a/tests/data/simple_cases/comments4.py b/tests/data/cases/comments4.py similarity index 98% rename from tests/data/simple_cases/comments4.py rename to tests/data/cases/comments4.py index 2147d41c9da..9f4f39d8359 100644 --- a/tests/data/simple_cases/comments4.py +++ b/tests/data/cases/comments4.py @@ -85,7 +85,7 @@ def foo2(list_a, list_b): def foo3(list_a, list_b): return ( - # Standlone comment but weirdly placed. + # Standalone comment but weirdly placed. User.query.filter(User.foo == "bar") .filter( db.or_(User.field_a.astext.in_(list_a), User.field_b.astext.in_(list_b)) diff --git a/tests/data/simple_cases/comments5.py b/tests/data/cases/comments5.py similarity index 100% rename from tests/data/simple_cases/comments5.py rename to tests/data/cases/comments5.py diff --git a/tests/data/simple_cases/comments6.py b/tests/data/cases/comments6.py similarity index 100% rename from tests/data/simple_cases/comments6.py rename to tests/data/cases/comments6.py diff --git a/tests/data/simple_cases/comments8.py b/tests/data/cases/comments8.py similarity index 100% rename from tests/data/simple_cases/comments8.py rename to tests/data/cases/comments8.py diff --git a/tests/data/simple_cases/comments9.py b/tests/data/cases/comments9.py similarity index 100% rename from tests/data/simple_cases/comments9.py rename to tests/data/cases/comments9.py diff --git a/tests/data/cases/comments_in_blocks.py b/tests/data/cases/comments_in_blocks.py new file mode 100644 index 00000000000..1221139b6d8 --- /dev/null +++ b/tests/data/cases/comments_in_blocks.py @@ -0,0 +1,111 @@ +# Test cases from: +# - https://github.com/psf/black/issues/1798 +# - https://github.com/psf/black/issues/1499 +# - https://github.com/psf/black/issues/1211 +# - https://github.com/psf/black/issues/563 + +( + lambda + # a comment + : None +) + +( + lambda: + # b comment + None +) + +( + lambda + # a comment + : + # b comment + None +) + +[ + x + # Let's do this + for + # OK? + x + # Some comment + # And another + in + # One more + y +] + +return [ + (offers[offer_index], 1.0) + for offer_index, _ + # avoid returning any offers that don't match the grammar so + # that the return values here are consistent with what would be + # returned in AcceptValidHeader + in self._parse_and_normalize_offers(offers) +] + +from foo import ( + bar, + # qux +) + + +def convert(collection): + # replace all variables by integers + replacement_dict = { + variable: f"{index}" + for index, variable + # 0 is reserved as line terminator + in enumerate(collection.variables(), start=1) + } + + +{ + i: i + for i + # a comment + in range(5) +} + + +def get_subtree_proof_nodes( + chunk_index_groups: Sequence[Tuple[int, ...], ...], +) -> Tuple[int, ...]: + subtree_node_paths = ( + # We take a candidate element from each group and shift it to + # remove the bits that are not common to other group members, then + # we convert it to a tree path that all elements from this group + # have in common. + chunk_index + for chunk_index, bits_to_truncate + # Each group will contain an even "power-of-two" number of# elements. + # This tells us how many tailing bits each element has# which need to + # be truncated to get the group's common prefix. + in ((group[0], (len(group) - 1).bit_length()) for group in chunk_index_groups) + ) + return subtree_node_paths + + +if ( + # comment1 + a + # comment2 + or ( + # comment3 + ( + # comment4 + b + ) + # comment5 + and + # comment6 + c + or ( + # comment7 + d + ) + ) +): + print("Foo") diff --git a/tests/data/simple_cases/comments_non_breaking_space.py b/tests/data/cases/comments_non_breaking_space.py similarity index 100% rename from tests/data/simple_cases/comments_non_breaking_space.py rename to tests/data/cases/comments_non_breaking_space.py diff --git a/tests/data/simple_cases/composition.py b/tests/data/cases/composition.py similarity index 100% rename from tests/data/simple_cases/composition.py rename to tests/data/cases/composition.py diff --git a/tests/data/simple_cases/composition_no_trailing_comma.py b/tests/data/cases/composition_no_trailing_comma.py similarity index 100% rename from tests/data/simple_cases/composition_no_trailing_comma.py rename to tests/data/cases/composition_no_trailing_comma.py diff --git a/tests/data/conditional_expression.py b/tests/data/cases/conditional_expression.py similarity index 99% rename from tests/data/conditional_expression.py rename to tests/data/cases/conditional_expression.py index 620a12dc986..c30cd76c791 100644 --- a/tests/data/conditional_expression.py +++ b/tests/data/cases/conditional_expression.py @@ -1,3 +1,4 @@ +# flags: --preview long_kwargs_single_line = my_function( foo="test, this is a sample value", bar=some_long_value_name_foo_bar_baz if some_boolean_variable else some_fallback_value_foo_bar_baz, diff --git a/tests/data/simple_cases/docstring.py b/tests/data/cases/docstring.py similarity index 97% rename from tests/data/simple_cases/docstring.py rename to tests/data/cases/docstring.py index c31d6a68783..e983c5bd438 100644 --- a/tests/data/simple_cases/docstring.py +++ b/tests/data/cases/docstring.py @@ -221,6 +221,12 @@ def stable_quote_normalization_with_immediate_inner_single_quote(self): ''' +def foo(): + """ + Docstring with a backslash followed by a space\ + and then another line + """ + # output class MyClass: @@ -442,3 +448,10 @@ def stable_quote_normalization_with_immediate_inner_single_quote(self): """ + + +def foo(): + """ + Docstring with a backslash followed by a space\ + and then another line + """ diff --git a/tests/data/simple_cases/docstring_no_extra_empty_line_before_eof.py b/tests/data/cases/docstring_no_extra_empty_line_before_eof.py similarity index 100% rename from tests/data/simple_cases/docstring_no_extra_empty_line_before_eof.py rename to tests/data/cases/docstring_no_extra_empty_line_before_eof.py diff --git a/tests/data/miscellaneous/docstring_no_string_normalization.py b/tests/data/cases/docstring_no_string_normalization.py similarity index 98% rename from tests/data/miscellaneous/docstring_no_string_normalization.py rename to tests/data/cases/docstring_no_string_normalization.py index a90b578f09a..4ec6b8a0153 100644 --- a/tests/data/miscellaneous/docstring_no_string_normalization.py +++ b/tests/data/cases/docstring_no_string_normalization.py @@ -1,3 +1,4 @@ +# flags: --skip-string-normalization class ALonelyClass: ''' A multiline class docstring. diff --git a/tests/data/simple_cases/docstring_preview.py b/tests/data/cases/docstring_preview.py similarity index 99% rename from tests/data/simple_cases/docstring_preview.py rename to tests/data/cases/docstring_preview.py index ff4819acb67..a3c656be2f8 100644 --- a/tests/data/simple_cases/docstring_preview.py +++ b/tests/data/cases/docstring_preview.py @@ -58,7 +58,8 @@ def docstring_almost_at_line_limit(): def docstring_almost_at_line_limit_with_prefix(): - f"""long docstring................................................................""" + f"""long docstring................................................................ + """ def mulitline_docstring_almost_at_line_limit(): diff --git a/tests/data/simple_cases/empty_lines.py b/tests/data/cases/empty_lines.py similarity index 100% rename from tests/data/simple_cases/empty_lines.py rename to tests/data/cases/empty_lines.py diff --git a/tests/data/simple_cases/expression.diff b/tests/data/cases/expression.diff similarity index 100% rename from tests/data/simple_cases/expression.diff rename to tests/data/cases/expression.diff diff --git a/tests/data/simple_cases/expression.py b/tests/data/cases/expression.py similarity index 100% rename from tests/data/simple_cases/expression.py rename to tests/data/cases/expression.py diff --git a/tests/data/cases/f_docstring.py b/tests/data/cases/f_docstring.py new file mode 100644 index 00000000000..667f550b353 --- /dev/null +++ b/tests/data/cases/f_docstring.py @@ -0,0 +1,20 @@ +def foo(e): + f""" {'.'.join(e)}""" + +def bar(e): + f"{'.'.join(e)}" + +def baz(e): + F""" {'.'.join(e)}""" + +# output +def foo(e): + f""" {'.'.join(e)}""" + + +def bar(e): + f"{'.'.join(e)}" + + +def baz(e): + f""" {'.'.join(e)}""" diff --git a/tests/data/simple_cases/fmtonoff.py b/tests/data/cases/fmtonoff.py similarity index 99% rename from tests/data/simple_cases/fmtonoff.py rename to tests/data/cases/fmtonoff.py index e40ea2c8d21..d1f15cd5c8b 100644 --- a/tests/data/simple_cases/fmtonoff.py +++ b/tests/data/cases/fmtonoff.py @@ -195,7 +195,6 @@ def single_literal_yapf_disable(): from third_party import X, Y, Z from library import some_connection, some_decorator - # fmt: off from third_party import (X, Y, Z) diff --git a/tests/data/simple_cases/fmtonoff2.py b/tests/data/cases/fmtonoff2.py similarity index 100% rename from tests/data/simple_cases/fmtonoff2.py rename to tests/data/cases/fmtonoff2.py diff --git a/tests/data/simple_cases/fmtonoff3.py b/tests/data/cases/fmtonoff3.py similarity index 100% rename from tests/data/simple_cases/fmtonoff3.py rename to tests/data/cases/fmtonoff3.py diff --git a/tests/data/simple_cases/fmtonoff4.py b/tests/data/cases/fmtonoff4.py similarity index 100% rename from tests/data/simple_cases/fmtonoff4.py rename to tests/data/cases/fmtonoff4.py diff --git a/tests/data/simple_cases/fmtonoff5.py b/tests/data/cases/fmtonoff5.py similarity index 100% rename from tests/data/simple_cases/fmtonoff5.py rename to tests/data/cases/fmtonoff5.py diff --git a/tests/data/cases/fmtpass_imports.py b/tests/data/cases/fmtpass_imports.py new file mode 100644 index 00000000000..8b3c0bc662a --- /dev/null +++ b/tests/data/cases/fmtpass_imports.py @@ -0,0 +1,19 @@ +# Regression test for https://github.com/psf/black/issues/3438 + +import ast +import collections # fmt: skip +import dataclasses +# fmt: off +import os +# fmt: on +import pathlib + +import re # fmt: skip +import secrets + +# fmt: off +import sys +# fmt: on + +import tempfile +import zoneinfo diff --git a/tests/data/simple_cases/fmtskip.py b/tests/data/cases/fmtskip.py similarity index 100% rename from tests/data/simple_cases/fmtskip.py rename to tests/data/cases/fmtskip.py diff --git a/tests/data/simple_cases/fmtskip2.py b/tests/data/cases/fmtskip2.py similarity index 78% rename from tests/data/simple_cases/fmtskip2.py rename to tests/data/cases/fmtskip2.py index e6248117aa9..0189d4e642d 100644 --- a/tests/data/simple_cases/fmtskip2.py +++ b/tests/data/cases/fmtskip2.py @@ -1,9 +1,12 @@ +# flags: --no-preview-line-length-1 +# l2 loses the comment with line-length=1 in preview mode l1 = ["This list should be broken up", "into multiple lines", "because it is way too long"] l2 = ["But this list shouldn't", "even though it also has", "way too many characters in it"] # fmt: skip l3 = ["I have", "trailing comma", "so I should be braked",] # output +# l2 loses the comment with line-length=1 in preview mode l1 = [ "This list should be broken up", "into multiple lines", @@ -14,4 +17,4 @@ "I have", "trailing comma", "so I should be braked", -] \ No newline at end of file +] diff --git a/tests/data/simple_cases/fmtskip3.py b/tests/data/cases/fmtskip3.py similarity index 100% rename from tests/data/simple_cases/fmtskip3.py rename to tests/data/cases/fmtskip3.py diff --git a/tests/data/simple_cases/fmtskip4.py b/tests/data/cases/fmtskip4.py similarity index 100% rename from tests/data/simple_cases/fmtskip4.py rename to tests/data/cases/fmtskip4.py diff --git a/tests/data/simple_cases/fmtskip5.py b/tests/data/cases/fmtskip5.py similarity index 100% rename from tests/data/simple_cases/fmtskip5.py rename to tests/data/cases/fmtskip5.py diff --git a/tests/data/simple_cases/fmtskip6.py b/tests/data/cases/fmtskip6.py similarity index 100% rename from tests/data/simple_cases/fmtskip6.py rename to tests/data/cases/fmtskip6.py diff --git a/tests/data/simple_cases/fmtskip7.py b/tests/data/cases/fmtskip7.py similarity index 100% rename from tests/data/simple_cases/fmtskip7.py rename to tests/data/cases/fmtskip7.py diff --git a/tests/data/simple_cases/fmtskip8.py b/tests/data/cases/fmtskip8.py similarity index 100% rename from tests/data/simple_cases/fmtskip8.py rename to tests/data/cases/fmtskip8.py diff --git a/tests/data/simple_cases/fstring.py b/tests/data/cases/fstring.py similarity index 82% rename from tests/data/simple_cases/fstring.py rename to tests/data/cases/fstring.py index 4b33231c01c..60560309376 100644 --- a/tests/data/simple_cases/fstring.py +++ b/tests/data/cases/fstring.py @@ -7,6 +7,8 @@ f"\"{f'{nested} inner'}\" outer" f"space between opening braces: { {a for a in (1, 2, 3)}}" f'Hello \'{tricky + "example"}\'' +f"Tried directories {str(rootdirs)} \ +but none started with prefix {parentdir_prefix}" # output @@ -19,3 +21,5 @@ f"\"{f'{nested} inner'}\" outer" f"space between opening braces: { {a for a in (1, 2, 3)}}" f'Hello \'{tricky + "example"}\'' +f"Tried directories {str(rootdirs)} \ +but none started with prefix {parentdir_prefix}" diff --git a/tests/data/cases/funcdef_return_type_trailing_comma.py b/tests/data/cases/funcdef_return_type_trailing_comma.py new file mode 100644 index 00000000000..9b9b9c673de --- /dev/null +++ b/tests/data/cases/funcdef_return_type_trailing_comma.py @@ -0,0 +1,301 @@ +# flags: --preview --minimum-version=3.10 +# normal, short, function definition +def foo(a, b) -> tuple[int, float]: ... + + +# normal, short, function definition w/o return type +def foo(a, b): ... + + +# no splitting +def foo(a: A, b: B) -> list[p, q]: + pass + + +# magic trailing comma in param list +def foo(a, b,): ... + + +# magic trailing comma in nested params in param list +def foo(a, b: tuple[int, float,]): ... + + +# magic trailing comma in return type, no params +def a() -> tuple[ + a, + b, +]: ... + + +# magic trailing comma in return type, params +def foo(a: A, b: B) -> list[ + p, + q, +]: + pass + + +# magic trailing comma in param list and in return type +def foo( + a: a, + b: b, +) -> list[ + a, + a, +]: + pass + + +# long function definition, param list is longer +def aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa( + bbbbbbbbbbbbbbbbbb, +) -> cccccccccccccccccccccccccccccc: ... + + +# long function definition, return type is longer +# this should maybe split on rhs? +def aaaaaaaaaaaaaaaaa(bbbbbbbbbbbbbbbbbb) -> list[ + Ccccccccccccccccccccccccccccccccccccccccccccccccccc, Dddddd +]: ... + + +# long return type, no param list +def foo() -> list[ + Loooooooooooooooooooooooooooooooooooong, + Loooooooooooooooooooong, + Looooooooooooong, +]: ... + + +# long function name, no param list, no return value +def thiiiiiiiiiiiiiiiiiis_iiiiiiiiiiiiiiiiiiiiiiiiiiiiiis_veeeeeeeeeeeeeeeeeeeeeeery_looooooong(): + pass + + +# long function name, no param list +def thiiiiiiiiiiiiiiiiiis_iiiiiiiiiiiiiiiiiiiiiiiiiiiiiis_veeeeeeeeeeeeeeeeeeeeeeery_looooooong() -> ( + list[int, float] +): ... + + +# long function name, no return value +def thiiiiiiiiiiiiiiiiiis_iiiiiiiiiiiiiiiiiiiiiiiiiiiiiis_veeeeeeeeeeeeeeeeeeeeeeery_looooooong( + a, b +): ... + + +# unskippable type hint (??) +def foo(a) -> list[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa]: # type: ignore + pass + + +def foo(a) -> list[ + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +]: # abpedeifnore + pass + +def foo(a, b: list[Bad],): ... # type: ignore + +# don't lose any comments (no magic) +def foo( # 1 + a, # 2 + b) -> list[ # 3 + a, # 4 + b]: # 5 + ... # 6 + + +# don't lose any comments (param list magic) +def foo( # 1 + a, # 2 + b,) -> list[ # 3 + a, # 4 + b]: # 5 + ... # 6 + + +# don't lose any comments (return type magic) +def foo( # 1 + a, # 2 + b) -> list[ # 3 + a, # 4 + b,]: # 5 + ... # 6 + + +# don't lose any comments (both magic) +def foo( # 1 + a, # 2 + b,) -> list[ # 3 + a, # 4 + b,]: # 5 + ... # 6 + +# real life example +def SimplePyFn( + context: hl.GeneratorContext, + buffer_input: Buffer[UInt8, 2], + func_input: Buffer[Int32, 2], + float_arg: Scalar[Float32], + offset: int = 0, +) -> tuple[ + Buffer[UInt8, 2], + Buffer[UInt8, 2], +]: ... +# output +# normal, short, function definition +def foo(a, b) -> tuple[int, float]: ... + + +# normal, short, function definition w/o return type +def foo(a, b): ... + + +# no splitting +def foo(a: A, b: B) -> list[p, q]: + pass + + +# magic trailing comma in param list +def foo( + a, + b, +): ... + + +# magic trailing comma in nested params in param list +def foo( + a, + b: tuple[ + int, + float, + ], +): ... + + +# magic trailing comma in return type, no params +def a() -> tuple[ + a, + b, +]: ... + + +# magic trailing comma in return type, params +def foo(a: A, b: B) -> list[ + p, + q, +]: + pass + + +# magic trailing comma in param list and in return type +def foo( + a: a, + b: b, +) -> list[ + a, + a, +]: + pass + + +# long function definition, param list is longer +def aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa( + bbbbbbbbbbbbbbbbbb, +) -> cccccccccccccccccccccccccccccc: ... + + +# long function definition, return type is longer +# this should maybe split on rhs? +def aaaaaaaaaaaaaaaaa( + bbbbbbbbbbbbbbbbbb, +) -> list[Ccccccccccccccccccccccccccccccccccccccccccccccccccc, Dddddd]: ... + + +# long return type, no param list +def foo() -> list[ + Loooooooooooooooooooooooooooooooooooong, + Loooooooooooooooooooong, + Looooooooooooong, +]: ... + + +# long function name, no param list, no return value +def thiiiiiiiiiiiiiiiiiis_iiiiiiiiiiiiiiiiiiiiiiiiiiiiiis_veeeeeeeeeeeeeeeeeeeeeeery_looooooong(): + pass + + +# long function name, no param list +def thiiiiiiiiiiiiiiiiiis_iiiiiiiiiiiiiiiiiiiiiiiiiiiiiis_veeeeeeeeeeeeeeeeeeeeeeery_looooooong() -> ( + list[int, float] +): ... + + +# long function name, no return value +def thiiiiiiiiiiiiiiiiiis_iiiiiiiiiiiiiiiiiiiiiiiiiiiiiis_veeeeeeeeeeeeeeeeeeeeeeery_looooooong( + a, b +): ... + + +# unskippable type hint (??) +def foo(a) -> list[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa]: # type: ignore + pass + + +def foo( + a, +) -> list[ + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +]: # abpedeifnore + pass + + +def foo( + a, + b: list[Bad], +): ... # type: ignore + + +# don't lose any comments (no magic) +def foo(a, b) -> list[a, b]: # 1 # 2 # 3 # 4 # 5 + ... # 6 + + +# don't lose any comments (param list magic) +def foo( # 1 + a, # 2 + b, +) -> list[a, b]: # 3 # 4 # 5 + ... # 6 + + +# don't lose any comments (return type magic) +def foo(a, b) -> list[ # 1 # 2 # 3 + a, # 4 + b, +]: # 5 + ... # 6 + + +# don't lose any comments (both magic) +def foo( # 1 + a, # 2 + b, +) -> list[ # 3 + a, # 4 + b, +]: # 5 + ... # 6 + + +# real life example +def SimplePyFn( + context: hl.GeneratorContext, + buffer_input: Buffer[UInt8, 2], + func_input: Buffer[Int32, 2], + float_arg: Scalar[Float32], + offset: int = 0, +) -> tuple[ + Buffer[UInt8, 2], + Buffer[UInt8, 2], +]: ... diff --git a/tests/data/simple_cases/function.py b/tests/data/cases/function.py similarity index 100% rename from tests/data/simple_cases/function.py rename to tests/data/cases/function.py diff --git a/tests/data/simple_cases/function2.py b/tests/data/cases/function2.py similarity index 100% rename from tests/data/simple_cases/function2.py rename to tests/data/cases/function2.py diff --git a/tests/data/simple_cases/function_trailing_comma.py b/tests/data/cases/function_trailing_comma.py similarity index 100% rename from tests/data/simple_cases/function_trailing_comma.py rename to tests/data/cases/function_trailing_comma.py diff --git a/tests/data/cases/ignore_pyi.py b/tests/data/cases/ignore_pyi.py new file mode 100644 index 00000000000..4fae7530eb9 --- /dev/null +++ b/tests/data/cases/ignore_pyi.py @@ -0,0 +1,42 @@ +# flags: --pyi +def f(): # type: ignore + ... + +class x: # some comment + ... + +class y: + ... # comment + +# whitespace doesn't matter (note the next line has a trailing space and tab) +class z: + ... + +def g(): + # hi + ... + +def h(): + ... + # bye + +# output + +def f(): # type: ignore + ... + +class x: # some comment + ... + +class y: ... # comment + +# whitespace doesn't matter (note the next line has a trailing space and tab) +class z: ... + +def g(): + # hi + ... + +def h(): + ... + # bye diff --git a/tests/data/simple_cases/import_spacing.py b/tests/data/cases/import_spacing.py similarity index 100% rename from tests/data/simple_cases/import_spacing.py rename to tests/data/cases/import_spacing.py diff --git a/tests/data/cases/line_ranges_basic.py b/tests/data/cases/line_ranges_basic.py new file mode 100644 index 00000000000..9f0fb2da70e --- /dev/null +++ b/tests/data/cases/line_ranges_basic.py @@ -0,0 +1,107 @@ +# flags: --line-ranges=5-6 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. +def foo1(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass +def foo2(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass +def foo3(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass +def foo4(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass + +# Adding some unformated code covering a wide range of syntaxes. + +if True: + # Incorrectly indented prefix comments. + pass + +import typing +from typing import ( + Any , + ) +class MyClass( object): # Trailing comment with extra leading space. + #NOTE: The following indentation is incorrect: + @decor( 1 * 3 ) + def my_func( arg): + pass + +try: # Trailing comment with extra leading space. + for i in range(10): # Trailing comment with extra leading space. + while condition: + if something: + then_something( ) + elif something_else: + then_something_else( ) +except ValueError as e: + unformatted( ) +finally: + unformatted( ) + +async def test_async_unformatted( ): # Trailing comment with extra leading space. + async for i in some_iter( unformatted ): # Trailing comment with extra leading space. + await asyncio.sleep( 1 ) + async with some_context( unformatted ): + print( "unformatted" ) + + +# output +# flags: --line-ranges=5-6 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. +def foo1(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass +def foo2( + parameter_1, + parameter_2, + parameter_3, + parameter_4, + parameter_5, + parameter_6, + parameter_7, +): + pass + + +def foo3( + parameter_1, + parameter_2, + parameter_3, + parameter_4, + parameter_5, + parameter_6, + parameter_7, +): + pass + + +def foo4(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass + +# Adding some unformated code covering a wide range of syntaxes. + +if True: + # Incorrectly indented prefix comments. + pass + +import typing +from typing import ( + Any , + ) +class MyClass( object): # Trailing comment with extra leading space. + #NOTE: The following indentation is incorrect: + @decor( 1 * 3 ) + def my_func( arg): + pass + +try: # Trailing comment with extra leading space. + for i in range(10): # Trailing comment with extra leading space. + while condition: + if something: + then_something( ) + elif something_else: + then_something_else( ) +except ValueError as e: + unformatted( ) +finally: + unformatted( ) + +async def test_async_unformatted( ): # Trailing comment with extra leading space. + async for i in some_iter( unformatted ): # Trailing comment with extra leading space. + await asyncio.sleep( 1 ) + async with some_context( unformatted ): + print( "unformatted" ) diff --git a/tests/data/cases/line_ranges_diff_edge_case.py b/tests/data/cases/line_ranges_diff_edge_case.py new file mode 100644 index 00000000000..f5cb2d0bb5f --- /dev/null +++ b/tests/data/cases/line_ranges_diff_edge_case.py @@ -0,0 +1,28 @@ +# flags: --line-ranges=10-11 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# Reproducible example for https://github.com/psf/black/issues/4033. +# This can be fixed in the future if we use a better diffing algorithm, or make Black +# perform formatting in a single pass. + +print ( "format me" ) +print ( "format me" ) +print ( "format me" ) +print ( "format me" ) +print ( "format me" ) + +# output +# flags: --line-ranges=10-11 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# Reproducible example for https://github.com/psf/black/issues/4033. +# This can be fixed in the future if we use a better diffing algorithm, or make Black +# perform formatting in a single pass. + +print ( "format me" ) +print("format me") +print("format me") +print("format me") +print("format me") diff --git a/tests/data/cases/line_ranges_fmt_off.py b/tests/data/cases/line_ranges_fmt_off.py new file mode 100644 index 00000000000..b51cef58fe5 --- /dev/null +++ b/tests/data/cases/line_ranges_fmt_off.py @@ -0,0 +1,49 @@ +# flags: --line-ranges=7-7 --line-ranges=17-23 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# fmt: off +import os +def myfunc( ): # Intentionally unformatted. + pass +# fmt: on + + +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: off +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: on + + +def myfunc( ): # This will be reformatted. + print( {"this will be reformatted"} ) + +# output + +# flags: --line-ranges=7-7 --line-ranges=17-23 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# fmt: off +import os +def myfunc( ): # Intentionally unformatted. + pass +# fmt: on + + +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: off +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: on + + +def myfunc(): # This will be reformatted. + print({"this will be reformatted"}) diff --git a/tests/data/cases/line_ranges_fmt_off_decorator.py b/tests/data/cases/line_ranges_fmt_off_decorator.py new file mode 100644 index 00000000000..065bf4328d7 --- /dev/null +++ b/tests/data/cases/line_ranges_fmt_off_decorator.py @@ -0,0 +1,47 @@ +# flags: --line-ranges=12-12 --line-ranges=21-21 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# Regression test for an edge case involving decorators and fmt: off/on. +class MyClass: + + # fmt: off + @decorator ( ) + # fmt: on + def method(): + print ( "str" ) + + @decor( + a=1, + # fmt: off + b=(2, 3), + # fmt: on + ) + def func(): + pass + + +# output + +# flags: --line-ranges=12-12 --line-ranges=21-21 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# Regression test for an edge case involving decorators and fmt: off/on. +class MyClass: + + # fmt: off + @decorator ( ) + # fmt: on + def method(): + print("str") + + @decor( + a=1, + # fmt: off + b=(2, 3), + # fmt: on + ) + def func(): + pass + diff --git a/tests/data/cases/line_ranges_fmt_off_overlap.py b/tests/data/cases/line_ranges_fmt_off_overlap.py new file mode 100644 index 00000000000..0391d17a843 --- /dev/null +++ b/tests/data/cases/line_ranges_fmt_off_overlap.py @@ -0,0 +1,37 @@ +# flags: --line-ranges=11-17 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + + +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: off +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: on + + +def myfunc( ): # This will be reformatted. + print( {"this will be reformatted"} ) + +# output + +# flags: --line-ranges=11-17 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + + +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: off +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: on + + +def myfunc(): # This will be reformatted. + print({"this will be reformatted"}) diff --git a/tests/data/cases/line_ranges_imports.py b/tests/data/cases/line_ranges_imports.py new file mode 100644 index 00000000000..76b18ffecb3 --- /dev/null +++ b/tests/data/cases/line_ranges_imports.py @@ -0,0 +1,9 @@ +# flags: --line-ranges=8-8 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# This test ensures no empty lines are added around import lines. +# It caused an issue before https://github.com/psf/black/pull/3610 is merged. +import os +import re +import sys diff --git a/tests/data/cases/line_ranges_indentation.py b/tests/data/cases/line_ranges_indentation.py new file mode 100644 index 00000000000..82d3ad69a5e --- /dev/null +++ b/tests/data/cases/line_ranges_indentation.py @@ -0,0 +1,27 @@ +# flags: --line-ranges=5-5 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. +if cond1: + print("first") + if cond2: + print("second") + else: + print("else") + +if another_cond: + print("will not be changed") + +# output + +# flags: --line-ranges=5-5 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. +if cond1: + print("first") + if cond2: + print("second") + else: + print("else") + +if another_cond: + print("will not be changed") diff --git a/tests/data/cases/line_ranges_two_passes.py b/tests/data/cases/line_ranges_two_passes.py new file mode 100644 index 00000000000..aeed3260b8e --- /dev/null +++ b/tests/data/cases/line_ranges_two_passes.py @@ -0,0 +1,27 @@ +# flags: --line-ranges=9-11 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# This is a specific case for Black's two-pass formatting behavior in `format_str`. +# The second pass must respect the line ranges before the first pass. + + +def restrict_to_this_line(arg1, + arg2, + arg3): + print ( "This should not be formatted." ) + print ( "Note that in the second pass, the original line range 9-11 will cover these print lines.") + +# output + +# flags: --line-ranges=9-11 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# This is a specific case for Black's two-pass formatting behavior in `format_str`. +# The second pass must respect the line ranges before the first pass. + + +def restrict_to_this_line(arg1, arg2, arg3): + print ( "This should not be formatted." ) + print ( "Note that in the second pass, the original line range 9-11 will cover these print lines.") diff --git a/tests/data/cases/line_ranges_unwrapping.py b/tests/data/cases/line_ranges_unwrapping.py new file mode 100644 index 00000000000..cd7751b9417 --- /dev/null +++ b/tests/data/cases/line_ranges_unwrapping.py @@ -0,0 +1,25 @@ +# flags: --line-ranges=5-5 --line-ranges=9-9 --line-ranges=13-13 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. +alist = [ + 1, 2 +] + +adict = { + "key" : "value" +} + +func_call ( + arg = value +) + +# output + +# flags: --line-ranges=5-5 --line-ranges=9-9 --line-ranges=13-13 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. +alist = [1, 2] + +adict = {"key": "value"} + +func_call(arg=value) diff --git a/tests/data/miscellaneous/linelength6.py b/tests/data/cases/linelength6.py similarity index 80% rename from tests/data/miscellaneous/linelength6.py rename to tests/data/cases/linelength6.py index 4fb342726f5..158038bf960 100644 --- a/tests/data/miscellaneous/linelength6.py +++ b/tests/data/cases/linelength6.py @@ -1,3 +1,4 @@ +# flags: --line-length=6 # Regression test for #3427, which reproes only with line length <= 6 def f(): """ diff --git a/tests/data/miscellaneous/long_strings_flag_disabled.py b/tests/data/cases/long_strings_flag_disabled.py similarity index 100% rename from tests/data/miscellaneous/long_strings_flag_disabled.py rename to tests/data/cases/long_strings_flag_disabled.py diff --git a/tests/data/cases/module_docstring_1.py b/tests/data/cases/module_docstring_1.py new file mode 100644 index 00000000000..d5897b4db60 --- /dev/null +++ b/tests/data/cases/module_docstring_1.py @@ -0,0 +1,26 @@ +# flags: --preview +"""Single line module-level docstring should be followed by single newline.""" + + + + +a = 1 + + +"""I'm just a string so should be followed by 2 newlines.""" + + + + +b = 2 + +# output +"""Single line module-level docstring should be followed by single newline.""" + +a = 1 + + +"""I'm just a string so should be followed by 2 newlines.""" + + +b = 2 diff --git a/tests/data/cases/module_docstring_2.py b/tests/data/cases/module_docstring_2.py new file mode 100644 index 00000000000..e1f81b4d76b --- /dev/null +++ b/tests/data/cases/module_docstring_2.py @@ -0,0 +1,68 @@ +# flags: --preview +"""I am a very helpful module docstring. + +Lorem ipsum dolor sit amet, consectetur adipiscing elit, +sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. +Ut enim ad minim veniam, +quis nostrud exercitation ullamco laboris +nisi ut aliquip ex ea commodo consequat. +Duis aute irure dolor in reprehenderit in voluptate +velit esse cillum dolore eu fugiat nulla pariatur. +Excepteur sint occaecat cupidatat non proident, +sunt in culpa qui officia deserunt mollit anim id est laborum. +""" + + + + +a = 1 + + +"""Look at me I'm a docstring... + +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +........................................................NOT! +""" + + + + +b = 2 + +# output +"""I am a very helpful module docstring. + +Lorem ipsum dolor sit amet, consectetur adipiscing elit, +sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. +Ut enim ad minim veniam, +quis nostrud exercitation ullamco laboris +nisi ut aliquip ex ea commodo consequat. +Duis aute irure dolor in reprehenderit in voluptate +velit esse cillum dolore eu fugiat nulla pariatur. +Excepteur sint occaecat cupidatat non proident, +sunt in culpa qui officia deserunt mollit anim id est laborum. +""" + +a = 1 + + +"""Look at me I'm a docstring... + +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +........................................................NOT! +""" + + +b = 2 diff --git a/tests/data/cases/module_docstring_3.py b/tests/data/cases/module_docstring_3.py new file mode 100644 index 00000000000..0631e136a3d --- /dev/null +++ b/tests/data/cases/module_docstring_3.py @@ -0,0 +1,8 @@ +# flags: --preview +"""Single line module-level docstring should be followed by single newline.""" +a = 1 + +# output +"""Single line module-level docstring should be followed by single newline.""" + +a = 1 diff --git a/tests/data/cases/module_docstring_4.py b/tests/data/cases/module_docstring_4.py new file mode 100644 index 00000000000..515174dcc04 --- /dev/null +++ b/tests/data/cases/module_docstring_4.py @@ -0,0 +1,9 @@ +# flags: --preview +"""Single line module-level docstring should be followed by single newline.""" + +a = 1 + +# output +"""Single line module-level docstring should be followed by single newline.""" + +a = 1 diff --git a/tests/data/cases/module_docstring_followed_by_class.py b/tests/data/cases/module_docstring_followed_by_class.py new file mode 100644 index 00000000000..6fdbfc8c240 --- /dev/null +++ b/tests/data/cases/module_docstring_followed_by_class.py @@ -0,0 +1,11 @@ +# flags: --preview +"""Two blank lines between module docstring and a class.""" +class MyClass: + pass + +# output +"""Two blank lines between module docstring and a class.""" + + +class MyClass: + pass diff --git a/tests/data/cases/module_docstring_followed_by_function.py b/tests/data/cases/module_docstring_followed_by_function.py new file mode 100644 index 00000000000..5913a59e1fe --- /dev/null +++ b/tests/data/cases/module_docstring_followed_by_function.py @@ -0,0 +1,11 @@ +# flags: --preview +"""Two blank lines between module docstring and a function def.""" +def function(): + pass + +# output +"""Two blank lines between module docstring and a function def.""" + + +def function(): + pass diff --git a/tests/data/cases/multiline_consecutive_open_parentheses_ignore.py b/tests/data/cases/multiline_consecutive_open_parentheses_ignore.py new file mode 100644 index 00000000000..6ec8bb45408 --- /dev/null +++ b/tests/data/cases/multiline_consecutive_open_parentheses_ignore.py @@ -0,0 +1,41 @@ +# This is a regression test. Issue #3737 + +a = ( # type: ignore + int( # type: ignore + int( # type: ignore + int( # type: ignore + 6 + ) + ) + ) +) + +b = ( + int( + 6 + ) +) + +print( "111") # type: ignore +print( "111" ) # type: ignore +print( "111" ) # type: ignore + + +# output + + +# This is a regression test. Issue #3737 + +a = ( # type: ignore + int( # type: ignore + int( # type: ignore + int(6) # type: ignore + ) + ) +) + +b = int(6) + +print("111") # type: ignore +print("111") # type: ignore +print("111") # type: ignore \ No newline at end of file diff --git a/tests/data/cases/nested_stub.py b/tests/data/cases/nested_stub.py new file mode 100644 index 00000000000..b81549ec115 --- /dev/null +++ b/tests/data/cases/nested_stub.py @@ -0,0 +1,44 @@ +# flags: --pyi --preview +import sys + +class Outer: + class InnerStub: ... + outer_attr_after_inner_stub: int + class Inner: + inner_attr: int + outer_attr: int + +if sys.version_info > (3, 7): + if sys.platform == "win32": + assignment = 1 + def function_definition(self): ... + def f1(self) -> str: ... + if sys.platform != "win32": + def function_definition(self): ... + assignment = 1 + def f2(self) -> str: ... + +# output + +import sys + +class Outer: + class InnerStub: ... + outer_attr_after_inner_stub: int + + class Inner: + inner_attr: int + + outer_attr: int + +if sys.version_info > (3, 7): + if sys.platform == "win32": + assignment = 1 + def function_definition(self): ... + + def f1(self) -> str: ... + if sys.platform != "win32": + def function_definition(self): ... + assignment = 1 + + def f2(self) -> str: ... \ No newline at end of file diff --git a/tests/data/py_36/numeric_literals.py b/tests/data/cases/numeric_literals.py similarity index 91% rename from tests/data/py_36/numeric_literals.py rename to tests/data/cases/numeric_literals.py index 254da68d330..99669328744 100644 --- a/tests/data/py_36/numeric_literals.py +++ b/tests/data/cases/numeric_literals.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3.6 - x = 123456789 x = 123456 x = .1 @@ -21,9 +19,6 @@ # output - -#!/usr/bin/env python3.6 - x = 123456789 x = 123456 x = 0.1 diff --git a/tests/data/py_36/numeric_literals_skip_underscores.py b/tests/data/cases/numeric_literals_skip_underscores.py similarity index 77% rename from tests/data/py_36/numeric_literals_skip_underscores.py rename to tests/data/cases/numeric_literals_skip_underscores.py index e345bb90276..6d60bdbb34d 100644 --- a/tests/data/py_36/numeric_literals_skip_underscores.py +++ b/tests/data/cases/numeric_literals_skip_underscores.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3.6 - x = 123456789 x = 1_2_3_4_5_6_7 x = 1E+1 @@ -11,8 +9,6 @@ # output -#!/usr/bin/env python3.6 - x = 123456789 x = 1_2_3_4_5_6_7 x = 1e1 diff --git a/tests/data/simple_cases/one_element_subscript.py b/tests/data/cases/one_element_subscript.py similarity index 100% rename from tests/data/simple_cases/one_element_subscript.py rename to tests/data/cases/one_element_subscript.py diff --git a/tests/data/py_310/parenthesized_context_managers.py b/tests/data/cases/parenthesized_context_managers.py similarity index 95% rename from tests/data/py_310/parenthesized_context_managers.py rename to tests/data/cases/parenthesized_context_managers.py index 1ef09a1bd34..16645a18baa 100644 --- a/tests/data/py_310/parenthesized_context_managers.py +++ b/tests/data/cases/parenthesized_context_managers.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.10 with (CtxManager() as example): ... diff --git a/tests/data/py_310/pattern_matching_complex.py b/tests/data/cases/pattern_matching_complex.py similarity index 89% rename from tests/data/py_310/pattern_matching_complex.py rename to tests/data/cases/pattern_matching_complex.py index 97ee194fd39..ba64f2639a0 100644 --- a/tests/data/py_310/pattern_matching_complex.py +++ b/tests/data/cases/pattern_matching_complex.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.10 # Cases sampled from Lib/test/test_patma.py # case black_test_patma_098 @@ -142,3 +143,13 @@ y = 1 case []: y = 2 +# issue 3790 +match (X.type, Y): + case _: + pass +# issue 3487 +match = re.match(r"(?PLD|MD|HD)(?PAL|SS)", "HDSS") + +match (match.group("grade"), match.group("material")): + case ("MD" | "HD", "SS" as code): + print("You will get here") diff --git a/tests/data/py_310/pattern_matching_extras.py b/tests/data/cases/pattern_matching_extras.py similarity index 88% rename from tests/data/py_310/pattern_matching_extras.py rename to tests/data/cases/pattern_matching_extras.py index 0242d264e5b..1aef8f16b5a 100644 --- a/tests/data/py_310/pattern_matching_extras.py +++ b/tests/data/cases/pattern_matching_extras.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.10 import match match something: @@ -29,22 +30,6 @@ def func(match: case, case: match) -> case: ... -match maybe, multiple: - case perhaps, 5: - pass - case perhaps, 6,: - pass - - -match more := (than, one), indeed,: - case _, (5, 6): - pass - case [[5], (6)], [7],: - pass - case _: - pass - - match a, *b, c: case [*_]: assert "seq" == _ @@ -66,12 +51,12 @@ def func(match: case, case: match) -> case: ), ): pass - case [a as match]: pass - case case: pass + case something: + pass match match: @@ -97,10 +82,8 @@ def func(match: case, case: match) -> case: match something: case 1 as a: pass - case 2 as b, 3 as c: pass - case 4 as d, (5 as e), (6 | 7 as g), *h: pass diff --git a/tests/data/py_310/pattern_matching_generic.py b/tests/data/cases/pattern_matching_generic.py similarity index 98% rename from tests/data/py_310/pattern_matching_generic.py rename to tests/data/cases/pattern_matching_generic.py index 00a0e4a677d..4b4d45f0bff 100644 --- a/tests/data/py_310/pattern_matching_generic.py +++ b/tests/data/cases/pattern_matching_generic.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.10 re.match() match = a with match() as match: diff --git a/tests/data/py_310/pattern_matching_simple.py b/tests/data/cases/pattern_matching_simple.py similarity index 98% rename from tests/data/py_310/pattern_matching_simple.py rename to tests/data/cases/pattern_matching_simple.py index 5ed62415a4b..6fa2000f0de 100644 --- a/tests/data/py_310/pattern_matching_simple.py +++ b/tests/data/cases/pattern_matching_simple.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.10 # Cases sampled from PEP 636 examples match command.split(): diff --git a/tests/data/py_310/pattern_matching_style.py b/tests/data/cases/pattern_matching_style.py similarity index 97% rename from tests/data/py_310/pattern_matching_style.py rename to tests/data/cases/pattern_matching_style.py index 8e18ce2ada6..2ee6ea2b6e9 100644 --- a/tests/data/py_310/pattern_matching_style.py +++ b/tests/data/cases/pattern_matching_style.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.10 match something: case b(): print(1+1) case c( diff --git a/tests/data/cases/pep604_union_types_line_breaks.py b/tests/data/cases/pep604_union_types_line_breaks.py new file mode 100644 index 00000000000..fee2b840494 --- /dev/null +++ b/tests/data/cases/pep604_union_types_line_breaks.py @@ -0,0 +1,188 @@ +# flags: --preview --minimum-version=3.10 +# This has always worked +z= Loooooooooooooooooooooooong | Loooooooooooooooooooooooong | Loooooooooooooooooooooooong | Loooooooooooooooooooooooong + +# "AnnAssign"s now also work +z: Loooooooooooooooooooooooong | Loooooooooooooooooooooooong | Loooooooooooooooooooooooong | Loooooooooooooooooooooooong +z: (Short + | Short2 + | Short3 + | Short4) +z: (int) +z: ((int)) + + +z: Loooooooooooooooooooooooong | Loooooooooooooooooooooooong | Loooooooooooooooooooooooong | Loooooooooooooooooooooooong = 7 +z: (Short + | Short2 + | Short3 + | Short4) = 8 +z: (int) = 2.3 +z: ((int)) = foo() + +# In case I go for not enforcing parantheses, this might get improved at the same time +x = ( + z + == 9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999, + y + == 9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999, +) + +x = ( + z == (9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999), + y == (9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999), +) + +# handle formatting of "tname"s in parameter list + +# remove unnecessary paren +def foo(i: (int)) -> None: ... + + +# this is a syntax error in the type annotation according to mypy, but it's not invalid *python* code, so make sure we don't mess with it and make it so. +def foo(i: (int,)) -> None: ... + +def foo( + i: int, + x: Loooooooooooooooooooooooong + | Looooooooooooooooong + | Looooooooooooooooooooong + | Looooooong, + *, + s: str, +) -> None: + pass + + +@app.get("/path/") +async def foo( + q: str + | None = Query(None, title="Some long title", description="Some long description") +): + pass + + +def f( + max_jobs: int + | None = Option( + None, help="Maximum number of jobs to launch. And some additional text." + ), + another_option: bool = False + ): + ... + + +# output +# This has always worked +z = ( + Loooooooooooooooooooooooong + | Loooooooooooooooooooooooong + | Loooooooooooooooooooooooong + | Loooooooooooooooooooooooong +) + +# "AnnAssign"s now also work +z: ( + Loooooooooooooooooooooooong + | Loooooooooooooooooooooooong + | Loooooooooooooooooooooooong + | Loooooooooooooooooooooooong +) +z: Short | Short2 | Short3 | Short4 +z: int +z: int + + +z: ( + Loooooooooooooooooooooooong + | Loooooooooooooooooooooooong + | Loooooooooooooooooooooooong + | Loooooooooooooooooooooooong +) = 7 +z: Short | Short2 | Short3 | Short4 = 8 +z: int = 2.3 +z: int = foo() + +# In case I go for not enforcing parantheses, this might get improved at the same time +x = ( + z + == 9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999, + y + == 9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999, +) + +x = ( + z + == ( + 9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999 + ), + y + == ( + 9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999 + ), +) + +# handle formatting of "tname"s in parameter list + + +# remove unnecessary paren +def foo(i: int) -> None: ... + + +# this is a syntax error in the type annotation according to mypy, but it's not invalid *python* code, so make sure we don't mess with it and make it so. +def foo(i: (int,)) -> None: ... + + +def foo( + i: int, + x: ( + Loooooooooooooooooooooooong + | Looooooooooooooooong + | Looooooooooooooooooooong + | Looooooong + ), + *, + s: str, +) -> None: + pass + + +@app.get("/path/") +async def foo( + q: str | None = Query( + None, title="Some long title", description="Some long description" + ) +): + pass + + +def f( + max_jobs: int | None = Option( + None, help="Maximum number of jobs to launch. And some additional text." + ), + another_option: bool = False, +): ... diff --git a/tests/data/py_38/pep_570.py b/tests/data/cases/pep_570.py similarity index 95% rename from tests/data/py_38/pep_570.py rename to tests/data/cases/pep_570.py index ca8f7ab1d95..2641c2b970e 100644 --- a/tests/data/py_38/pep_570.py +++ b/tests/data/cases/pep_570.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.8 def positional_only_arg(a, /): pass diff --git a/tests/data/py_38/pep_572.py b/tests/data/cases/pep_572.py similarity index 96% rename from tests/data/py_38/pep_572.py rename to tests/data/cases/pep_572.py index d41805f1cb1..742b6d5b7e4 100644 --- a/tests/data/py_38/pep_572.py +++ b/tests/data/cases/pep_572.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.8 (a := 1) (a := a) if (match := pattern.search(data)) is None: diff --git a/tests/data/fast/pep_572_do_not_remove_parens.py b/tests/data/cases/pep_572_do_not_remove_parens.py similarity index 96% rename from tests/data/fast/pep_572_do_not_remove_parens.py rename to tests/data/cases/pep_572_do_not_remove_parens.py index 05619ddcc2b..08dba3ffdf9 100644 --- a/tests/data/fast/pep_572_do_not_remove_parens.py +++ b/tests/data/cases/pep_572_do_not_remove_parens.py @@ -1,3 +1,4 @@ +# flags: --fast # Most of the following examples are really dumb, some of them aren't even accepted by Python, # we're fixing them only so fuzzers (which follow the grammar which actually allows these # examples matter of fact!) don't yell at us :p diff --git a/tests/data/py_310/pep_572_py310.py b/tests/data/cases/pep_572_py310.py similarity index 93% rename from tests/data/py_310/pep_572_py310.py rename to tests/data/cases/pep_572_py310.py index cb82b2d23f8..9f999deeb89 100644 --- a/tests/data/py_310/pep_572_py310.py +++ b/tests/data/cases/pep_572_py310.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.10 # Unparenthesized walruses are now allowed in indices since Python 3.10. x[a:=0] x[a:=0, b:=1] diff --git a/tests/data/py_39/pep_572_py39.py b/tests/data/cases/pep_572_py39.py similarity index 89% rename from tests/data/py_39/pep_572_py39.py rename to tests/data/cases/pep_572_py39.py index b8b081b8c45..d1614624d99 100644 --- a/tests/data/py_39/pep_572_py39.py +++ b/tests/data/cases/pep_572_py39.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.9 # Unparenthesized walruses are now allowed in set literals & set comprehensions # since Python 3.9 {x := 1, 2, 3} diff --git a/tests/data/py_38/pep_572_remove_parens.py b/tests/data/cases/pep_572_remove_parens.py similarity index 98% rename from tests/data/py_38/pep_572_remove_parens.py rename to tests/data/cases/pep_572_remove_parens.py index b952b2940c5..24f1ac29168 100644 --- a/tests/data/py_38/pep_572_remove_parens.py +++ b/tests/data/cases/pep_572_remove_parens.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.8 if (foo := 0): pass diff --git a/tests/data/cases/pep_604.py b/tests/data/cases/pep_604.py new file mode 100644 index 00000000000..b68d59d6440 --- /dev/null +++ b/tests/data/cases/pep_604.py @@ -0,0 +1,25 @@ +def some_very_long_name_function() -> my_module.Asdf | my_module.AnotherType | my_module.YetAnotherType | None: + pass + + +def some_very_long_name_function() -> my_module.Asdf | my_module.AnotherType | my_module.YetAnotherType | my_module.EvenMoreType | None: + pass + + +# output + + +def some_very_long_name_function() -> ( + my_module.Asdf | my_module.AnotherType | my_module.YetAnotherType | None +): + pass + + +def some_very_long_name_function() -> ( + my_module.Asdf + | my_module.AnotherType + | my_module.YetAnotherType + | my_module.EvenMoreType + | None +): + pass diff --git a/tests/data/py_311/pep_646.py b/tests/data/cases/pep_646.py similarity index 98% rename from tests/data/py_311/pep_646.py rename to tests/data/cases/pep_646.py index e843ecf39d8..92b568a379c 100644 --- a/tests/data/py_311/pep_646.py +++ b/tests/data/cases/pep_646.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.11 A[*b] A[*b] = 1 A diff --git a/tests/data/py_311/pep_654.py b/tests/data/cases/pep_654.py similarity index 96% rename from tests/data/py_311/pep_654.py rename to tests/data/cases/pep_654.py index 387c0816f4b..12e49180e41 100644 --- a/tests/data/py_311/pep_654.py +++ b/tests/data/cases/pep_654.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.11 try: raise OSError("blah") except* ExceptionGroup as e: diff --git a/tests/data/py_311/pep_654_style.py b/tests/data/cases/pep_654_style.py similarity index 98% rename from tests/data/py_311/pep_654_style.py rename to tests/data/cases/pep_654_style.py index 9fc7c0c84db..0d34650e098 100644 --- a/tests/data/py_311/pep_654_style.py +++ b/tests/data/cases/pep_654_style.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.11 try: raise OSError("blah") except * ExceptionGroup as e: diff --git a/tests/data/miscellaneous/power_op_newline.py b/tests/data/cases/power_op_newline.py similarity index 73% rename from tests/data/miscellaneous/power_op_newline.py rename to tests/data/cases/power_op_newline.py index 85d434d63f6..d9b31403c9d 100644 --- a/tests/data/miscellaneous/power_op_newline.py +++ b/tests/data/cases/power_op_newline.py @@ -1,3 +1,4 @@ +# flags: --line-length=0 importA;()<<0**0# # output diff --git a/tests/data/simple_cases/power_op_spacing.py b/tests/data/cases/power_op_spacing.py similarity index 95% rename from tests/data/simple_cases/power_op_spacing.py rename to tests/data/cases/power_op_spacing.py index c95fa788fc3..b3ef0aae084 100644 --- a/tests/data/simple_cases/power_op_spacing.py +++ b/tests/data/cases/power_op_spacing.py @@ -29,6 +29,13 @@ def function_dont_replace_spaces(): p = {(k, k**2): v**2 for k, v in pairs} q = [10**i for i in range(6)] r = x**y +s = 1 ** 1 +t = ( + 1 + ** 1 + **1 + ** 1 +) a = 5.0**~4.0 b = 5.0 ** f() @@ -47,6 +54,13 @@ def function_dont_replace_spaces(): o = settings(max_examples=10**6.0) p = {(k, k**2): v**2.0 for k, v in pairs} q = [10.5**i for i in range(6)] +s = 1.0 ** 1.0 +t = ( + 1.0 + ** 1.0 + **1.0 + ** 1.0 +) # WE SHOULD DEFINITELY NOT EAT THESE COMMENTS (https://github.com/psf/black/issues/2873) @@ -97,6 +111,8 @@ def function_dont_replace_spaces(): p = {(k, k**2): v**2 for k, v in pairs} q = [10**i for i in range(6)] r = x**y +s = 1**1 +t = 1**1**1**1 a = 5.0**~4.0 b = 5.0 ** f() @@ -115,6 +131,8 @@ def function_dont_replace_spaces(): o = settings(max_examples=10**6.0) p = {(k, k**2): v**2.0 for k, v in pairs} q = [10.5**i for i in range(6)] +s = 1.0**1.0 +t = 1.0**1.0**1.0**1.0 # WE SHOULD DEFINITELY NOT EAT THESE COMMENTS (https://github.com/psf/black/issues/2873) diff --git a/tests/data/simple_cases/prefer_rhs_split_reformatted.py b/tests/data/cases/prefer_rhs_split_reformatted.py similarity index 85% rename from tests/data/simple_cases/prefer_rhs_split_reformatted.py rename to tests/data/cases/prefer_rhs_split_reformatted.py index 781e75be0aa..e15e5ddc86d 100644 --- a/tests/data/simple_cases/prefer_rhs_split_reformatted.py +++ b/tests/data/cases/prefer_rhs_split_reformatted.py @@ -7,7 +7,7 @@ arg2, ) -# Make when when the left side of assignement plus the opening paren "... = (" is +# Make when when the left side of assignment plus the opening paren "... = (" is # exactly line length limit + 1, it won't be split like that. xxxxxxxxx_yyy_zzzzzzzz[xx.xxxxxx(x_yyy_zzzzzz.xxxxx[0]), x_yyy_zzzzzz.xxxxxx(xxxx=1)] = 1 @@ -31,7 +31,7 @@ arg2, ) -# Make when when the left side of assignement plus the opening paren "... = (" is +# Make when when the left side of assignment plus the opening paren "... = (" is # exactly line length limit + 1, it won't be split like that. xxxxxxxxx_yyy_zzzzzzzz[ xx.xxxxxx(x_yyy_zzzzzz.xxxxx[0]), x_yyy_zzzzzz.xxxxxx(xxxx=1) diff --git a/tests/data/cases/preview_allow_empty_first_line.py b/tests/data/cases/preview_allow_empty_first_line.py new file mode 100644 index 00000000000..3e14fa15250 --- /dev/null +++ b/tests/data/cases/preview_allow_empty_first_line.py @@ -0,0 +1,128 @@ +# flags: --preview +def foo(): + """ + Docstring + """ + + # Here we go + if x: + + # This is also now fine + a = 123 + + else: + # But not necessary + a = 123 + + if y: + + while True: + + """ + Long comment here + """ + a = 123 + + if z: + + for _ in range(100): + a = 123 + else: + + try: + + # this should be ok + a = 123 + except: + + """also this""" + a = 123 + + +def bar(): + + if x: + a = 123 + + +def baz(): + + # OK + if x: + a = 123 + +def quux(): + + new_line = here + + +class Cls: + + def method(self): + + pass + +# output + +def foo(): + """ + Docstring + """ + + # Here we go + if x: + + # This is also now fine + a = 123 + + else: + # But not necessary + a = 123 + + if y: + + while True: + + """ + Long comment here + """ + a = 123 + + if z: + + for _ in range(100): + a = 123 + else: + + try: + + # this should be ok + a = 123 + except: + + """also this""" + a = 123 + + +def bar(): + + if x: + a = 123 + + +def baz(): + + # OK + if x: + a = 123 + + +def quux(): + + new_line = here + + +class Cls: + def method(self): + + pass diff --git a/tests/data/cases/preview_async_stmts.py b/tests/data/cases/preview_async_stmts.py new file mode 100644 index 00000000000..0a7671be5a6 --- /dev/null +++ b/tests/data/cases/preview_async_stmts.py @@ -0,0 +1,28 @@ +# flags: --preview +async def func() -> (int): + return 0 + + +@decorated +async def func() -> (int): + return 0 + + +async for (item) in async_iter: + pass + + +# output + + +async def func() -> int: + return 0 + + +@decorated +async def func() -> int: + return 0 + + +async for item in async_iter: + pass diff --git a/tests/data/preview/cantfit.py b/tests/data/cases/preview_cantfit.py similarity index 92% rename from tests/data/preview/cantfit.py rename to tests/data/cases/preview_cantfit.py index cade382e30d..d5da6654f0c 100644 --- a/tests/data/preview/cantfit.py +++ b/tests/data/cases/preview_cantfit.py @@ -1,3 +1,4 @@ +# flags: --preview # long variable name this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it = 0 this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it = 1 # with a comment @@ -79,14 +80,10 @@ ) # long arguments normal_name = normal_function_name( - ( - "but with super long string arguments that on their own exceed the line limit" - " so there's no way it can ever fit" - ), - ( - "eggs with spam and eggs and spam with eggs with spam and eggs and spam with" - " eggs with spam and eggs and spam with eggs" - ), + "but with super long string arguments that on their own exceed the line limit so" + " there's no way it can ever fit", + "eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs" + " with spam and eggs and spam with eggs", this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it=0, ) string_variable_name = "a string that is waaaaaaaayyyyyyyy too long, even in parens, there's nothing you can do" # noqa diff --git a/tests/data/preview/comments7.py b/tests/data/cases/preview_comments7.py similarity index 91% rename from tests/data/preview/comments7.py rename to tests/data/cases/preview_comments7.py index ec2dc501d8e..006d4f7266f 100644 --- a/tests/data/preview/comments7.py +++ b/tests/data/cases/preview_comments7.py @@ -1,3 +1,4 @@ +# flags: --preview from .config import ( Any, Bool, @@ -131,6 +132,18 @@ def test_fails_invalid_post_data( square = Square(4) # type: Optional[Square] +# Regression test for https://github.com/psf/black/issues/3756. +[ + ( + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" # aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + ), +] +[ + ( # aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" # aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + ), +] + # output from .config import ( @@ -278,8 +291,19 @@ class C: ) def test_fails_invalid_post_data( self, pyramid_config, db_request, post_data, message - ): - ... + ): ... square = Square(4) # type: Optional[Square] + +# Regression test for https://github.com/psf/black/issues/3756. +[ + ( # aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ), +] +[ + ( # aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" # aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + ), +] diff --git a/tests/data/preview_context_managers/targeting_py38.py b/tests/data/cases/preview_context_managers_38.py similarity index 63% rename from tests/data/preview_context_managers/targeting_py38.py rename to tests/data/cases/preview_context_managers_38.py index 6ec4684e441..719d94fdcc5 100644 --- a/tests/data/preview_context_managers/targeting_py38.py +++ b/tests/data/cases/preview_context_managers_38.py @@ -1,3 +1,4 @@ +# flags: --preview --minimum-version=3.8 with \ make_context_manager1() as cm1, \ make_context_manager2() as cm2, \ @@ -23,6 +24,14 @@ pass +with mock.patch.object( + self.my_runner, "first_method", autospec=True +) as mock_run_adb, mock.patch.object( + self.my_runner, "second_method", autospec=True, return_value="foo" +): + pass + + # output @@ -36,3 +45,11 @@ with new_new_new1() as cm1, new_new_new2(): pass + + +with mock.patch.object( + self.my_runner, "first_method", autospec=True +) as mock_run_adb, mock.patch.object( + self.my_runner, "second_method", autospec=True, return_value="foo" +): + pass diff --git a/tests/data/preview_context_managers/targeting_py39.py b/tests/data/cases/preview_context_managers_39.py similarity index 52% rename from tests/data/preview_context_managers/targeting_py39.py rename to tests/data/cases/preview_context_managers_39.py index 5cb8763040a..589e00ad187 100644 --- a/tests/data/preview_context_managers/targeting_py39.py +++ b/tests/data/cases/preview_context_managers_39.py @@ -1,3 +1,4 @@ +# flags: --preview --minimum-version=3.9 with \ make_context_manager1() as cm1, \ make_context_manager2() as cm2, \ @@ -49,6 +50,41 @@ pass +with mock.patch.object( + self.my_runner, "first_method", autospec=True +) as mock_run_adb, mock.patch.object( + self.my_runner, "second_method", autospec=True, return_value="foo" +): + pass + + +with xxxxxxxx.some_kind_of_method( + some_argument=[ + "first", + "second", + "third", + ] +).another_method() as cmd: + pass + + +async def func(): + async with \ + make_context_manager1() as cm1, \ + make_context_manager2() as cm2, \ + make_context_manager3() as cm3, \ + make_context_manager4() as cm4 \ + : + pass + + async with some_function( + argument1, argument2, argument3="some_value" + ) as some_cm, some_other_function( + argument1, argument2, argument3="some_value" + ): + pass + + # output @@ -84,7 +120,7 @@ # First comment. new_new_new1() as cm1, # Second comment. - new_new_new2() + new_new_new2(), # Last comment. ): pass @@ -102,3 +138,38 @@ ) as cm2, ): pass + + +with ( + mock.patch.object(self.my_runner, "first_method", autospec=True) as mock_run_adb, + mock.patch.object( + self.my_runner, "second_method", autospec=True, return_value="foo" + ), +): + pass + + +with xxxxxxxx.some_kind_of_method( + some_argument=[ + "first", + "second", + "third", + ] +).another_method() as cmd: + pass + + +async def func(): + async with ( + make_context_manager1() as cm1, + make_context_manager2() as cm2, + make_context_manager3() as cm3, + make_context_manager4() as cm4, + ): + pass + + async with ( + some_function(argument1, argument2, argument3="some_value") as some_cm, + some_other_function(argument1, argument2, argument3="some_value"), + ): + pass diff --git a/tests/data/preview_context_managers/auto_detect/features_3_10.py b/tests/data/cases/preview_context_managers_autodetect_310.py similarity index 93% rename from tests/data/preview_context_managers/auto_detect/features_3_10.py rename to tests/data/cases/preview_context_managers_autodetect_310.py index 1458df1cb41..a9e31076f03 100644 --- a/tests/data/preview_context_managers/auto_detect/features_3_10.py +++ b/tests/data/cases/preview_context_managers_autodetect_310.py @@ -1,3 +1,4 @@ +# flags: --preview --minimum-version=3.10 # This file uses pattern matching introduced in Python 3.10. diff --git a/tests/data/preview_context_managers/auto_detect/features_3_11.py b/tests/data/cases/preview_context_managers_autodetect_311.py similarity index 92% rename from tests/data/preview_context_managers/auto_detect/features_3_11.py rename to tests/data/cases/preview_context_managers_autodetect_311.py index f83c5330ab3..af1e83fe74c 100644 --- a/tests/data/preview_context_managers/auto_detect/features_3_11.py +++ b/tests/data/cases/preview_context_managers_autodetect_311.py @@ -1,3 +1,4 @@ +# flags: --preview --minimum-version=3.11 # This file uses except* clause in Python 3.11. diff --git a/tests/data/preview_context_managers/auto_detect/features_3_8.py b/tests/data/cases/preview_context_managers_autodetect_38.py similarity index 61% rename from tests/data/preview_context_managers/auto_detect/features_3_8.py rename to tests/data/cases/preview_context_managers_autodetect_38.py index e05094e1421..25217a40604 100644 --- a/tests/data/preview_context_managers/auto_detect/features_3_8.py +++ b/tests/data/cases/preview_context_managers_autodetect_38.py @@ -1,3 +1,4 @@ +# flags: --preview # This file doesn't use any Python 3.9+ only grammars. @@ -16,6 +17,14 @@ pass +with mock.patch.object( + self.my_runner, "first_method", autospec=True +) as mock_run_adb, mock.patch.object( + self.my_runner, "second_method", autospec=True, return_value="foo" +): + pass + + # output # This file doesn't use any Python 3.9+ only grammars. @@ -28,3 +37,11 @@ with make_context_manager1() as cm1, make_context_manager2() as cm2, make_context_manager3() as cm3, make_context_manager4() as cm4: pass + + +with mock.patch.object( + self.my_runner, "first_method", autospec=True +) as mock_run_adb, mock.patch.object( + self.my_runner, "second_method", autospec=True, return_value="foo" +): + pass diff --git a/tests/data/preview_context_managers/auto_detect/features_3_9.py b/tests/data/cases/preview_context_managers_autodetect_39.py similarity index 93% rename from tests/data/preview_context_managers/auto_detect/features_3_9.py rename to tests/data/cases/preview_context_managers_autodetect_39.py index 0d28f993108..3f72e48db9d 100644 --- a/tests/data/preview_context_managers/auto_detect/features_3_9.py +++ b/tests/data/cases/preview_context_managers_autodetect_39.py @@ -1,3 +1,4 @@ +# flags: --preview --minimum-version=3.9 # This file uses parenthesized context managers introduced in Python 3.9. diff --git a/tests/data/miscellaneous/docstring_preview_no_string_normalization.py b/tests/data/cases/preview_docstring_no_string_normalization.py similarity index 88% rename from tests/data/miscellaneous/docstring_preview_no_string_normalization.py rename to tests/data/cases/preview_docstring_no_string_normalization.py index 338cc01d33e..712c7364f51 100644 --- a/tests/data/miscellaneous/docstring_preview_no_string_normalization.py +++ b/tests/data/cases/preview_docstring_no_string_normalization.py @@ -1,3 +1,4 @@ +# flags: --preview --skip-string-normalization def do_not_touch_this_prefix(): R"""There was a bug where docstring prefixes would be normalized even with -S.""" diff --git a/tests/data/cases/preview_dummy_implementations.py b/tests/data/cases/preview_dummy_implementations.py new file mode 100644 index 00000000000..28b23bb8609 --- /dev/null +++ b/tests/data/cases/preview_dummy_implementations.py @@ -0,0 +1,123 @@ +# flags: --preview +from typing import NoReturn, Protocol, Union, overload + +class Empty: + ... + +def dummy(a): ... +async def other(b): ... + + +@overload +def a(arg: int) -> int: ... +@overload +def a(arg: str) -> str: ... +@overload +def a(arg: object) -> NoReturn: ... +def a(arg: Union[int, str, object]) -> Union[int, str]: + if not isinstance(arg, (int, str)): + raise TypeError + return arg + +class Proto(Protocol): + def foo(self, a: int) -> int: + ... + + def bar(self, b: str) -> str: ... + def baz(self, c: bytes) -> str: + ... + + +def dummy_two(): + ... +@dummy +def dummy_three(): + ... + +def dummy_four(): + ... + +@overload +def b(arg: int) -> int: ... + +@overload +def b(arg: str) -> str: ... +@overload +def b(arg: object) -> NoReturn: ... + +def b(arg: Union[int, str, object]) -> Union[int, str]: + if not isinstance(arg, (int, str)): + raise TypeError + return arg + +def has_comment(): + ... # still a dummy + +if some_condition: + ... + +if already_dummy: ... + +# output + +from typing import NoReturn, Protocol, Union, overload + + +class Empty: ... + + +def dummy(a): ... +async def other(b): ... + + +@overload +def a(arg: int) -> int: ... +@overload +def a(arg: str) -> str: ... +@overload +def a(arg: object) -> NoReturn: ... +def a(arg: Union[int, str, object]) -> Union[int, str]: + if not isinstance(arg, (int, str)): + raise TypeError + return arg + + +class Proto(Protocol): + def foo(self, a: int) -> int: ... + + def bar(self, b: str) -> str: ... + def baz(self, c: bytes) -> str: ... + + +def dummy_two(): ... +@dummy +def dummy_three(): ... + + +def dummy_four(): ... + + +@overload +def b(arg: int) -> int: ... + + +@overload +def b(arg: str) -> str: ... +@overload +def b(arg: object) -> NoReturn: ... + + +def b(arg: Union[int, str, object]) -> Union[int, str]: + if not isinstance(arg, (int, str)): + raise TypeError + return arg + + +def has_comment(): ... # still a dummy + + +if some_condition: + ... + +if already_dummy: + ... diff --git a/tests/data/cases/preview_form_feeds.py b/tests/data/cases/preview_form_feeds.py new file mode 100644 index 00000000000..c236f177a95 --- /dev/null +++ b/tests/data/cases/preview_form_feeds.py @@ -0,0 +1,226 @@ +# flags: --preview + + +# Warning! This file contains form feeds (ASCII 0x0C, often represented by \f or ^L). +# These may be invisible in your editor: ensure you can see them before making changes here. + +# There's one at the start that'll get stripped + +# Comment and statement processing is different enough that we'll test variations of both +# contexts here + +# + + +# + + +# + + + +# + + + +# + + + +# + + +# + + + +# + +# + +# + + \ +# +pass + +pass + + +pass + + +pass + + + +pass + + + +pass + + + +pass + + +pass + + + +pass + +pass + +pass + + +# form feed after a dedent +def foo(): + pass + +pass + + +# form feeds are prohibited inside blocks, or on a line with nonwhitespace + def bar( a = 1 ,b : bool = False ) : + + + pass + + +class Baz: + + def __init__(self): + pass + + + def something(self): + pass + + + +# +pass +pass # +a = 1 + # + pass + a = 1 + +a = [ + +] + +# as internal whitespace of a comment is allowed but why +"form feed literal in a string is okay " + +# form feeds at the very end get removed. + + + +# output + +# Warning! This file contains form feeds (ASCII 0x0C, often represented by \f or ^L). +# These may be invisible in your editor: ensure you can see them before making changes here. + +# There's one at the start that'll get stripped + +# Comment and statement processing is different enough that we'll test variations of both +# contexts here + +# + + +# + + +# + + +# + + +# + + +# + + +# + + +# + +# + +# + +# +pass + +pass + + +pass + + +pass + + +pass + + +pass + + +pass + + +pass + + +pass + +pass + +pass + + +# form feed after a dedent +def foo(): + pass + + +pass + + +# form feeds are prohibited inside blocks, or on a line with nonwhitespace +def bar(a=1, b: bool = False): + + pass + + +class Baz: + def __init__(self): + pass + + def something(self): + pass + + +# +pass +pass # +a = 1 +# +pass +a = 1 + +a = [] + +# as internal whitespace of a comment is allowed but why +"form feed literal in a string is okay " + +# form feeds at the very end get removed. diff --git a/tests/data/preview/format_unicode_escape_seq.py b/tests/data/cases/preview_format_unicode_escape_seq.py similarity index 96% rename from tests/data/preview/format_unicode_escape_seq.py rename to tests/data/cases/preview_format_unicode_escape_seq.py index 3440696c303..65c3d8d166e 100644 --- a/tests/data/preview/format_unicode_escape_seq.py +++ b/tests/data/cases/preview_format_unicode_escape_seq.py @@ -1,3 +1,4 @@ +# flags: --preview x = "\x1F" x = "\\x1B" x = "\\\x1B" diff --git a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py new file mode 100644 index 00000000000..47a6a0bcae6 --- /dev/null +++ b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py @@ -0,0 +1,347 @@ +# flags: --preview +def foo_brackets(request): + return JsonResponse( + { + "var_1": foo, + "var_2": bar, + } + ) + +def foo_square_brackets(request): + return JsonResponse( + [ + "var_1", + "var_2", + ] + ) + +func({"a": 37, "b": 42, "c": 927, "aaaaaaaaaaaaaaaaaaaaaaaaa": 11111111111111111111111111111111111111111}) + +func(["random_string_number_one","random_string_number_two","random_string_number_three","random_string_number_four"]) + +func( + { + # expand me + 'a':37, + 'b':42, + 'c':927 + } +) + +func( + [ + 'a', + 'b', + 'c', + ] +) + +func( + [ + 'a', + 'b', + 'c', + ], +) + +func( # a + [ # b + "c", # c + "d", # d + "e", # e + ] # f +) # g + +func( # a + { # b + "c": 1, # c + "d": 2, # d + "e": 3, # e + } # f +) # g + +func( + # preserve me + [ + "c", + "d", + "e", + ] +) + +func( + [ # preserve me but hug brackets + "c", + "d", + "e", + ] +) + +func( + [ + # preserve me but hug brackets + "c", + "d", + "e", + ] +) + +func( + [ + "c", + # preserve me but hug brackets + "d", + "e", + ] +) + +func( + [ + "c", + "d", + "e", + # preserve me but hug brackets + ] +) + +func( + [ + "c", + "d", + "e", + ] # preserve me but hug brackets +) + +func( + [ + "c", + "d", + "e", + ] + # preserve me +) + +func([x for x in "short line"]) +func([x for x in "long line long line long line long line long line long line long line"]) +func([x for x in [x for x in "long line long line long line long line long line long line long line"]]) + +foooooooooooooooooooo( + [{c: n + 1 for c in range(256)} for n in range(100)] + [{}], {size} +) + +baaaaaaaaaaaaar( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], {x}, "a string", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +) + +nested_mapping = {"key": [{"a very long key 1": "with a very long value", "a very long key 2": "with a very long value"}]} +explicit_exploding = [[["short", "line",],],] +single_item_do_not_explode = Context({ + "version": get_docs_version(), +}) + +foo(*[str(i) for i in range(100000000000000000000000000000000000000000000000000000000000)]) + +foo( + **{ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": 1, + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb": 2, + "ccccccccccccccccccccccccccccccccc": 3, + **other, + } +) + +foo(**{x: y for x, y in enumerate(["long long long long line","long long long long line"])}) + +# Edge case when deciding whether to hug the brackets without inner content. +very_very_very_long_variable = very_very_very_long_module.VeryVeryVeryVeryLongClassName([[]]) + +for foo in ["a", "b"]: + output.extend([ + individual + for + # Foobar + container in xs_by_y[foo] + # Foobar + for individual in container["nested"] + ]) + +# output +def foo_brackets(request): + return JsonResponse({ + "var_1": foo, + "var_2": bar, + }) + + +def foo_square_brackets(request): + return JsonResponse([ + "var_1", + "var_2", + ]) + + +func({ + "a": 37, + "b": 42, + "c": 927, + "aaaaaaaaaaaaaaaaaaaaaaaaa": 11111111111111111111111111111111111111111, +}) + +func([ + "random_string_number_one", + "random_string_number_two", + "random_string_number_three", + "random_string_number_four", +]) + +func({ + # expand me + "a": 37, + "b": 42, + "c": 927, +}) + +func([ + "a", + "b", + "c", +]) + +func( + [ + "a", + "b", + "c", + ], +) + +func([ # a # b + "c", # c + "d", # d + "e", # e +]) # f # g + +func({ # a # b + "c": 1, # c + "d": 2, # d + "e": 3, # e +}) # f # g + +func( + # preserve me + [ + "c", + "d", + "e", + ] +) + +func([ # preserve me but hug brackets + "c", + "d", + "e", +]) + +func([ + # preserve me but hug brackets + "c", + "d", + "e", +]) + +func([ + "c", + # preserve me but hug brackets + "d", + "e", +]) + +func([ + "c", + "d", + "e", + # preserve me but hug brackets +]) + +func([ + "c", + "d", + "e", +]) # preserve me but hug brackets + +func( + [ + "c", + "d", + "e", + ] + # preserve me +) + +func([x for x in "short line"]) +func( + [x for x in "long line long line long line long line long line long line long line"] +) +func([ + x + for x in [ + x + for x in "long line long line long line long line long line long line long line" + ] +]) + +foooooooooooooooooooo( + [{c: n + 1 for c in range(256)} for n in range(100)] + [{}], {size} +) + +baaaaaaaaaaaaar( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], {x}, "a string", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +) + +nested_mapping = { + "key": [{ + "a very long key 1": "with a very long value", + "a very long key 2": "with a very long value", + }] +} +explicit_exploding = [ + [ + [ + "short", + "line", + ], + ], +] +single_item_do_not_explode = Context({ + "version": get_docs_version(), +}) + +foo(*[ + str(i) for i in range(100000000000000000000000000000000000000000000000000000000000) +]) + +foo(**{ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": 1, + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb": 2, + "ccccccccccccccccccccccccccccccccc": 3, + **other, +}) + +foo(**{ + x: y for x, y in enumerate(["long long long long line", "long long long long line"]) +}) + +# Edge case when deciding whether to hug the brackets without inner content. +very_very_very_long_variable = very_very_very_long_module.VeryVeryVeryVeryLongClassName( + [[]] +) + +for foo in ["a", "b"]: + output.extend([ + individual + for + # Foobar + container in xs_by_y[foo] + # Foobar + for individual in container["nested"] + ]) diff --git a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets_no_ll1.py b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets_no_ll1.py new file mode 100644 index 00000000000..fdebdf69c20 --- /dev/null +++ b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets_no_ll1.py @@ -0,0 +1,106 @@ +# flags: --preview --no-preview-line-length-1 +# split out from preview_hug_parens_with_brackes_and_square_brackets, as it produces +# different code on the second pass with line-length 1 in many cases. +# Seems to be about whether the last string in a sequence gets wrapped in parens or not. +foo(*["long long long long long line", "long long long long long line", "long long long long long line"]) +func({"short line"}) +func({"long line", "long long line", "long long long line", "long long long long line", "long long long long long line"}) +func({{"long line", "long long line", "long long long line", "long long long long line", "long long long long long line"}}) +func(("long line", "long long line", "long long long line", "long long long long line", "long long long long long line")) +func((("long line", "long long line", "long long long line", "long long long long line", "long long long long long line"))) +func([["long line", "long long line", "long long long line", "long long long long line", "long long long long long line"]]) + + +# Do not hug if the argument fits on a single line. +func({"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"}) +func(("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line")) +func(["fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"]) +func(**{"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit---"}) +func(*("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit----")) +array = [{"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"}] +array = [("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line")] +array = [["fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"]] + +nested_array = [[["long line", "long long line", "long long long line", "long long long long line", "long long long long long line"]]] + +# output + +# split out from preview_hug_parens_with_brackes_and_square_brackets, as it produces +# different code on the second pass with line-length 1 in many cases. +# Seems to be about whether the last string in a sequence gets wrapped in parens or not. +foo(*[ + "long long long long long line", + "long long long long long line", + "long long long long long line", +]) +func({"short line"}) +func({ + "long line", + "long long line", + "long long long line", + "long long long long line", + "long long long long long line", +}) +func({{ + "long line", + "long long line", + "long long long line", + "long long long long line", + "long long long long long line", +}}) +func(( + "long line", + "long long line", + "long long long line", + "long long long long line", + "long long long long long line", +)) +func((( + "long line", + "long long line", + "long long long line", + "long long long long line", + "long long long long long line", +))) +func([[ + "long line", + "long long line", + "long long long line", + "long long long long line", + "long long long long long line", +]]) + + +# Do not hug if the argument fits on a single line. +func( + {"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"} +) +func( + ("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line") +) +func( + ["fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"] +) +func( + **{"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit---"} +) +func( + *("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit----") +) +array = [ + {"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"} +] +array = [ + ("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line") +] +array = [ + ["fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"] +] + +nested_array = [[[ + "long line", + "long long line", + "long long long line", + "long long long long line", + "long long long long long line", +]]] diff --git a/tests/data/preview/long_dict_values.py b/tests/data/cases/preview_long_dict_values.py similarity index 99% rename from tests/data/preview/long_dict_values.py rename to tests/data/cases/preview_long_dict_values.py index 4c515180028..fbbacd13d1d 100644 --- a/tests/data/preview/long_dict_values.py +++ b/tests/data/cases/preview_long_dict_values.py @@ -1,3 +1,4 @@ +# flags: --preview my_dict = { "something_something": r"Lorem ipsum dolor sit amet, an sed convenire eloquentiam \t" diff --git a/tests/data/preview/long_strings.py b/tests/data/cases/preview_long_strings.py similarity index 85% rename from tests/data/preview/long_strings.py rename to tests/data/cases/preview_long_strings.py index b7a0a42f82a..19ac47a7032 100644 --- a/tests/data/preview/long_strings.py +++ b/tests/data/cases/preview_long_strings.py @@ -1,3 +1,4 @@ +# flags: --preview x = "This is a really long string that can't possibly be expected to fit all together on one line. In fact it may even take up three or more lines... like four or five... but probably just three." x += "This is a really long string that can't possibly be expected to fit all together on one line. In fact it may even take up three or more lines... like four or five... but probably just three." @@ -26,15 +27,15 @@ ("This is a really long string that can't be expected to fit in one line and is used as a dict's key"): ["value1", "value2"], } -L1 = ["The is a short string", "This is a really long string that can't possibly be expected to fit all together on one line. Also it is inside a list literal, so it's expected to be wrapped in parens when spliting to avoid implicit str concatenation.", short_call("arg", {"key": "value"}), "This is another really really (not really) long string that also can't be expected to fit on one line and is, like the other string, inside a list literal.", ("parens should be stripped for short string in list")] +L1 = ["The is a short string", "This is a really long string that can't possibly be expected to fit all together on one line. Also it is inside a list literal, so it's expected to be wrapped in parens when splitting to avoid implicit str concatenation.", short_call("arg", {"key": "value"}), "This is another really really (not really) long string that also can't be expected to fit on one line and is, like the other string, inside a list literal.", ("parens should be stripped for short string in list")] L2 = ["This is a really long string that can't be expected to fit in one line and is the only child of a list literal."] -S1 = {"The is a short string", "This is a really long string that can't possibly be expected to fit all together on one line. Also it is inside a set literal, so it's expected to be wrapped in parens when spliting to avoid implicit str concatenation.", short_call("arg", {"key": "value"}), "This is another really really (not really) long string that also can't be expected to fit on one line and is, like the other string, inside a set literal.", ("parens should be stripped for short string in set")} +S1 = {"The is a short string", "This is a really long string that can't possibly be expected to fit all together on one line. Also it is inside a set literal, so it's expected to be wrapped in parens when splitting to avoid implicit str concatenation.", short_call("arg", {"key": "value"}), "This is another really really (not really) long string that also can't be expected to fit on one line and is, like the other string, inside a set literal.", ("parens should be stripped for short string in set")} S2 = {"This is a really long string that can't be expected to fit in one line and is the only child of a set literal."} -T1 = ("The is a short string", "This is a really long string that can't possibly be expected to fit all together on one line. Also it is inside a tuple literal, so it's expected to be wrapped in parens when spliting to avoid implicit str concatenation.", short_call("arg", {"key": "value"}), "This is another really really (not really) long string that also can't be expected to fit on one line and is, like the other string, inside a tuple literal.", ("parens should be stripped for short string in list")) +T1 = ("The is a short string", "This is a really long string that can't possibly be expected to fit all together on one line. Also it is inside a tuple literal, so it's expected to be wrapped in parens when splitting to avoid implicit str concatenation.", short_call("arg", {"key": "value"}), "This is another really really (not really) long string that also can't be expected to fit on one line and is, like the other string, inside a tuple literal.", ("parens should be stripped for short string in list")) T2 = ("This is a really long string that can't be expected to fit in one line and is the only child of a tuple literal.",) @@ -304,6 +305,29 @@ def foo(): # Test case of an outer string' parens enclose an inner string's parens. call(body=("%s %s" % ((",".join(items)), suffix))) +log.info(f'Skipping: {desc["db_id"]=} {desc["ms_name"]} {money=} {dte=} {pos_share=} {desc["status"]=} {desc["exposure_max"]=}') + +log.info(f"Skipping: {desc['db_id']=} {desc['ms_name']} {money=} {dte=} {pos_share=} {desc['status']=} {desc['exposure_max']=}") + +log.info(f'Skipping: {desc["db_id"]} {foo("bar",x=123)} {"foo" != "bar"} {(x := "abc=")} {pos_share=} {desc["status"]} {desc["exposure_max"]}') + +log.info(f'Skipping: {desc["db_id"]} {desc["ms_name"]} {money=} {(x := "abc=")=} {pos_share=} {desc["status"]} {desc["exposure_max"]}') + +log.info(f'Skipping: {desc["db_id"]} {foo("bar",x=123)=} {money=} {dte=} {pos_share=} {desc["status"]} {desc["exposure_max"]}') + +log.info(f'Skipping: {foo("asdf")=} {desc["ms_name"]} {money=} {dte=} {pos_share=} {desc["status"]} {desc["exposure_max"]}') + +log.info(f'Skipping: {"a" == "b" == "c" == "d"} {desc["ms_name"]} {money=} {dte=} {pos_share=} {desc["status"]} {desc["exposure_max"]}') + +log.info(f'Skipping: {"a" == "b" == "c" == "d"=} {desc["ms_name"]} {money=} {dte=} {pos_share=} {desc["status"]} {desc["exposure_max"]}') + +log.info(f'Skipping: { longer_longer_longer_longer_longer_longer_name [ "db_id" ] [ "another_key" ] = : .3f }') + +log.info(f'''Skipping: {"a" == 'b'} {desc["ms_name"]} {money=} {dte=} {pos_share=} {desc["status"]} {desc["exposure_max"]}''') + +log.info(f'''Skipping: {'a' == "b"=} {desc["ms_name"]} {money=} {dte=} {pos_share=} {desc["status"]} {desc["exposure_max"]}''') + +log.info(f"""Skipping: {'a' == 'b'} {desc['ms_name']} {money=} {dte=} {pos_share=} {desc['status']} {desc['exposure_max']}""") # output @@ -323,10 +347,8 @@ def foo(): y = "Short string" print( - ( - "This is a really long string inside of a print statement with extra arguments" - " attached at the end of it." - ), + "This is a really long string inside of a print statement with extra arguments" + " attached at the end of it.", x, y, z, @@ -410,7 +432,7 @@ def foo(): ( "This is a really long string that can't possibly be expected to fit all" " together on one line. Also it is inside a list literal, so it's expected to" - " be wrapped in parens when spliting to avoid implicit str concatenation." + " be wrapped in parens when splitting to avoid implicit str concatenation." ), short_call("arg", {"key": "value"}), ( @@ -431,7 +453,7 @@ def foo(): ( "This is a really long string that can't possibly be expected to fit all" " together on one line. Also it is inside a set literal, so it's expected to be" - " wrapped in parens when spliting to avoid implicit str concatenation." + " wrapped in parens when splitting to avoid implicit str concatenation." ), short_call("arg", {"key": "value"}), ( @@ -452,7 +474,7 @@ def foo(): ( "This is a really long string that can't possibly be expected to fit all" " together on one line. Also it is inside a tuple literal, so it's expected to" - " be wrapped in parens when spliting to avoid implicit str concatenation." + " be wrapped in parens when splitting to avoid implicit str concatenation." ), short_call("arg", {"key": "value"}), ( @@ -501,15 +523,13 @@ def foo(): ) bad_split_func1( - ( - "But what should happen when code has already " - "been formatted but in the wrong way? Like " - "with a space at the end instead of the " - "beginning. Or what about when it is split too " - "soon? In the case of a split that is too " - "short, black will try to honer the custom " - "split." - ), + "But what should happen when code has already " + "been formatted but in the wrong way? Like " + "with a space at the end instead of the " + "beginning. Or what about when it is split too " + "soon? In the case of a split that is too " + "short, black will try to honer the custom " + "split.", xxx, yyy, zzz, @@ -612,11 +632,9 @@ def foo(): ) arg_comment_string = print( - ( # This comment gets thrown to the top. - "Long lines with inline comments which are apart of (and not the only member" - " of) an argument list should have their comments appended to the reformatted" - " string's enclosing left parentheses." - ), + "Long lines with inline comments which are apart of (and not the only member of) an" + " argument list should have their comments appended to the reformatted string's" + " enclosing left parentheses.", # This comment gets thrown to the top. "Arg #2", "Arg #3", "Arg #4", @@ -676,31 +694,23 @@ def foo(): ) func_with_bad_comma( - ( - "This is a really long string argument to a function that has a trailing comma" - " which should NOT be there." - ), + "This is a really long string argument to a function that has a trailing comma" + " which should NOT be there.", ) func_with_bad_comma( - ( # comment after comma - "This is a really long string argument to a function that has a trailing comma" - " which should NOT be there." - ), + "This is a really long string argument to a function that has a trailing comma" + " which should NOT be there.", # comment after comma ) func_with_bad_comma( - ( - "This is a really long string argument to a function that has a trailing comma" - " which should NOT be there." - ), + "This is a really long string argument to a function that has a trailing comma" + " which should NOT be there.", ) func_with_bad_comma( - ( # comment after comma - "This is a really long string argument to a function that has a trailing comma" - " which should NOT be there." - ), + "This is a really long string argument to a function that has a trailing comma" + " which should NOT be there.", # comment after comma ) func_with_bad_parens_that_wont_fit_in_one_line( @@ -859,3 +869,60 @@ def foo(): # Test case of an outer string' parens enclose an inner string's parens. call(body="%s %s" % (",".join(items), suffix)) + +log.info( + "Skipping:" + f' {desc["db_id"]=} {desc["ms_name"]} {money=} {dte=} {pos_share=} {desc["status"]=} {desc["exposure_max"]=}' +) + +log.info( + "Skipping:" + f" {desc['db_id']=} {desc['ms_name']} {money=} {dte=} {pos_share=} {desc['status']=} {desc['exposure_max']=}" +) + +log.info( + "Skipping:" + f" {desc['db_id']} {foo('bar',x=123)} {'foo' != 'bar'} {(x := 'abc=')} {pos_share=} {desc['status']} {desc['exposure_max']}" +) + +log.info( + "Skipping:" + f' {desc["db_id"]} {desc["ms_name"]} {money=} {(x := "abc=")=} {pos_share=} {desc["status"]} {desc["exposure_max"]}' +) + +log.info( + "Skipping:" + f' {desc["db_id"]} {foo("bar",x=123)=} {money=} {dte=} {pos_share=} {desc["status"]} {desc["exposure_max"]}' +) + +log.info( + "Skipping:" + f' {foo("asdf")=} {desc["ms_name"]} {money=} {dte=} {pos_share=} {desc["status"]} {desc["exposure_max"]}' +) + +log.info( + "Skipping:" + f" {'a' == 'b' == 'c' == 'd'} {desc['ms_name']} {money=} {dte=} {pos_share=} {desc['status']} {desc['exposure_max']}" +) + +log.info( + "Skipping:" + f' {"a" == "b" == "c" == "d"=} {desc["ms_name"]} {money=} {dte=} {pos_share=} {desc["status"]} {desc["exposure_max"]}' +) + +log.info( + "Skipping:" + f' { longer_longer_longer_longer_longer_longer_name [ "db_id" ] [ "another_key" ] = : .3f }' +) + +log.info( + f"""Skipping: {"a" == 'b'} {desc["ms_name"]} {money=} {dte=} {pos_share=} {desc["status"]} {desc["exposure_max"]}""" +) + +log.info( + f"""Skipping: {'a' == "b"=} {desc["ms_name"]} {money=} {dte=} {pos_share=} {desc["status"]} {desc["exposure_max"]}""" +) + +log.info( + f"""Skipping: {'a' == 'b'} {desc['ms_name']} {money=} {dte=} {pos_share=} {desc['status']} {desc['exposure_max']}""" +) diff --git a/tests/data/cases/preview_long_strings__east_asian_width.py b/tests/data/cases/preview_long_strings__east_asian_width.py new file mode 100644 index 00000000000..d190f422a60 --- /dev/null +++ b/tests/data/cases/preview_long_strings__east_asian_width.py @@ -0,0 +1,26 @@ +# flags: --preview +# The following strings do not have not-so-many chars, but are long enough +# when these are rendered in a monospace font (if the renderer respects +# Unicode East Asian Width properties). +hangul = '코드포인트 수는 적으나 실제 터미널이나 에디터에서 렌더링될 땐 너무 길어서 줄바꿈이 필요한 문자열' +hanzi = '中文測試：代碼點數量少，但在真正的終端模擬器或編輯器中呈現時太長，因此需要換行的字符串。' +japanese = 'コードポイントの数は少ないが、実際の端末エミュレータやエディタでレンダリングされる時は長すぎる為、改行が要る文字列' + +# output + +# The following strings do not have not-so-many chars, but are long enough +# when these are rendered in a monospace font (if the renderer respects +# Unicode East Asian Width properties). +hangul = ( + "코드포인트 수는 적으나 실제 터미널이나 에디터에서 렌더링될 땐 너무 길어서 줄바꿈이" + " 필요한 문자열" +) +hanzi = ( + "中文測試：代碼點數量少，但在真正的終端模擬器或編輯器中呈現時太長，" + "因此需要換行的字符串。" +) +japanese = ( + "コードポイントの数は少ないが、" + "実際の端末エミュレータやエディタでレンダリングされる時は長すぎる為、" + "改行が要る文字列" +) diff --git a/tests/data/preview/long_strings__edge_case.py b/tests/data/cases/preview_long_strings__edge_case.py similarity index 99% rename from tests/data/preview/long_strings__edge_case.py rename to tests/data/cases/preview_long_strings__edge_case.py index 2bc0b6ed328..a8e8971968c 100644 --- a/tests/data/preview/long_strings__edge_case.py +++ b/tests/data/cases/preview_long_strings__edge_case.py @@ -1,3 +1,4 @@ +# flags: --preview some_variable = "This string is long but not so long that it needs to be split just yet" some_variable = 'This string is long but not so long that it needs to be split just yet' some_variable = "This string is long, just long enough that it needs to be split, u get?" diff --git a/tests/data/preview/long_strings__regression.py b/tests/data/cases/preview_long_strings__regression.py similarity index 96% rename from tests/data/preview/long_strings__regression.py rename to tests/data/cases/preview_long_strings__regression.py index eead8c204a9..5e76a8cf61c 100644 --- a/tests/data/preview/long_strings__regression.py +++ b/tests/data/cases/preview_long_strings__regression.py @@ -1,3 +1,4 @@ +# flags: --preview class A: def foo(): result = type(message)("") @@ -209,8 +210,8 @@ def foo(): some_tuple = ("some string", "some string" " which should be joined") -some_commented_string = ( - "This string is long but not so long that it needs hahahah toooooo be so greatttt" # This comment gets thrown to the top. +some_commented_string = ( # This comment stays at the top. + "This string is long but not so long that it needs hahahah toooooo be so greatttt" " {} that I just can't think of any more good words to say about it at" " allllllllllll".format("ha") # comments here are fine ) @@ -610,14 +611,13 @@ def foo(): class A: def foo(): - XXXXXXXXXXXX.append( - ( - "xxx_xxxxxxxxxx(xxxxx={}, xxxx={}, xxxxx, xxxx_xxxx_xxxxxxxxxx={})" - .format(xxxxx, xxxx, xxxx_xxxx_xxxxxxxxxx), - my_var, - my_other_var, - ) - ) + XXXXXXXXXXXX.append(( + "xxx_xxxxxxxxxx(xxxxx={}, xxxx={}, xxxxx, xxxx_xxxx_xxxxxxxxxx={})".format( + xxxxx, xxxx, xxxx_xxxx_xxxxxxxxxx + ), + my_var, + my_other_var, + )) class A: @@ -715,11 +715,9 @@ class A: def foo(): some_func_call( "xxxxxxxxxx", - ( - "xx {xxxxxxxxxxx}/xxxxxxxxxxx.xxx xxxx.xxx && xxxxxx -x " - '"xxxx xxxxxxx xxxxxx xxxx; xxxx xxxxxx_xxxxx xxxxxx xxxx; ' - "xxxx.xxxx_xxxxxx(['xxxx.xxx'], xxxx.xxxxxxx().xxxxxxxxxx)\" " - ), + "xx {xxxxxxxxxxx}/xxxxxxxxxxx.xxx xxxx.xxx && xxxxxx -x " + '"xxxx xxxxxxx xxxxxx xxxx; xxxx xxxxxx_xxxxx xxxxxx xxxx; ' + "xxxx.xxxx_xxxxxx(['xxxx.xxx'], xxxx.xxxxxxx().xxxxxxxxxx)\" ", None, ("xxxxxxxxxxx",), ), @@ -728,11 +726,9 @@ def foo(): class A: def foo(): some_func_call( - ( - "xx {xxxxxxxxxxx}/xxxxxxxxxxx.xxx xxxx.xxx && xxxxxx -x " - "xxxx, ('xxxxxxx xxxxxx xxxx, xxxx') xxxxxx_xxxxx xxxxxx xxxx; " - "xxxx.xxxx_xxxxxx(['xxxx.xxx'], xxxx.xxxxxxx().xxxxxxxxxx)\" " - ), + "xx {xxxxxxxxxxx}/xxxxxxxxxxx.xxx xxxx.xxx && xxxxxx -x " + "xxxx, ('xxxxxxx xxxxxx xxxx, xxxx') xxxxxx_xxxxx xxxxxx xxxx; " + "xxxx.xxxx_xxxxxx(['xxxx.xxx'], xxxx.xxxxxxx().xxxxxxxxxx)\" ", None, ("xxxxxxxxxxx",), ), @@ -837,7 +833,7 @@ def foo(): some_tuple = ("some string", "some string which should be joined") -some_commented_string = ( # This comment gets thrown to the top. +some_commented_string = ( # This comment stays at the top. "This string is long but not so long that it needs hahahah toooooo be so greatttt" " {} that I just can't think of any more good words to say about it at" " allllllllllll".format("ha") # comments here are fine @@ -850,10 +846,8 @@ def foo(): ) lpar_and_rpar_have_comments = func_call( # LPAR Comment - ( # Comma Comment - "Long really ridiculous type of string that shouldn't really even exist at all." - " I mean commmme onnn!!!" - ), + "Long really ridiculous type of string that shouldn't really even exist at all. I" + " mean commmme onnn!!!", # Comma Comment ) # RPAR Comment cmd_fstring = ( @@ -967,19 +961,17 @@ def who(self): ) -xxxxxxx_xxxxxx_xxxxxxx = xxx( - [ - xxxxxxxxxxxx( - xxxxxx_xxxxxxx=( - '((x.aaaaaaaaa = "xxxxxx.xxxxxxxxxxxxxxxxxxxxx") || (x.xxxxxxxxx =' - ' "xxxxxxxxxxxx")) && ' - # xxxxx xxxxxxxxxxxx xxxx xxx (xxxxxxxxxxxxxxxx) xx x xxxxxxxxx xx xxxxxx. - "(x.bbbbbbbbbbbb.xxx != " - '"xxx:xxx:xxx::cccccccccccc:xxxxxxx-xxxx/xxxxxxxxxxx/xxxxxxxxxxxxxxxxx") && ' - ) +xxxxxxx_xxxxxx_xxxxxxx = xxx([ + xxxxxxxxxxxx( + xxxxxx_xxxxxxx=( + '((x.aaaaaaaaa = "xxxxxx.xxxxxxxxxxxxxxxxxxxxx") || (x.xxxxxxxxx =' + ' "xxxxxxxxxxxx")) && ' + # xxxxx xxxxxxxxxxxx xxxx xxx (xxxxxxxxxxxxxxxx) xx x xxxxxxxxx xx xxxxxx. + "(x.bbbbbbbbbbbb.xxx != " + '"xxx:xxx:xxx::cccccccccccc:xxxxxxx-xxxx/xxxxxxxxxxx/xxxxxxxxxxxxxxxxx") && ' ) - ] -) + ) +]) if __name__ == "__main__": for i in range(4, 8): diff --git a/tests/data/preview/long_strings__type_annotations.py b/tests/data/cases/preview_long_strings__type_annotations.py similarity index 95% rename from tests/data/preview/long_strings__type_annotations.py rename to tests/data/cases/preview_long_strings__type_annotations.py index 41d7ee2b67b..8beb877bdd1 100644 --- a/tests/data/preview/long_strings__type_annotations.py +++ b/tests/data/cases/preview_long_strings__type_annotations.py @@ -1,3 +1,4 @@ +# flags: --preview def func( arg1, arg2, @@ -54,6 +55,6 @@ def func( def func( - argument: ("int |" "str"), + argument: "int |" "str", ) -> Set["int |" " str"]: pass diff --git a/tests/data/cases/preview_multiline_strings.py b/tests/data/cases/preview_multiline_strings.py new file mode 100644 index 00000000000..3ff643610b7 --- /dev/null +++ b/tests/data/cases/preview_multiline_strings.py @@ -0,0 +1,387 @@ +# flags: --preview +"""cow +say""", +call(3, "dogsay", textwrap.dedent("""dove + coo""" % "cowabunga")) +call(3, "dogsay", textwrap.dedent("""dove +coo""" % "cowabunga")) +call(3, textwrap.dedent("""cow + moo""" % "cowabunga"), "dogsay") +call(3, "dogsay", textwrap.dedent("""crow + caw""" % "cowabunga"),) +call(3, textwrap.dedent("""cat + meow""" % "cowabunga"), {"dog", "say"}) +call(3, {"dog", "say"}, textwrap.dedent("""horse + neigh""" % "cowabunga")) +call(3, {"dog", "say"}, textwrap.dedent("""pig + oink""" % "cowabunga"),) +textwrap.dedent("""A one-line triple-quoted string.""") +textwrap.dedent("""A two-line triple-quoted string +since it goes to the next line.""") +textwrap.dedent("""A three-line triple-quoted string +that not only goes to the next line +but also goes one line beyond.""") +textwrap.dedent("""\ + A triple-quoted string + actually leveraging the textwrap.dedent functionality + that ends in a trailing newline, + representing e.g. file contents. +""") +path.write_text(textwrap.dedent("""\ + A triple-quoted string + actually leveraging the textwrap.dedent functionality + that ends in a trailing newline, + representing e.g. file contents. +""")) +path.write_text(textwrap.dedent("""\ + A triple-quoted string + actually leveraging the textwrap.dedent functionality + that ends in a trailing newline, + representing e.g. {config_filename} file contents. +""".format("config_filename", config_filename))) +# Another use case +data = yaml.load("""\ +a: 1 +b: 2 +""") +data = yaml.load("""\ +a: 1 +b: 2 +""",) +data = yaml.load( + """\ + a: 1 + b: 2 +""" +) + +MULTILINE = """ +foo +""".replace("\n", "") +generated_readme = lambda project_name: """ +{} + + +""".strip().format(project_name) +parser.usage += """ +Custom extra help summary. + +Extra test: +- with +- bullets +""" + + +def get_stuff(cr, value): + # original + cr.execute(""" + SELECT whatever + FROM some_table t + WHERE id = %s + """, [value]) + return cr.fetchone() + + +def get_stuff(cr, value): + # preferred + cr.execute( + """ + SELECT whatever + FROM some_table t + WHERE id = %s + """, + [value], + ) + return cr.fetchone() + + +call(arg1, arg2, """ +short +""", arg3=True) +test_vectors = [ + "one-liner\n", + "two\nliner\n", + """expressed +as a three line +mulitline string""", +] + +_wat = re.compile( + r""" + regex + """, + re.MULTILINE | re.VERBOSE, +) +dis_c_instance_method = """\ +%3d 0 LOAD_FAST 1 (x) + 2 LOAD_CONST 1 (1) + 4 COMPARE_OP 2 (==) + 6 LOAD_FAST 0 (self) + 8 STORE_ATTR 0 (x) + 10 LOAD_CONST 0 (None) + 12 RETURN_VALUE +""" % (_C.__init__.__code__.co_firstlineno + 1,) +path.write_text(textwrap.dedent("""\ + A triple-quoted string + actually {verb} the textwrap.dedent functionality + that ends in a trailing newline, + representing e.g. {file_type} file contents. +""".format(verb="using", file_type="test"))) +{"""cow +moos"""} +["""cow +moos"""] +["""cow +moos""", """dog +woofs +and +barks"""] +def dastardly_default_value( + cow: String = json.loads("""this +is +quite +the +dastadardly +value!"""), + **kwargs, +): + pass + +print(f""" + This {animal} + moos and barks +{animal} say +""") +msg = f"""The arguments {bad_arguments} were passed in. +Please use `--build-option` instead, +`--global-option` is reserved to flags like `--verbose` or `--quiet`. +""" + +this_will_become_one_line = ( + "a" + "b" + "c" +) + +this_will_stay_on_three_lines = ( + "a" # comment + "b" + "c" +) + +this_will_also_become_one_line = ( # comment + "a" + "b" + "c" +) + +# output +"""cow +say""", +call( + 3, + "dogsay", + textwrap.dedent("""dove + coo""" % "cowabunga"), +) +call( + 3, + "dogsay", + textwrap.dedent("""dove +coo""" % "cowabunga"), +) +call( + 3, + textwrap.dedent("""cow + moo""" % "cowabunga"), + "dogsay", +) +call( + 3, + "dogsay", + textwrap.dedent("""crow + caw""" % "cowabunga"), +) +call( + 3, + textwrap.dedent("""cat + meow""" % "cowabunga"), + {"dog", "say"}, +) +call( + 3, + {"dog", "say"}, + textwrap.dedent("""horse + neigh""" % "cowabunga"), +) +call( + 3, + {"dog", "say"}, + textwrap.dedent("""pig + oink""" % "cowabunga"), +) +textwrap.dedent("""A one-line triple-quoted string.""") +textwrap.dedent("""A two-line triple-quoted string +since it goes to the next line.""") +textwrap.dedent("""A three-line triple-quoted string +that not only goes to the next line +but also goes one line beyond.""") +textwrap.dedent("""\ + A triple-quoted string + actually leveraging the textwrap.dedent functionality + that ends in a trailing newline, + representing e.g. file contents. +""") +path.write_text(textwrap.dedent("""\ + A triple-quoted string + actually leveraging the textwrap.dedent functionality + that ends in a trailing newline, + representing e.g. file contents. +""")) +path.write_text(textwrap.dedent("""\ + A triple-quoted string + actually leveraging the textwrap.dedent functionality + that ends in a trailing newline, + representing e.g. {config_filename} file contents. +""".format("config_filename", config_filename))) +# Another use case +data = yaml.load("""\ +a: 1 +b: 2 +""") +data = yaml.load( + """\ +a: 1 +b: 2 +""", +) +data = yaml.load("""\ + a: 1 + b: 2 +""") + +MULTILINE = """ +foo +""".replace("\n", "") +generated_readme = lambda project_name: """ +{} + + +""".strip().format(project_name) +parser.usage += """ +Custom extra help summary. + +Extra test: +- with +- bullets +""" + + +def get_stuff(cr, value): + # original + cr.execute( + """ + SELECT whatever + FROM some_table t + WHERE id = %s + """, + [value], + ) + return cr.fetchone() + + +def get_stuff(cr, value): + # preferred + cr.execute( + """ + SELECT whatever + FROM some_table t + WHERE id = %s + """, + [value], + ) + return cr.fetchone() + + +call( + arg1, + arg2, + """ +short +""", + arg3=True, +) +test_vectors = [ + "one-liner\n", + "two\nliner\n", + """expressed +as a three line +mulitline string""", +] + +_wat = re.compile( + r""" + regex + """, + re.MULTILINE | re.VERBOSE, +) +dis_c_instance_method = """\ +%3d 0 LOAD_FAST 1 (x) + 2 LOAD_CONST 1 (1) + 4 COMPARE_OP 2 (==) + 6 LOAD_FAST 0 (self) + 8 STORE_ATTR 0 (x) + 10 LOAD_CONST 0 (None) + 12 RETURN_VALUE +""" % (_C.__init__.__code__.co_firstlineno + 1,) +path.write_text(textwrap.dedent("""\ + A triple-quoted string + actually {verb} the textwrap.dedent functionality + that ends in a trailing newline, + representing e.g. {file_type} file contents. +""".format(verb="using", file_type="test"))) +{"""cow +moos"""} +["""cow +moos"""] +[ + """cow +moos""", + """dog +woofs +and +barks""", +] + + +def dastardly_default_value( + cow: String = json.loads("""this +is +quite +the +dastadardly +value!"""), + **kwargs, +): + pass + + +print(f""" + This {animal} + moos and barks +{animal} say +""") +msg = f"""The arguments {bad_arguments} were passed in. +Please use `--build-option` instead, +`--global-option` is reserved to flags like `--verbose` or `--quiet`. +""" + +this_will_become_one_line = "abc" + +this_will_stay_on_three_lines = ( + "a" # comment + "b" + "c" +) + +this_will_also_become_one_line = "abc" # comment diff --git a/tests/data/cases/preview_no_blank_line_before_docstring.py b/tests/data/cases/preview_no_blank_line_before_docstring.py new file mode 100644 index 00000000000..303035a7efb --- /dev/null +++ b/tests/data/cases/preview_no_blank_line_before_docstring.py @@ -0,0 +1,59 @@ +# flags: --preview +def line_before_docstring(): + + """Please move me up""" + + +class LineBeforeDocstring: + + """Please move me up""" + + +class EvenIfThereIsAMethodAfter: + + """I'm the docstring""" + def method(self): + pass + + +class TwoLinesBeforeDocstring: + + + """I want to be treated the same as if I were closer""" + + +class MultilineDocstringsAsWell: + + """I'm so far + + and on so many lines... + """ + + +# output + + +def line_before_docstring(): + """Please move me up""" + + +class LineBeforeDocstring: + """Please move me up""" + + +class EvenIfThereIsAMethodAfter: + """I'm the docstring""" + + def method(self): + pass + + +class TwoLinesBeforeDocstring: + """I want to be treated the same as if I were closer""" + + +class MultilineDocstringsAsWell: + """I'm so far + + and on so many lines... + """ diff --git a/tests/data/cases/preview_pattern_matching_long.py b/tests/data/cases/preview_pattern_matching_long.py new file mode 100644 index 00000000000..df849fdc4f2 --- /dev/null +++ b/tests/data/cases/preview_pattern_matching_long.py @@ -0,0 +1,34 @@ +# flags: --preview --minimum-version=3.10 +match x: + case "abcd" | "abcd" | "abcd" : + pass + case "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd": + pass + case xxxxxxxxxxxxxxxxxxxxxxx: + pass + +# output + +match x: + case "abcd" | "abcd" | "abcd": + pass + case ( + "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + ): + pass + case xxxxxxxxxxxxxxxxxxxxxxx: + pass diff --git a/tests/data/cases/preview_pattern_matching_trailing_comma.py b/tests/data/cases/preview_pattern_matching_trailing_comma.py new file mode 100644 index 00000000000..e6c0d88bb80 --- /dev/null +++ b/tests/data/cases/preview_pattern_matching_trailing_comma.py @@ -0,0 +1,39 @@ +# flags: --preview --minimum-version=3.10 +match maybe, multiple: + case perhaps, 5: + pass + case perhaps, 6,: + pass + + +match more := (than, one), indeed,: + case _, (5, 6): + pass + case [[5], (6)], [7],: + pass + case _: + pass + + +# output + +match maybe, multiple: + case perhaps, 5: + pass + case ( + perhaps, + 6, + ): + pass + + +match more := (than, one), indeed,: + case _, (5, 6): + pass + case ( + [[5], (6)], + [7], + ): + pass + case _: + pass \ No newline at end of file diff --git a/tests/data/cases/preview_pep_572.py b/tests/data/cases/preview_pep_572.py new file mode 100644 index 00000000000..8e801ff6cdc --- /dev/null +++ b/tests/data/cases/preview_pep_572.py @@ -0,0 +1,7 @@ +# flags: --preview +x[(a:=0):] +x[:(a:=0)] + +# output +x[(a := 0):] +x[:(a := 0)] diff --git a/tests/data/preview/percent_precedence.py b/tests/data/cases/preview_percent_precedence.py similarity index 96% rename from tests/data/preview/percent_precedence.py rename to tests/data/cases/preview_percent_precedence.py index b895443fb46..aeaf450ff5e 100644 --- a/tests/data/preview/percent_precedence.py +++ b/tests/data/cases/preview_percent_precedence.py @@ -1,3 +1,4 @@ +# flags: --preview ("" % a) ** 2 ("" % a)[0] ("" % a)() diff --git a/tests/data/cases/preview_power_op_spacing.py b/tests/data/cases/preview_power_op_spacing.py new file mode 100644 index 00000000000..650c6fecb20 --- /dev/null +++ b/tests/data/cases/preview_power_op_spacing.py @@ -0,0 +1,97 @@ +# flags: --preview +a = 1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1 +b = 1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1 +c = 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 +d = 1**1 ** 1**1 ** 1**1 ** 1**1 ** 1**1**1 ** 1 ** 1**1 ** 1**1**1**1**1 ** 1 ** 1**1**1 **1**1** 1 ** 1 ** 1 +e = 𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟 +f = 𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟 + +a = 1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0 +b = 1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0 +c = 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 +d = 1.0**1.0 ** 1.0**1.0 ** 1.0**1.0 ** 1.0**1.0 ** 1.0**1.0**1.0 ** 1.0 ** 1.0**1.0 ** 1.0**1.0**1.0 + +# output +a = 1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1 +b = ( + 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 +) +c = 1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1 +d = 1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1 +e = 𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟 +f = ( + 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 +) + +a = 1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0 +b = ( + 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 +) +c = 1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0 +d = 1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0 diff --git a/tests/data/preview/prefer_rhs_split.py b/tests/data/cases/preview_prefer_rhs_split.py similarity index 74% rename from tests/data/preview/prefer_rhs_split.py rename to tests/data/cases/preview_prefer_rhs_split.py index 2f3cf33db41..28d89c368c0 100644 --- a/tests/data/preview/prefer_rhs_split.py +++ b/tests/data/cases/preview_prefer_rhs_split.py @@ -1,3 +1,4 @@ +# flags: --preview first_item, second_item = ( some_looooooooong_module.some_looooooooooooooong_function_name( first_argument, second_argument, third_argument @@ -69,7 +70,7 @@ ] = lambda obj: obj.some_long_named_method() -# Make when when the left side of assignement plus the opening paren "... = (" is +# Make when when the left side of assignment plus the opening paren "... = (" is # exactly line length limit + 1, it won't be split like that. xxxxxxxxx_yyy_zzzzzzzz[ xx.xxxxxx(x_yyy_zzzzzz.xxxxx[0]), x_yyy_zzzzzz.xxxxxx(xxxx=1) @@ -83,3 +84,24 @@ ) or ( isinstance(some_other_var, BaseClass) and table.something != table.some_other_thing ) + +# Multiple targets +a = b = ( + ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc +) + +a = b = c = d = e = f = g = ( + hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh +) = i = j = ( + kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +) + +a = ( + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +) = c + +a = ( + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +) = ( + cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc +) = ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd diff --git a/tests/data/cases/preview_return_annotation_brackets_string.py b/tests/data/cases/preview_return_annotation_brackets_string.py new file mode 100644 index 00000000000..fea0ea6839a --- /dev/null +++ b/tests/data/cases/preview_return_annotation_brackets_string.py @@ -0,0 +1,24 @@ +# flags: --preview +# Long string example +def frobnicate() -> "ThisIsTrulyUnreasonablyExtremelyLongClassName | list[ThisIsTrulyUnreasonablyExtremelyLongClassName]": + pass + +# splitting the string breaks if there's any parameters +def frobnicate(a) -> "ThisIsTrulyUnreasonablyExtremelyLongClassName | list[ThisIsTrulyUnreasonablyExtremelyLongClassName]": + pass + +# output + +# Long string example +def frobnicate() -> ( + "ThisIsTrulyUnreasonablyExtremelyLongClassName |" + " list[ThisIsTrulyUnreasonablyExtremelyLongClassName]" +): + pass + + +# splitting the string breaks if there's any parameters +def frobnicate( + a, +) -> "ThisIsTrulyUnreasonablyExtremelyLongClassName | list[ThisIsTrulyUnreasonablyExtremelyLongClassName]": + pass diff --git a/tests/data/cases/preview_single_line_format_skip_with_multiple_comments.py b/tests/data/cases/preview_single_line_format_skip_with_multiple_comments.py new file mode 100644 index 00000000000..efde662baa8 --- /dev/null +++ b/tests/data/cases/preview_single_line_format_skip_with_multiple_comments.py @@ -0,0 +1,20 @@ +# flags: --preview +foo = 123 # fmt: skip # noqa: E501 # pylint +bar = ( + 123 , + ( 1 + 5 ) # pylint # fmt:skip +) +baz = "a" + "b" # pylint; fmt: skip; noqa: E501 +skip_will_not_work = "a" + "b" # pylint fmt:skip +skip_will_not_work2 = "a" + "b" # some text; fmt:skip happens to be part of it + +# output + +foo = 123 # fmt: skip # noqa: E501 # pylint +bar = ( + 123 , + ( 1 + 5 ) # pylint # fmt:skip +) +baz = "a" + "b" # pylint; fmt: skip; noqa: E501 +skip_will_not_work = "a" + "b" # pylint fmt:skip +skip_will_not_work2 = "a" + "b" # some text; fmt:skip happens to be part of it diff --git a/tests/data/cases/preview_trailing_comma.py b/tests/data/cases/preview_trailing_comma.py new file mode 100644 index 00000000000..bba7e7ad16d --- /dev/null +++ b/tests/data/cases/preview_trailing_comma.py @@ -0,0 +1,56 @@ +# flags: --preview +e = { + "a": fun(msg, "ts"), + "longggggggggggggggid": ..., + "longgggggggggggggggggggkey": ..., "created": ... + # "longkey": ... +} +f = [ + arg1, + arg2, + arg3, arg4 + # comment +] +g = ( + arg1, + arg2, + arg3, arg4 + # comment +) +h = { + arg1, + arg2, + arg3, arg4 + # comment +} + +# output + +e = { + "a": fun(msg, "ts"), + "longggggggggggggggid": ..., + "longgggggggggggggggggggkey": ..., + "created": ..., + # "longkey": ... +} +f = [ + arg1, + arg2, + arg3, + arg4, + # comment +] +g = ( + arg1, + arg2, + arg3, + arg4, + # comment +) +h = { + arg1, + arg2, + arg3, + arg4, + # comment +} diff --git a/tests/data/cases/py310_pep572.py b/tests/data/cases/py310_pep572.py new file mode 100644 index 00000000000..172be3898d6 --- /dev/null +++ b/tests/data/cases/py310_pep572.py @@ -0,0 +1,13 @@ +# flags: --preview --minimum-version=3.10 +x[a:=0] +x[a := 0] +x[a := 0, b := 1] +x[5, b := 0] +x[a:=0,b:=1] + +# output +x[a := 0] +x[a := 0] +x[a := 0, b := 1] +x[5, b := 0] +x[a := 0, b := 1] diff --git a/tests/data/py_37/python37.py b/tests/data/cases/python37.py similarity index 95% rename from tests/data/py_37/python37.py rename to tests/data/cases/python37.py index dab8b404a73..3f61106c45d 100644 --- a/tests/data/py_37/python37.py +++ b/tests/data/cases/python37.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.7 +# flags: --minimum-version=3.7 def f(): @@ -33,9 +33,6 @@ def make_arange(n): # output -#!/usr/bin/env python3.7 - - def f(): return (i * 2 async for i in arange(42)) diff --git a/tests/data/py_38/python38.py b/tests/data/cases/python38.py similarity index 93% rename from tests/data/py_38/python38.py rename to tests/data/cases/python38.py index 63b0588bc27..919ea6aeed4 100644 --- a/tests/data/py_38/python38.py +++ b/tests/data/cases/python38.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.8 +# flags: --minimum-version=3.8 def starred_return(): @@ -22,9 +22,6 @@ def t(): # output -#!/usr/bin/env python3.8 - - def starred_return(): my_list = ["value2", "value3"] return "value1", *my_list diff --git a/tests/data/py_39/python39.py b/tests/data/cases/python39.py similarity index 91% rename from tests/data/py_39/python39.py rename to tests/data/cases/python39.py index ae67c2257eb..1b9536c1529 100644 --- a/tests/data/py_39/python39.py +++ b/tests/data/cases/python39.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.9 +# flags: --minimum-version=3.9 @relaxed_decorator[0] def f(): @@ -14,10 +14,6 @@ def f(): # output - -#!/usr/bin/env python3.9 - - @relaxed_decorator[0] def f(): ... diff --git a/tests/data/cases/raw_docstring.py b/tests/data/cases/raw_docstring.py new file mode 100644 index 00000000000..751fd3201df --- /dev/null +++ b/tests/data/cases/raw_docstring.py @@ -0,0 +1,32 @@ +# flags: --preview --skip-string-normalization +class C: + + r"""Raw""" + +def f(): + + r"""Raw""" + +class SingleQuotes: + + + r'''Raw''' + +class UpperCaseR: + R"""Raw""" + +# output +class C: + r"""Raw""" + + +def f(): + r"""Raw""" + + +class SingleQuotes: + r'''Raw''' + + +class UpperCaseR: + R"""Raw""" diff --git a/tests/data/simple_cases/remove_await_parens.py b/tests/data/cases/remove_await_parens.py similarity index 90% rename from tests/data/simple_cases/remove_await_parens.py rename to tests/data/cases/remove_await_parens.py index 8c7223d2f39..073150c5f08 100644 --- a/tests/data/simple_cases/remove_await_parens.py +++ b/tests/data/cases/remove_await_parens.py @@ -80,6 +80,15 @@ async def main(): async def main(): await (yield) +async def main(): + await (a ** b) + await (a[b] ** c) + await (a ** b[c]) + await ((a + b) ** (c + d)) + await (a + b) + await (a[b]) + await (a[b ** c]) + # output import asyncio @@ -174,3 +183,13 @@ async def main(): async def main(): await (yield) + + +async def main(): + await (a**b) + await (a[b] ** c) + await (a ** b[c]) + await ((a + b) ** (c + d)) + await (a + b) + await a[b] + await a[b**c] diff --git a/tests/data/simple_cases/remove_except_parens.py b/tests/data/cases/remove_except_parens.py similarity index 100% rename from tests/data/simple_cases/remove_except_parens.py rename to tests/data/cases/remove_except_parens.py diff --git a/tests/data/simple_cases/remove_for_brackets.py b/tests/data/cases/remove_for_brackets.py similarity index 100% rename from tests/data/simple_cases/remove_for_brackets.py rename to tests/data/cases/remove_for_brackets.py diff --git a/tests/data/simple_cases/remove_newline_after_code_block_open.py b/tests/data/cases/remove_newline_after_code_block_open.py similarity index 100% rename from tests/data/simple_cases/remove_newline_after_code_block_open.py rename to tests/data/cases/remove_newline_after_code_block_open.py diff --git a/tests/data/py_310/remove_newline_after_match.py b/tests/data/cases/remove_newline_after_match.py similarity index 88% rename from tests/data/py_310/remove_newline_after_match.py rename to tests/data/cases/remove_newline_after_match.py index f7bcfbf27a2..fe6592b664d 100644 --- a/tests/data/py_310/remove_newline_after_match.py +++ b/tests/data/cases/remove_newline_after_match.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.10 def http_status(status): match status: diff --git a/tests/data/simple_cases/remove_parens.py b/tests/data/cases/remove_parens.py similarity index 100% rename from tests/data/simple_cases/remove_parens.py rename to tests/data/cases/remove_parens.py diff --git a/tests/data/py_39/remove_with_brackets.py b/tests/data/cases/remove_with_brackets.py similarity index 98% rename from tests/data/py_39/remove_with_brackets.py rename to tests/data/cases/remove_with_brackets.py index ea58ab93a16..3ee64902a30 100644 --- a/tests/data/py_39/remove_with_brackets.py +++ b/tests/data/cases/remove_with_brackets.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.9 with (open("bla.txt")): pass diff --git a/tests/data/simple_cases/return_annotation_brackets.py b/tests/data/cases/return_annotation_brackets.py similarity index 92% rename from tests/data/simple_cases/return_annotation_brackets.py rename to tests/data/cases/return_annotation_brackets.py index 265c30220d8..8509ecdb92c 100644 --- a/tests/data/simple_cases/return_annotation_brackets.py +++ b/tests/data/cases/return_annotation_brackets.py @@ -87,6 +87,11 @@ def foo() -> tuple[loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo def foo() -> tuple[int, int, int,]: return 2 +# Magic trailing comma example, with params +# this is broken - the trailing comma is transferred to the param list. Fixed in preview +def foo(a,b) -> tuple[int, int, int,]: + return 2 + # output # Control def double(a: int) -> int: @@ -208,3 +213,11 @@ def foo() -> ( ] ): return 2 + + +# Magic trailing comma example, with params +# this is broken - the trailing comma is transferred to the param list. Fixed in preview +def foo( + a, b +) -> tuple[int, int, int,]: + return 2 diff --git a/tests/data/simple_cases/skip_magic_trailing_comma.py b/tests/data/cases/skip_magic_trailing_comma.py similarity index 97% rename from tests/data/simple_cases/skip_magic_trailing_comma.py rename to tests/data/cases/skip_magic_trailing_comma.py index c020db79864..4dda5df40f0 100644 --- a/tests/data/simple_cases/skip_magic_trailing_comma.py +++ b/tests/data/cases/skip_magic_trailing_comma.py @@ -1,3 +1,4 @@ +# flags: --skip-magic-trailing-comma # We should not remove the trailing comma in a single-element subscript. a: tuple[int,] b = tuple[int,] diff --git a/tests/data/simple_cases/slices.py b/tests/data/cases/slices.py similarity index 100% rename from tests/data/simple_cases/slices.py rename to tests/data/cases/slices.py diff --git a/tests/data/py_310/starred_for_target.py b/tests/data/cases/starred_for_target.py similarity index 92% rename from tests/data/py_310/starred_for_target.py rename to tests/data/cases/starred_for_target.py index 8fc8e059ed3..13e517816d6 100644 --- a/tests/data/py_310/starred_for_target.py +++ b/tests/data/cases/starred_for_target.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.10 for x in *a, *b: print(x) diff --git a/tests/data/simple_cases/string_prefixes.py b/tests/data/cases/string_prefixes.py similarity index 100% rename from tests/data/simple_cases/string_prefixes.py rename to tests/data/cases/string_prefixes.py diff --git a/tests/data/miscellaneous/stub.pyi b/tests/data/cases/stub.py similarity index 99% rename from tests/data/miscellaneous/stub.pyi rename to tests/data/cases/stub.py index af2cd2c2c02..f3828d55ba2 100644 --- a/tests/data/miscellaneous/stub.pyi +++ b/tests/data/cases/stub.py @@ -1,3 +1,4 @@ +# flags: --pyi X: int def f(): ... diff --git a/tests/data/simple_cases/torture.py b/tests/data/cases/torture.py similarity index 100% rename from tests/data/simple_cases/torture.py rename to tests/data/cases/torture.py diff --git a/tests/data/simple_cases/trailing_comma_optional_parens1.py b/tests/data/cases/trailing_comma_optional_parens1.py similarity index 100% rename from tests/data/simple_cases/trailing_comma_optional_parens1.py rename to tests/data/cases/trailing_comma_optional_parens1.py diff --git a/tests/data/simple_cases/trailing_comma_optional_parens2.py b/tests/data/cases/trailing_comma_optional_parens2.py similarity index 100% rename from tests/data/simple_cases/trailing_comma_optional_parens2.py rename to tests/data/cases/trailing_comma_optional_parens2.py diff --git a/tests/data/simple_cases/trailing_comma_optional_parens3.py b/tests/data/cases/trailing_comma_optional_parens3.py similarity index 100% rename from tests/data/simple_cases/trailing_comma_optional_parens3.py rename to tests/data/cases/trailing_comma_optional_parens3.py diff --git a/tests/data/simple_cases/trailing_commas_in_leading_parts.py b/tests/data/cases/trailing_commas_in_leading_parts.py similarity index 100% rename from tests/data/simple_cases/trailing_commas_in_leading_parts.py rename to tests/data/cases/trailing_commas_in_leading_parts.py diff --git a/tests/data/simple_cases/tricky_unicode_symbols.py b/tests/data/cases/tricky_unicode_symbols.py similarity index 100% rename from tests/data/simple_cases/tricky_unicode_symbols.py rename to tests/data/cases/tricky_unicode_symbols.py diff --git a/tests/data/simple_cases/tupleassign.py b/tests/data/cases/tupleassign.py similarity index 100% rename from tests/data/simple_cases/tupleassign.py rename to tests/data/cases/tupleassign.py diff --git a/tests/data/cases/type_aliases.py b/tests/data/cases/type_aliases.py new file mode 100644 index 00000000000..7c2009e8202 --- /dev/null +++ b/tests/data/cases/type_aliases.py @@ -0,0 +1,30 @@ +# flags: --minimum-version=3.12 + +type A=int +type Gen[T]=list[T] +type Alias[T]=lambda: T +type And[T]=T and T +type IfElse[T]=T if T else T +type One = int; type Another = str +class X: type InClass = int + +type = aliased +print(type(42)) + +# output + +type A = int +type Gen[T] = list[T] +type Alias[T] = lambda: T +type And[T] = T and T +type IfElse[T] = T if T else T +type One = int +type Another = str + + +class X: + type InClass = int + + +type = aliased +print(type(42)) diff --git a/tests/data/cases/type_comment_syntax_error.py b/tests/data/cases/type_comment_syntax_error.py new file mode 100644 index 00000000000..2e5ca2ede8c --- /dev/null +++ b/tests/data/cases/type_comment_syntax_error.py @@ -0,0 +1,11 @@ +def foo( + # type: Foo + x): pass + +# output + +def foo( + # type: Foo + x, +): + pass diff --git a/tests/data/cases/type_params.py b/tests/data/cases/type_params.py new file mode 100644 index 00000000000..720a775ef31 --- /dev/null +++ b/tests/data/cases/type_params.py @@ -0,0 +1,58 @@ +# flags: --minimum-version=3.12 +def func [T ](): pass +async def func [ T ] (): pass +class C[ T ] : pass + +def all_in[T : int,U : (bytes, str),* Ts,**P](): pass + +def really_long[WhatIsTheLongestTypeVarNameYouCanThinkOfEnoughToMakeBlackSplitThisLine](): pass + +def even_longer[WhatIsTheLongestTypeVarNameYouCanThinkOfEnoughToMakeBlackSplitThisLine: WhatIfItHadABound](): pass + +def it_gets_worse[WhatIsTheLongestTypeVarNameYouCanThinkOfEnoughToMakeBlackSplitThisLine, ItCouldBeGenericOverMultipleTypeVars](): pass + +def magic[Trailing, Comma,](): pass + +# output + + +def func[T](): + pass + + +async def func[T](): + pass + + +class C[T]: + pass + + +def all_in[T: int, U: (bytes, str), *Ts, **P](): + pass + + +def really_long[ + WhatIsTheLongestTypeVarNameYouCanThinkOfEnoughToMakeBlackSplitThisLine +](): + pass + + +def even_longer[ + WhatIsTheLongestTypeVarNameYouCanThinkOfEnoughToMakeBlackSplitThisLine: WhatIfItHadABound +](): + pass + + +def it_gets_worse[ + WhatIsTheLongestTypeVarNameYouCanThinkOfEnoughToMakeBlackSplitThisLine, + ItCouldBeGenericOverMultipleTypeVars, +](): + pass + + +def magic[ + Trailing, + Comma, +](): + pass diff --git a/tests/data/simple_cases/whitespace.py b/tests/data/cases/whitespace.py similarity index 100% rename from tests/data/simple_cases/whitespace.py rename to tests/data/cases/whitespace.py diff --git a/tests/data/invalid_line_ranges.toml b/tests/data/invalid_line_ranges.toml new file mode 100644 index 00000000000..791573f2625 --- /dev/null +++ b/tests/data/invalid_line_ranges.toml @@ -0,0 +1,2 @@ +[tool.black] +line-ranges = "1-1" diff --git a/tests/data/line_ranges_formatted/basic.py b/tests/data/line_ranges_formatted/basic.py new file mode 100644 index 00000000000..b419b1f16ae --- /dev/null +++ b/tests/data/line_ranges_formatted/basic.py @@ -0,0 +1,50 @@ +"""Module doc.""" + +from typing import ( + Callable, + Literal, +) + + +# fmt: off +class Unformatted: + def should_also_work(self): + pass +# fmt: on + + +a = [1, 2] # fmt: skip + + +# This should cover as many syntaxes as possible. +class Foo: + """Class doc.""" + + def __init__(self) -> None: + pass + + @add_logging + @memoize.memoize(max_items=2) + def plus_one( + self, + number: int, + ) -> int: + return number + 1 + + async def async_plus_one(self, number: int) -> int: + await asyncio.sleep(1) + async with some_context(): + return number + 1 + + +try: + for i in range(10): + while condition: + if something: + then_something() + elif something_else: + then_something_else() +except ValueError as e: + handle(e) +finally: + done() diff --git a/tests/data/line_ranges_formatted/pattern_matching.py b/tests/data/line_ranges_formatted/pattern_matching.py new file mode 100644 index 00000000000..cd98efdd504 --- /dev/null +++ b/tests/data/line_ranges_formatted/pattern_matching.py @@ -0,0 +1,25 @@ +# flags: --minimum-version=3.10 + + +def pattern_matching(): + match status: + case 1: + return "1" + case [single]: + return "single" + case [ + action, + obj, + ]: + return "act on obj" + case Point(x=0): + return "class pattern" + case {"text": message}: + return "mapping" + case { + "text": message, + "format": _, + }: + return "mapping" + case _: + return "fallback" diff --git a/tests/data/miscellaneous/force_py36.py b/tests/data/miscellaneous/force_py36.py index cad935e525a..4c9b70336e7 100644 --- a/tests/data/miscellaneous/force_py36.py +++ b/tests/data/miscellaneous/force_py36.py @@ -1,6 +1,6 @@ # The input source must not contain any Py36-specific syntax (e.g. argument type # annotations, trailing comma after *rest) or this test becomes invalid. -def long_function_name(argument_one, argument_two, argument_three, argument_four, argument_five, argument_six, *rest): ... +def long_function_name(argument_one, argument_two, argument_three, argument_four, argument_five, argument_six, *rest): pass # output # The input source must not contain any Py36-specific syntax (e.g. argument type # annotations, trailing comma after *rest) or this test becomes invalid. @@ -13,4 +13,4 @@ def long_function_name( argument_six, *rest, ): - ... + pass diff --git a/tests/data/miscellaneous/force_pyi.py b/tests/data/miscellaneous/force_pyi.py index 07ed93c6879..40caf30a983 100644 --- a/tests/data/miscellaneous/force_pyi.py +++ b/tests/data/miscellaneous/force_pyi.py @@ -1,3 +1,4 @@ +# flags: --pyi from typing import Union @bird diff --git a/tests/data/miscellaneous/string_quotes.py b/tests/data/miscellaneous/string_quotes.py index 3384241f4ad..6ec088ac79b 100644 --- a/tests/data/miscellaneous/string_quotes.py +++ b/tests/data/miscellaneous/string_quotes.py @@ -1,4 +1,5 @@ '''''' + '\'' '"' "'" @@ -59,6 +60,7 @@ # output """""" + "'" '"' "'" diff --git a/tests/data/preview/return_annotation_brackets_string.py b/tests/data/preview/return_annotation_brackets_string.py deleted file mode 100644 index 6978829fd5c..00000000000 --- a/tests/data/preview/return_annotation_brackets_string.py +++ /dev/null @@ -1,12 +0,0 @@ -# Long string example -def frobnicate() -> "ThisIsTrulyUnreasonablyExtremelyLongClassName | list[ThisIsTrulyUnreasonablyExtremelyLongClassName]": - pass - -# output - -# Long string example -def frobnicate() -> ( - "ThisIsTrulyUnreasonablyExtremelyLongClassName |" - " list[ThisIsTrulyUnreasonablyExtremelyLongClassName]" -): - pass diff --git a/tests/optional.py b/tests/optional.py index 8a39cc440a6..70ee823e316 100644 --- a/tests/optional.py +++ b/tests/optional.py @@ -26,7 +26,15 @@ from pytest import StashKey except ImportError: # pytest < 7 - from _pytest.store import StoreKey as StashKey # type: ignore[no-redef] + # + # "isort: skip" is needed or it moves the "type: ignore" to the following line + # because of the line length, and then mypy complains. + # Of course, adding the "isort: skip" means that + # flake8-bugbear then also complains about the line length, + # so we *also* need a "noqa" comment for good measure :) + from _pytest.store import ( # type: ignore[import-not-found, no-redef] # isort: skip # noqa: B950 + StoreKey as StashKey, + ) log = logging.getLogger(__name__) diff --git a/tests/test_black.py b/tests/test_black.py index e5e17777715..0af5fd2a1f4 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -8,8 +8,8 @@ import os import re import sys +import textwrap import types -import unittest from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager, redirect_stderr from dataclasses import replace @@ -25,6 +25,7 @@ List, Optional, Sequence, + Set, Type, TypeVar, Union, @@ -41,7 +42,7 @@ import black.files from black import Feature, TargetVersion from black import re_compile_maybe_verbose as compile_pattern -from black.cache import get_cache_dir, get_cache_file +from black.cache import FileData, get_cache_dir, get_cache_file from black.debug import DebugVisitor from black.output import color_diff, diff from black.report import Report @@ -104,6 +105,7 @@ class FakeContext(click.Context): def __init__(self) -> None: self.default_map: Dict[str, Any] = {} + self.params: Dict[str, Any] = {} # Dummy root, since most of the tests don't care about it self.obj: Dict[str, Any] = {"root": PROJECT_ROOT} @@ -148,8 +150,7 @@ def test_empty_ff(self) -> None: tmp_file = Path(black.dump_to_file()) try: self.assertFalse(ff(tmp_file, write_back=black.WriteBack.YES)) - with open(tmp_file, encoding="utf8") as f: - actual = f.read() + actual = tmp_file.read_text(encoding="utf-8") finally: os.unlink(tmp_file) self.assertFormatEqual(expected, actual) @@ -177,7 +178,7 @@ def test_one_empty_line_ff(self) -> None: ff(tmp_file, mode=mode, write_back=black.WriteBack.YES) ) with open(tmp_file, "rb") as f: - actual = f.read().decode("utf8") + actual = f.read().decode("utf-8") finally: os.unlink(tmp_file) self.assertFormatEqual(expected, actual) @@ -188,7 +189,9 @@ def test_experimental_string_processing_warns(self) -> None: ) def test_piping(self) -> None: - source, expected = read_data_from_file(PROJECT_ROOT / "src/black/__init__.py") + _, source, expected = read_data_from_file( + PROJECT_ROOT / "src/black/__init__.py" + ) result = BlackRunner().invoke( black.main, [ @@ -197,7 +200,7 @@ def test_piping(self) -> None: f"--line-length={black.DEFAULT_LINE_LENGTH}", f"--config={EMPTY_CONFIG}", ], - input=BytesIO(source.encode("utf8")), + input=BytesIO(source.encode("utf-8")), ) self.assertEqual(result.exit_code, 0) self.assertFormatEqual(expected, result.output) @@ -207,11 +210,11 @@ def test_piping(self) -> None: def test_piping_diff(self) -> None: diff_header = re.compile( - r"(STDIN|STDOUT)\t\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d\.\d\d\d\d\d\d " - r"\+\d\d\d\d" + r"(STDIN|STDOUT)\t\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d\.\d\d\d\d\d\d" + r"\+\d\d:\d\d" ) - source, _ = read_data("simple_cases", "expression.py") - expected, _ = read_data("simple_cases", "expression.diff") + source, _ = read_data("cases", "expression.py") + expected, _ = read_data("cases", "expression.diff") args = [ "-", "--fast", @@ -220,7 +223,7 @@ def test_piping_diff(self) -> None: f"--config={EMPTY_CONFIG}", ] result = BlackRunner().invoke( - black.main, args, input=BytesIO(source.encode("utf8")) + black.main, args, input=BytesIO(source.encode("utf-8")) ) self.assertEqual(result.exit_code, 0) actual = diff_header.sub(DETERMINISTIC_HEADER, result.output) @@ -228,7 +231,7 @@ def test_piping_diff(self) -> None: self.assertEqual(expected, actual) def test_piping_diff_with_color(self) -> None: - source, _ = read_data("simple_cases", "expression.py") + source, _ = read_data("cases", "expression.py") args = [ "-", "--fast", @@ -238,7 +241,7 @@ def test_piping_diff_with_color(self) -> None: f"--config={EMPTY_CONFIG}", ] result = BlackRunner().invoke( - black.main, args, input=BytesIO(source.encode("utf8")) + black.main, args, input=BytesIO(source.encode("utf-8")) ) actual = result.output # Again, the contents are checked in a different test, so only look for colors. @@ -264,20 +267,28 @@ def _test_wip(self) -> None: black.assert_stable(source, actual, black.FileMode()) def test_pep_572_version_detection(self) -> None: - source, _ = read_data("py_38", "pep_572") + source, _ = read_data("cases", "pep_572") root = black.lib2to3_parse(source) features = black.get_features_used(root) self.assertIn(black.Feature.ASSIGNMENT_EXPRESSIONS, features) versions = black.detect_target_versions(root) self.assertIn(black.TargetVersion.PY38, versions) + def test_pep_695_version_detection(self) -> None: + for file in ("type_aliases", "type_params"): + source, _ = read_data("cases", file) + root = black.lib2to3_parse(source) + features = black.get_features_used(root) + self.assertIn(black.Feature.TYPE_PARAMS, features) + versions = black.detect_target_versions(root) + self.assertIn(black.TargetVersion.PY312, versions) + def test_expression_ff(self) -> None: - source, expected = read_data("simple_cases", "expression.py") + source, expected = read_data("cases", "expression.py") tmp_file = Path(black.dump_to_file(source)) try: self.assertTrue(ff(tmp_file, write_back=black.WriteBack.YES)) - with open(tmp_file, encoding="utf8") as f: - actual = f.read() + actual = tmp_file.read_text(encoding="utf-8") finally: os.unlink(tmp_file) self.assertFormatEqual(expected, actual) @@ -286,12 +297,12 @@ def test_expression_ff(self) -> None: black.assert_stable(source, actual, DEFAULT_MODE) def test_expression_diff(self) -> None: - source, _ = read_data("simple_cases", "expression.py") - expected, _ = read_data("simple_cases", "expression.diff") + source, _ = read_data("cases", "expression.py") + expected, _ = read_data("cases", "expression.diff") tmp_file = Path(black.dump_to_file(source)) diff_header = re.compile( rf"{re.escape(str(tmp_file))}\t\d\d\d\d-\d\d-\d\d " - r"\d\d:\d\d:\d\d\.\d\d\d\d\d\d \+\d\d\d\d" + r"\d\d:\d\d:\d\d\.\d\d\d\d\d\d\+\d\d:\d\d" ) try: result = BlackRunner().invoke( @@ -307,13 +318,13 @@ def test_expression_diff(self) -> None: msg = ( "Expected diff isn't equal to the actual. If you made changes to" " expression.py and this is an anticipated difference, overwrite" - f" tests/data/expression.diff with {dump}" + f" tests/data/cases/expression.diff with {dump}" ) self.assertEqual(expected, actual, msg) def test_expression_diff_with_color(self) -> None: - source, _ = read_data("simple_cases", "expression.py") - expected, _ = read_data("simple_cases", "expression.diff") + source, _ = read_data("cases", "expression.py") + expected, _ = read_data("cases", "expression.diff") tmp_file = Path(black.dump_to_file(source)) try: result = BlackRunner().invoke( @@ -332,7 +343,7 @@ def test_expression_diff_with_color(self) -> None: self.assertIn("\033[0m", actual) def test_detect_pos_only_arguments(self) -> None: - source, _ = read_data("py_38", "pep_570") + source, _ = read_data("cases", "pep_570") root = black.lib2to3_parse(source) features = black.get_features_used(root) self.assertIn(black.Feature.POS_ONLY_ARGUMENTS, features) @@ -380,8 +391,7 @@ def test_skip_source_first_line(self) -> None: black.main, [str(tmp_file), "-x", f"--config={EMPTY_CONFIG}"] ) self.assertEqual(result.exit_code, 0) - with open(tmp_file, encoding="utf8") as f: - actual = f.read() + actual = tmp_file.read_text(encoding="utf-8") self.assertFormatEqual(source, actual) def test_skip_source_first_line_when_mixing_newlines(self) -> None: @@ -395,14 +405,14 @@ def test_skip_source_first_line_when_mixing_newlines(self) -> None: self.assertEqual(test_file.read_bytes(), expected) def test_skip_magic_trailing_comma(self) -> None: - source, _ = read_data("simple_cases", "expression") + source, _ = read_data("cases", "expression") expected, _ = read_data( "miscellaneous", "expression_skip_magic_trailing_comma.diff" ) tmp_file = Path(black.dump_to_file(source)) diff_header = re.compile( rf"{re.escape(str(tmp_file))}\t\d\d\d\d-\d\d-\d\d " - r"\d\d:\d\d:\d\d\.\d\d\d\d\d\d \+\d\d\d\d" + r"\d\d:\d\d:\d\d\.\d\d\d\d\d\d\+\d\d:\d\d" ) try: result = BlackRunner().invoke( @@ -427,7 +437,7 @@ def test_skip_magic_trailing_comma(self) -> None: @patch("black.dump_to_file", dump_to_stderr) def test_async_as_identifier(self) -> None: source_path = get_case_path("miscellaneous", "async_as_identifier") - source, expected = read_data_from_file(source_path) + _, source, expected = read_data_from_file(source_path) actual = fs(source) self.assertFormatEqual(expected, actual) major, minor = sys.version_info[:2] @@ -441,8 +451,8 @@ def test_async_as_identifier(self) -> None: @patch("black.dump_to_file", dump_to_stderr) def test_python37(self) -> None: - source_path = get_case_path("py_37", "python37") - source, expected = read_data_from_file(source_path) + source_path = get_case_path("cases", "python37") + _, source, expected = read_data_from_file(source_path) actual = fs(source) self.assertFormatEqual(expected, actual) major, minor = sys.version_info[:2] @@ -485,9 +495,7 @@ def test_false_positive_symlink_output_issue_3384(self) -> None: project_root = Path(THIS_DIR / "data" / "nested_gitignore_tests") working_directory = project_root / "root" target_abspath = working_directory / "child" - target_contents = ( - src.relative_to(working_directory) for src in target_abspath.iterdir() - ) + target_contents = list(target_abspath.iterdir()) def mock_n_calls(responses: List[bool]) -> Callable[[], bool]: def _mocked_calls() -> bool: @@ -498,13 +506,13 @@ def _mocked_calls() -> bool: return _mocked_calls with patch("pathlib.Path.iterdir", return_value=target_contents), patch( - "pathlib.Path.cwd", return_value=working_directory + "pathlib.Path.resolve", return_value=target_abspath ), patch("pathlib.Path.is_dir", side_effect=mock_n_calls([True])): - ctx = FakeContext() - ctx.obj["root"] = project_root + # Note that the root folder (project_root) isn't the folder + # named "root" (aka working_directory) report = MagicMock(verbose=True) black.get_sources( - ctx=ctx, + root=project_root, src=("./child",), quiet=False, verbose=True, @@ -520,7 +528,8 @@ def _mocked_calls() -> bool: for _, mock_args, _ in report.path_ignored.mock_calls ), "A symbolic link was reported." report.path_ignored.assert_called_once_with( - Path("child", "b.py"), "matches a .gitignore file content" + Path(working_directory, "child", "b.py"), + "matches a .gitignore file content", ) def test_report_verbose(self) -> None: @@ -567,10 +576,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(err_lines[-1], "error: cannot format e1: boom") self.assertEqual( unstyle(str(report)), - ( - "1 file reformatted, 2 files left unchanged, 1 file failed to" - " reformat." - ), + "1 file reformatted, 2 files left unchanged, 1 file failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.done(Path("f3"), black.Changed.YES) @@ -579,10 +586,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(out_lines[-1], "reformatted f3") self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 2 files left unchanged, 1 file failed to" - " reformat." - ), + "2 files reformatted, 2 files left unchanged, 1 file failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.failed(Path("e2"), "boom") @@ -591,10 +596,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(err_lines[-1], "error: cannot format e2: boom") self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 2 files left unchanged, 2 files failed to" - " reformat." - ), + "2 files reformatted, 2 files left unchanged, 2 files failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.path_ignored(Path("wat"), "no match") @@ -603,10 +606,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(out_lines[-1], "wat ignored: no match") self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 2 files left unchanged, 2 files failed to" - " reformat." - ), + "2 files reformatted, 2 files left unchanged, 2 files failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.done(Path("f4"), black.Changed.NO) @@ -615,28 +616,22 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(out_lines[-1], "f4 already well formatted, good job.") self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 3 files left unchanged, 2 files failed to" - " reformat." - ), + "2 files reformatted, 3 files left unchanged, 2 files failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.check = True self.assertEqual( unstyle(str(report)), - ( - "2 files would be reformatted, 3 files would be left unchanged, 2" - " files would fail to reformat." - ), + "2 files would be reformatted, 3 files would be left unchanged, 2" + " files would fail to reformat.", ) report.check = False report.diff = True self.assertEqual( unstyle(str(report)), - ( - "2 files would be reformatted, 3 files would be left unchanged, 2" - " files would fail to reformat." - ), + "2 files would be reformatted, 3 files would be left unchanged, 2" + " files would fail to reformat.", ) def test_report_quiet(self) -> None: @@ -678,10 +673,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(err_lines[-1], "error: cannot format e1: boom") self.assertEqual( unstyle(str(report)), - ( - "1 file reformatted, 2 files left unchanged, 1 file failed to" - " reformat." - ), + "1 file reformatted, 2 files left unchanged, 1 file failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.done(Path("f3"), black.Changed.YES) @@ -689,10 +682,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(len(err_lines), 1) self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 2 files left unchanged, 1 file failed to" - " reformat." - ), + "2 files reformatted, 2 files left unchanged, 1 file failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.failed(Path("e2"), "boom") @@ -701,10 +692,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(err_lines[-1], "error: cannot format e2: boom") self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 2 files left unchanged, 2 files failed to" - " reformat." - ), + "2 files reformatted, 2 files left unchanged, 2 files failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.path_ignored(Path("wat"), "no match") @@ -712,10 +701,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(len(err_lines), 2) self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 2 files left unchanged, 2 files failed to" - " reformat." - ), + "2 files reformatted, 2 files left unchanged, 2 files failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.done(Path("f4"), black.Changed.NO) @@ -723,28 +710,22 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(len(err_lines), 2) self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 3 files left unchanged, 2 files failed to" - " reformat." - ), + "2 files reformatted, 3 files left unchanged, 2 files failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.check = True self.assertEqual( unstyle(str(report)), - ( - "2 files would be reformatted, 3 files would be left unchanged, 2" - " files would fail to reformat." - ), + "2 files would be reformatted, 3 files would be left unchanged, 2" + " files would fail to reformat.", ) report.check = False report.diff = True self.assertEqual( unstyle(str(report)), - ( - "2 files would be reformatted, 3 files would be left unchanged, 2" - " files would fail to reformat." - ), + "2 files would be reformatted, 3 files would be left unchanged, 2" + " files would fail to reformat.", ) def test_report_normal(self) -> None: @@ -788,10 +769,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(err_lines[-1], "error: cannot format e1: boom") self.assertEqual( unstyle(str(report)), - ( - "1 file reformatted, 2 files left unchanged, 1 file failed to" - " reformat." - ), + "1 file reformatted, 2 files left unchanged, 1 file failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.done(Path("f3"), black.Changed.YES) @@ -800,10 +779,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(out_lines[-1], "reformatted f3") self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 2 files left unchanged, 1 file failed to" - " reformat." - ), + "2 files reformatted, 2 files left unchanged, 1 file failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.failed(Path("e2"), "boom") @@ -812,10 +789,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(err_lines[-1], "error: cannot format e2: boom") self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 2 files left unchanged, 2 files failed to" - " reformat." - ), + "2 files reformatted, 2 files left unchanged, 2 files failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.path_ignored(Path("wat"), "no match") @@ -823,10 +798,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(len(err_lines), 2) self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 2 files left unchanged, 2 files failed to" - " reformat." - ), + "2 files reformatted, 2 files left unchanged, 2 files failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.done(Path("f4"), black.Changed.NO) @@ -834,28 +807,22 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(len(err_lines), 2) self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 3 files left unchanged, 2 files failed to" - " reformat." - ), + "2 files reformatted, 3 files left unchanged, 2 files failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.check = True self.assertEqual( unstyle(str(report)), - ( - "2 files would be reformatted, 3 files would be left unchanged, 2" - " files would fail to reformat." - ), + "2 files would be reformatted, 3 files would be left unchanged, 2" + " files would fail to reformat.", ) report.check = False report.diff = True self.assertEqual( unstyle(str(report)), - ( - "2 files would be reformatted, 3 files would be left unchanged, 2" - " files would fail to reformat." - ), + "2 files would be reformatted, 3 files would be left unchanged, 2" + " files would fail to reformat.", ) def test_lib2to3_parse(self) -> None: @@ -908,71 +875,88 @@ def test_get_features_used_decorator(self) -> None: ) def test_get_features_used(self) -> None: - node = black.lib2to3_parse("def f(*, arg): ...\n") - self.assertEqual(black.get_features_used(node), set()) - node = black.lib2to3_parse("def f(*, arg,): ...\n") - self.assertEqual(black.get_features_used(node), {Feature.TRAILING_COMMA_IN_DEF}) - node = black.lib2to3_parse("f(*arg,)\n") - self.assertEqual( - black.get_features_used(node), {Feature.TRAILING_COMMA_IN_CALL} + self.check_features_used("def f(*, arg): ...\n", set()) + self.check_features_used( + "def f(*, arg,): ...\n", {Feature.TRAILING_COMMA_IN_DEF} ) - node = black.lib2to3_parse("def f(*, arg): f'string'\n") - self.assertEqual(black.get_features_used(node), {Feature.F_STRINGS}) - node = black.lib2to3_parse("123_456\n") - self.assertEqual(black.get_features_used(node), {Feature.NUMERIC_UNDERSCORES}) - node = black.lib2to3_parse("123456\n") - self.assertEqual(black.get_features_used(node), set()) - source, expected = read_data("simple_cases", "function") - node = black.lib2to3_parse(source) + self.check_features_used("f(*arg,)\n", {Feature.TRAILING_COMMA_IN_CALL}) + self.check_features_used("def f(*, arg): f'string'\n", {Feature.F_STRINGS}) + self.check_features_used("123_456\n", {Feature.NUMERIC_UNDERSCORES}) + self.check_features_used("123456\n", set()) + + source, expected = read_data("cases", "function") expected_features = { Feature.TRAILING_COMMA_IN_CALL, Feature.TRAILING_COMMA_IN_DEF, Feature.F_STRINGS, } - self.assertEqual(black.get_features_used(node), expected_features) - node = black.lib2to3_parse(expected) - self.assertEqual(black.get_features_used(node), expected_features) - source, expected = read_data("simple_cases", "expression") - node = black.lib2to3_parse(source) - self.assertEqual(black.get_features_used(node), set()) - node = black.lib2to3_parse(expected) - self.assertEqual(black.get_features_used(node), set()) - node = black.lib2to3_parse("lambda a, /, b: ...") - self.assertEqual(black.get_features_used(node), {Feature.POS_ONLY_ARGUMENTS}) - node = black.lib2to3_parse("def fn(a, /, b): ...") - self.assertEqual(black.get_features_used(node), {Feature.POS_ONLY_ARGUMENTS}) - node = black.lib2to3_parse("def fn(): yield a, b") - self.assertEqual(black.get_features_used(node), set()) - node = black.lib2to3_parse("def fn(): return a, b") - self.assertEqual(black.get_features_used(node), set()) - node = black.lib2to3_parse("def fn(): yield *b, c") - self.assertEqual(black.get_features_used(node), {Feature.UNPACKING_ON_FLOW}) - node = black.lib2to3_parse("def fn(): return a, *b, c") - self.assertEqual(black.get_features_used(node), {Feature.UNPACKING_ON_FLOW}) - node = black.lib2to3_parse("x = a, *b, c") - self.assertEqual(black.get_features_used(node), set()) - node = black.lib2to3_parse("x: Any = regular") - self.assertEqual(black.get_features_used(node), set()) - node = black.lib2to3_parse("x: Any = (regular, regular)") - self.assertEqual(black.get_features_used(node), set()) - node = black.lib2to3_parse("x: Any = Complex(Type(1))[something]") - self.assertEqual(black.get_features_used(node), set()) - node = black.lib2to3_parse("x: Tuple[int, ...] = a, b, c") - self.assertEqual( - black.get_features_used(node), {Feature.ANN_ASSIGN_EXTENDED_RHS} + self.check_features_used(source, expected_features) + self.check_features_used(expected, expected_features) + + source, expected = read_data("cases", "expression") + self.check_features_used(source, set()) + self.check_features_used(expected, set()) + + self.check_features_used("lambda a, /, b: ...\n", {Feature.POS_ONLY_ARGUMENTS}) + self.check_features_used("def fn(a, /, b): ...", {Feature.POS_ONLY_ARGUMENTS}) + + self.check_features_used("def fn(): yield a, b", set()) + self.check_features_used("def fn(): return a, b", set()) + self.check_features_used("def fn(): yield *b, c", {Feature.UNPACKING_ON_FLOW}) + self.check_features_used( + "def fn(): return a, *b, c", {Feature.UNPACKING_ON_FLOW} + ) + self.check_features_used("x = a, *b, c", set()) + + self.check_features_used("x: Any = regular", set()) + self.check_features_used("x: Any = (regular, regular)", set()) + self.check_features_used("x: Any = Complex(Type(1))[something]", set()) + self.check_features_used( + "x: Tuple[int, ...] = a, b, c", {Feature.ANN_ASSIGN_EXTENDED_RHS} + ) + + self.check_features_used("try: pass\nexcept Something: pass", set()) + self.check_features_used("try: pass\nexcept (*Something,): pass", set()) + self.check_features_used( + "try: pass\nexcept *Group: pass", {Feature.EXCEPT_STAR} ) - node = black.lib2to3_parse("try: pass\nexcept Something: pass") - self.assertEqual(black.get_features_used(node), set()) - node = black.lib2to3_parse("try: pass\nexcept (*Something,): pass") - self.assertEqual(black.get_features_used(node), set()) - node = black.lib2to3_parse("try: pass\nexcept *Group: pass") - self.assertEqual(black.get_features_used(node), {Feature.EXCEPT_STAR}) - node = black.lib2to3_parse("a[*b]") - self.assertEqual(black.get_features_used(node), {Feature.VARIADIC_GENERICS}) - node = black.lib2to3_parse("a[x, *y(), z] = t") - self.assertEqual(black.get_features_used(node), {Feature.VARIADIC_GENERICS}) - node = black.lib2to3_parse("def fn(*args: *T): pass") - self.assertEqual(black.get_features_used(node), {Feature.VARIADIC_GENERICS}) + + self.check_features_used("a[*b]", {Feature.VARIADIC_GENERICS}) + self.check_features_used("a[x, *y(), z] = t", {Feature.VARIADIC_GENERICS}) + self.check_features_used("def fn(*args: *T): pass", {Feature.VARIADIC_GENERICS}) + + self.check_features_used("with a: pass", set()) + self.check_features_used("with a, b: pass", set()) + self.check_features_used("with a as b: pass", set()) + self.check_features_used("with a as b, c as d: pass", set()) + self.check_features_used("with (a): pass", set()) + self.check_features_used("with (a, b): pass", set()) + self.check_features_used("with (a, b) as (c, d): pass", set()) + self.check_features_used( + "with (a as b): pass", {Feature.PARENTHESIZED_CONTEXT_MANAGERS} + ) + self.check_features_used( + "with ((a as b)): pass", {Feature.PARENTHESIZED_CONTEXT_MANAGERS} + ) + self.check_features_used( + "with (a, b as c): pass", {Feature.PARENTHESIZED_CONTEXT_MANAGERS} + ) + self.check_features_used( + "with (a, (b as c)): pass", {Feature.PARENTHESIZED_CONTEXT_MANAGERS} + ) + self.check_features_used( + "with ((a, ((b as c)))): pass", {Feature.PARENTHESIZED_CONTEXT_MANAGERS} + ) + + def check_features_used(self, source: str, expected: Set[Feature]) -> None: + node = black.lib2to3_parse(source) + actual = black.get_features_used(node) + msg = f"Expected {expected} but got {actual} for {source!r}" + try: + self.assertEqual(actual, expected, msg=msg) + except AssertionError: + DebugVisitor.show(node) + raise def test_get_features_used_for_future_flags(self) -> None: for src, features in [ @@ -1084,9 +1068,10 @@ def test_endmarker(self) -> None: self.assertEqual(len(n.children), 1) self.assertEqual(n.children[0].type, black.token.ENDMARKER) + @patch("tests.conftest.PRINT_FULL_TREE", True) + @patch("tests.conftest.PRINT_TREE_DIFF", False) @pytest.mark.incompatible_with_mypyc - @unittest.skipIf(os.environ.get("SKIP_AST_PRINT"), "user set SKIP_AST_PRINT") - def test_assertFormatEqual(self) -> None: + def test_assertFormatEqual_print_full_tree(self) -> None: out_lines = [] err_lines = [] @@ -1105,6 +1090,29 @@ def err(msg: str, **kwargs: Any) -> None: self.assertIn("Actual tree:", out_str) self.assertEqual("".join(err_lines), "") + @patch("tests.conftest.PRINT_FULL_TREE", False) + @patch("tests.conftest.PRINT_TREE_DIFF", True) + @pytest.mark.incompatible_with_mypyc + def test_assertFormatEqual_print_tree_diff(self) -> None: + out_lines = [] + err_lines = [] + + def out(msg: str, **kwargs: Any) -> None: + out_lines.append(msg) + + def err(msg: str, **kwargs: Any) -> None: + err_lines.append(msg) + + with patch("black.output._out", out), patch("black.output._err", err): + with self.assertRaises(AssertionError): + self.assertFormatEqual("j = [1, 2, 3]\n", "j = [1, 2, 3,]\n") + + out_str = "".join(out_lines) + self.assertIn("Tree Diff:", out_str) + self.assertIn("+ COMMA", out_str) + self.assertIn("+ ','", out_str) + self.assertEqual("".join(err_lines), "") + @event_loop() @patch("concurrent.futures.ProcessPoolExecutor", MagicMock(side_effect=OSError)) def test_works_in_mono_process_only_environment(self) -> None: @@ -1113,7 +1121,7 @@ def test_works_in_mono_process_only_environment(self) -> None: (workspace / "one.py").resolve(), (workspace / "two.py").resolve(), ]: - f.write_text('print("hello")\n') + f.write_text('print("hello")\n', encoding="utf-8") self.invokeBlack([str(workspace)]) @event_loop() @@ -1123,7 +1131,7 @@ def test_check_diff_use_together(self) -> None: src1 = get_case_path("miscellaneous", "string_quotes") self.invokeBlack([str(src1), "--diff", "--check"], exit_code=1) # Files which will not be reformatted. - src2 = get_case_path("simple_cases", "composition") + src2 = get_case_path("cases", "composition") self.invokeBlack([str(src2), "--diff", "--check"]) # Multi file command. self.invokeBlack([str(src1), str(src2), "--diff", "--check"], exit_code=1) @@ -1150,16 +1158,14 @@ def test_single_file_force_pyi(self) -> None: contents, expected = read_data("miscellaneous", "force_pyi") with cache_dir() as workspace: path = (workspace / "file.py").resolve() - with open(path, "w") as fh: - fh.write(contents) + path.write_text(contents, encoding="utf-8") self.invokeBlack([str(path), "--pyi"]) - with open(path, "r") as fh: - actual = fh.read() + actual = path.read_text(encoding="utf-8") # verify cache with --pyi is separate - pyi_cache = black.read_cache(pyi_mode) - self.assertIn(str(path), pyi_cache) - normal_cache = black.read_cache(DEFAULT_MODE) - self.assertNotIn(str(path), normal_cache) + pyi_cache = black.Cache.read(pyi_mode) + assert not pyi_cache.is_changed(path) + normal_cache = black.Cache.read(DEFAULT_MODE) + assert normal_cache.is_changed(path) self.assertFormatEqual(expected, actual) black.assert_equivalent(contents, actual) black.assert_stable(contents, actual, pyi_mode) @@ -1175,24 +1181,22 @@ def test_multi_file_force_pyi(self) -> None: (workspace / "file2.py").resolve(), ] for path in paths: - with open(path, "w") as fh: - fh.write(contents) + path.write_text(contents, encoding="utf-8") self.invokeBlack([str(p) for p in paths] + ["--pyi"]) for path in paths: - with open(path, "r") as fh: - actual = fh.read() + actual = path.read_text(encoding="utf-8") self.assertEqual(actual, expected) # verify cache with --pyi is separate - pyi_cache = black.read_cache(pyi_mode) - normal_cache = black.read_cache(reg_mode) + pyi_cache = black.Cache.read(pyi_mode) + normal_cache = black.Cache.read(reg_mode) for path in paths: - self.assertIn(str(path), pyi_cache) - self.assertNotIn(str(path), normal_cache) + assert not pyi_cache.is_changed(path) + assert normal_cache.is_changed(path) def test_pipe_force_pyi(self) -> None: source, expected = read_data("miscellaneous", "force_pyi") result = CliRunner().invoke( - black.main, ["-", "-q", "--pyi"], input=BytesIO(source.encode("utf8")) + black.main, ["-", "-q", "--pyi"], input=BytesIO(source.encode("utf-8")) ) self.assertEqual(result.exit_code, 0) actual = result.output @@ -1204,16 +1208,14 @@ def test_single_file_force_py36(self) -> None: source, expected = read_data("miscellaneous", "force_py36") with cache_dir() as workspace: path = (workspace / "file.py").resolve() - with open(path, "w") as fh: - fh.write(source) + path.write_text(source, encoding="utf-8") self.invokeBlack([str(path), *PY36_ARGS]) - with open(path, "r") as fh: - actual = fh.read() + actual = path.read_text(encoding="utf-8") # verify cache with --target-version is separate - py36_cache = black.read_cache(py36_mode) - self.assertIn(str(path), py36_cache) - normal_cache = black.read_cache(reg_mode) - self.assertNotIn(str(path), normal_cache) + py36_cache = black.Cache.read(py36_mode) + assert not py36_cache.is_changed(path) + normal_cache = black.Cache.read(reg_mode) + assert normal_cache.is_changed(path) self.assertEqual(actual, expected) @event_loop() @@ -1227,26 +1229,24 @@ def test_multi_file_force_py36(self) -> None: (workspace / "file2.py").resolve(), ] for path in paths: - with open(path, "w") as fh: - fh.write(source) + path.write_text(source, encoding="utf-8") self.invokeBlack([str(p) for p in paths] + PY36_ARGS) for path in paths: - with open(path, "r") as fh: - actual = fh.read() + actual = path.read_text(encoding="utf-8") self.assertEqual(actual, expected) # verify cache with --target-version is separate - pyi_cache = black.read_cache(py36_mode) - normal_cache = black.read_cache(reg_mode) + pyi_cache = black.Cache.read(py36_mode) + normal_cache = black.Cache.read(reg_mode) for path in paths: - self.assertIn(str(path), pyi_cache) - self.assertNotIn(str(path), normal_cache) + assert not pyi_cache.is_changed(path) + assert normal_cache.is_changed(path) def test_pipe_force_py36(self) -> None: source, expected = read_data("miscellaneous", "force_py36") result = CliRunner().invoke( black.main, ["-", "-q", "--target-version=py36"], - input=BytesIO(source.encode("utf8")), + input=BytesIO(source.encode("utf-8")), ) self.assertEqual(result.exit_code, 0) actual = result.output @@ -1288,7 +1288,7 @@ def test_reformat_one_with_stdin_filename(self) -> None: report=report, ) fsts.assert_called_once_with( - fast=True, write_back=black.WriteBack.YES, mode=DEFAULT_MODE + fast=True, write_back=black.WriteBack.YES, mode=DEFAULT_MODE, lines=() ) # __BLACK_STDIN_FILENAME__ should have been stripped report.done.assert_called_with(expected, black.Changed.YES) @@ -1314,6 +1314,7 @@ def test_reformat_one_with_stdin_filename_pyi(self) -> None: fast=True, write_back=black.WriteBack.YES, mode=replace(DEFAULT_MODE, is_pyi=True), + lines=(), ) # __BLACK_STDIN_FILENAME__ should have been stripped report.done.assert_called_with(expected, black.Changed.YES) @@ -1339,6 +1340,7 @@ def test_reformat_one_with_stdin_filename_ipynb(self) -> None: fast=True, write_back=black.WriteBack.YES, mode=replace(DEFAULT_MODE, is_ipynb=True), + lines=(), ) # __BLACK_STDIN_FILENAME__ should have been stripped report.done.assert_called_with(expected, black.Changed.YES) @@ -1352,7 +1354,7 @@ def test_reformat_one_with_stdin_and_existing_path(self) -> None: report = MagicMock() # Even with an existing file, since we are forcing stdin, black # should output to stdout and not modify the file inplace - p = THIS_DIR / "data" / "simple_cases" / "collections.py" + p = THIS_DIR / "data" / "cases" / "collections.py" # Make sure is_file actually returns True self.assertTrue(p.is_file()) path = Path(f"__BLACK_STDIN_FILENAME__{p}") @@ -1475,11 +1477,11 @@ def test_preserves_line_endings_via_stdin(self) -> None: contents = nl.join(["def f( ):", " pass"]) runner = BlackRunner() result = runner.invoke( - black.main, ["-", "--fast"], input=BytesIO(contents.encode("utf8")) + black.main, ["-", "--fast"], input=BytesIO(contents.encode("utf-8")) ) self.assertEqual(result.exit_code, 0) output = result.stdout_bytes - self.assertIn(nl.encode("utf8"), output) + self.assertIn(nl.encode("utf-8"), output) if nl == "\n": self.assertNotIn(b"\r\n", output) @@ -1498,30 +1500,6 @@ def test_assert_equivalent_different_asts(self) -> None: with self.assertRaises(AssertionError): black.assert_equivalent("{}", "None") - def test_shhh_click(self) -> None: - try: - from click import _unicodefun # type: ignore - except ImportError: - self.skipTest("Incompatible Click version") - - if not hasattr(_unicodefun, "_verify_python_env"): - self.skipTest("Incompatible Click version") - - # First, let's see if Click is crashing with a preferred ASCII charset. - with patch("locale.getpreferredencoding") as gpe: - gpe.return_value = "ASCII" - with self.assertRaises(RuntimeError): - _unicodefun._verify_python_env() - # Now, let's silence Click... - black.patch_click() - # ...and confirm it's silent. - with patch("locale.getpreferredencoding") as gpe: - gpe.return_value = "ASCII" - try: - _unicodefun._verify_python_env() - except RuntimeError as re: - self.fail(f"`patch_click()` failed, exception still raised: {re}") - def test_root_logger_not_used_directly(self) -> None: def fail(*args: Any, **kwargs: Any) -> None: self.fail("Record created with root logger") @@ -1575,14 +1553,25 @@ def test_infer_target_version(self) -> None: for version, expected in [ ("3.6", [TargetVersion.PY36]), ("3.11.0rc1", [TargetVersion.PY311]), - (">=3.10", [TargetVersion.PY310, TargetVersion.PY311]), - (">=3.10.6", [TargetVersion.PY310, TargetVersion.PY311]), + (">=3.10", [TargetVersion.PY310, TargetVersion.PY311, TargetVersion.PY312]), + ( + ">=3.10.6", + [TargetVersion.PY310, TargetVersion.PY311, TargetVersion.PY312], + ), ("<3.6", [TargetVersion.PY33, TargetVersion.PY34, TargetVersion.PY35]), (">3.7,<3.10", [TargetVersion.PY38, TargetVersion.PY39]), - (">3.7,!=3.8,!=3.9", [TargetVersion.PY310, TargetVersion.PY311]), + ( + ">3.7,!=3.8,!=3.9", + [TargetVersion.PY310, TargetVersion.PY311, TargetVersion.PY312], + ), ( "> 3.9.4, != 3.10.3", - [TargetVersion.PY39, TargetVersion.PY310, TargetVersion.PY311], + [ + TargetVersion.PY39, + TargetVersion.PY310, + TargetVersion.PY311, + TargetVersion.PY312, + ], ), ( "!=3.3,!=3.4", @@ -1594,6 +1583,7 @@ def test_infer_target_version(self) -> None: TargetVersion.PY39, TargetVersion.PY310, TargetVersion.PY311, + TargetVersion.PY312, ], ), ( @@ -1608,6 +1598,7 @@ def test_infer_target_version(self) -> None: TargetVersion.PY39, TargetVersion.PY310, TargetVersion.PY311, + TargetVersion.PY312, ], ), ("==3.8.*", [TargetVersion.PY38]), @@ -1640,6 +1631,39 @@ def test_read_pyproject_toml(self) -> None: self.assertEqual(config["exclude"], r"\.pyi?$") self.assertEqual(config["include"], r"\.py?$") + def test_read_pyproject_toml_from_stdin(self) -> None: + with TemporaryDirectory() as workspace: + root = Path(workspace) + + src_dir = root / "src" + src_dir.mkdir() + + src_pyproject = src_dir / "pyproject.toml" + src_pyproject.touch() + + test_toml_content = (THIS_DIR / "test.toml").read_text(encoding="utf-8") + src_pyproject.write_text(test_toml_content, encoding="utf-8") + + src_python = src_dir / "foo.py" + src_python.touch() + + fake_ctx = FakeContext() + fake_ctx.params["src"] = ("-",) + fake_ctx.params["stdin_filename"] = str(src_python) + + with change_directory(root): + black.read_pyproject_toml(fake_ctx, FakeParameter(), None) + + config = fake_ctx.default_map + self.assertEqual(config["verbose"], "1") + self.assertEqual(config["check"], "no") + self.assertEqual(config["diff"], "y") + self.assertEqual(config["color"], "True") + self.assertEqual(config["line_length"], "79") + self.assertEqual(config["target_version"], ["py36", "py37", "py38"]) + self.assertEqual(config["exclude"], r"\.pyi?$") + self.assertEqual(config["include"], r"\.py?$") + @pytest.mark.incompatible_with_mypyc def test_find_project_root(self) -> None: with TemporaryDirectory() as workspace: @@ -1770,7 +1794,7 @@ def test_bpo_2142_workaround(self) -> None: tmp_file = Path(black.dump_to_file(source, ensure_final_newline=False)) diff_header = re.compile( rf"{re.escape(str(tmp_file))}\t\d\d\d\d-\d\d-\d\d " - r"\d\d:\d\d:\d\d\.\d\d\d\d\d\d \+\d\d\d\d" + r"\d\d:\d\d:\d\d\.\d\d\d\d\d\d\+\d\d:\d\d" ) try: result = BlackRunner().invoke(black.main, ["--diff", str(tmp_file)]) @@ -1938,6 +1962,88 @@ def test_equivalency_ast_parse_failure_includes_error(self) -> None: err.match("invalid character") err.match(r"\(, line 1\)") + def test_line_ranges_with_code_option(self) -> None: + code = textwrap.dedent("""\ + if a == b: + print ( "OK" ) + """) + args = ["--line-ranges=1-1", "--code", code] + result = CliRunner().invoke(black.main, args) + + expected = textwrap.dedent("""\ + if a == b: + print ( "OK" ) + """) + self.compare_results(result, expected, expected_exit_code=0) + + def test_line_ranges_with_stdin(self) -> None: + code = textwrap.dedent("""\ + if a == b: + print ( "OK" ) + """) + runner = BlackRunner() + result = runner.invoke( + black.main, ["--line-ranges=1-1", "-"], input=BytesIO(code.encode("utf-8")) + ) + + expected = textwrap.dedent("""\ + if a == b: + print ( "OK" ) + """) + self.compare_results(result, expected, expected_exit_code=0) + + def test_line_ranges_with_source(self) -> None: + with TemporaryDirectory() as workspace: + test_file = Path(workspace) / "test.py" + test_file.write_text( + textwrap.dedent("""\ + if a == b: + print ( "OK" ) + """), + encoding="utf-8", + ) + args = ["--line-ranges=1-1", str(test_file)] + result = CliRunner().invoke(black.main, args) + assert not result.exit_code + + formatted = test_file.read_text(encoding="utf-8") + expected = textwrap.dedent("""\ + if a == b: + print ( "OK" ) + """) + assert expected == formatted + + def test_line_ranges_with_multiple_sources(self) -> None: + with TemporaryDirectory() as workspace: + test1_file = Path(workspace) / "test1.py" + test1_file.write_text("", encoding="utf-8") + test2_file = Path(workspace) / "test2.py" + test2_file.write_text("", encoding="utf-8") + args = ["--line-ranges=1-1", str(test1_file), str(test2_file)] + result = CliRunner().invoke(black.main, args) + assert result.exit_code == 1 + assert "Cannot use --line-ranges to format multiple files" in result.output + + def test_line_ranges_with_ipynb(self) -> None: + with TemporaryDirectory() as workspace: + test_file = Path(workspace) / "test.ipynb" + test_file.write_text("{}", encoding="utf-8") + args = ["--line-ranges=1-1", "--ipynb", str(test_file)] + result = CliRunner().invoke(black.main, args) + assert "Cannot use --line-ranges with ipynb files" in result.output + assert result.exit_code == 1 + + def test_line_ranges_in_pyproject_toml(self) -> None: + config = THIS_DIR / "data" / "invalid_line_ranges.toml" + result = BlackRunner().invoke( + black.main, ["--code", "print()", "--config", str(config)] + ) + assert result.exit_code == 2 + assert result.stderr_bytes is not None + assert ( + b"Cannot use line-ranges in the pyproject.toml file." in result.stderr_bytes + ) + class TestCaching: def test_get_cache_dir( @@ -1961,32 +2067,33 @@ def test_get_cache_dir( # If BLACK_CACHE_DIR is not set, use user_cache_dir monkeypatch.delenv("BLACK_CACHE_DIR", raising=False) with patch_user_cache_dir: - assert get_cache_dir() == workspace1 + assert get_cache_dir().parent == workspace1 # If it is set, use the path provided in the env var. monkeypatch.setenv("BLACK_CACHE_DIR", str(workspace2)) - assert get_cache_dir() == workspace2 + assert get_cache_dir().parent == workspace2 def test_cache_broken_file(self) -> None: mode = DEFAULT_MODE with cache_dir() as workspace: cache_file = get_cache_file(mode) - cache_file.write_text("this is not a pickle") - assert black.read_cache(mode) == {} + cache_file.write_text("this is not a pickle", encoding="utf-8") + assert black.Cache.read(mode).file_data == {} src = (workspace / "test.py").resolve() - src.write_text("print('hello')") + src.write_text("print('hello')", encoding="utf-8") invokeBlack([str(src)]) - cache = black.read_cache(mode) - assert str(src) in cache + cache = black.Cache.read(mode) + assert not cache.is_changed(src) def test_cache_single_file_already_cached(self) -> None: mode = DEFAULT_MODE with cache_dir() as workspace: src = (workspace / "test.py").resolve() - src.write_text("print('hello')") - black.write_cache({}, [src], mode) + src.write_text("print('hello')", encoding="utf-8") + cache = black.Cache.read(mode) + cache.write([src]) invokeBlack([str(src)]) - assert src.read_text() == "print('hello')" + assert src.read_text(encoding="utf-8") == "print('hello')" @event_loop() def test_cache_multiple_files(self) -> None: @@ -1995,30 +2102,27 @@ def test_cache_multiple_files(self) -> None: "concurrent.futures.ProcessPoolExecutor", new=ThreadPoolExecutor ): one = (workspace / "one.py").resolve() - with one.open("w") as fobj: - fobj.write("print('hello')") + one.write_text("print('hello')", encoding="utf-8") two = (workspace / "two.py").resolve() - with two.open("w") as fobj: - fobj.write("print('hello')") - black.write_cache({}, [one], mode) + two.write_text("print('hello')", encoding="utf-8") + cache = black.Cache.read(mode) + cache.write([one]) invokeBlack([str(workspace)]) - with one.open("r") as fobj: - assert fobj.read() == "print('hello')" - with two.open("r") as fobj: - assert fobj.read() == 'print("hello")\n' - cache = black.read_cache(mode) - assert str(one) in cache - assert str(two) in cache + assert one.read_text(encoding="utf-8") == "print('hello')" + assert two.read_text(encoding="utf-8") == 'print("hello")\n' + cache = black.Cache.read(mode) + assert not cache.is_changed(one) + assert not cache.is_changed(two) + @pytest.mark.incompatible_with_mypyc @pytest.mark.parametrize("color", [False, True], ids=["no-color", "with-color"]) def test_no_cache_when_writeback_diff(self, color: bool) -> None: mode = DEFAULT_MODE with cache_dir() as workspace: src = (workspace / "test.py").resolve() - with src.open("w") as fobj: - fobj.write("print('hello')") - with patch("black.read_cache") as read_cache, patch( - "black.write_cache" + src.write_text("print('hello')", encoding="utf-8") + with patch.object(black.Cache, "read") as read_cache, patch.object( + black.Cache, "write" ) as write_cache: cmd = [str(src), "--diff"] if color: @@ -2026,8 +2130,8 @@ def test_no_cache_when_writeback_diff(self, color: bool) -> None: invokeBlack(cmd) cache_file = get_cache_file(mode) assert cache_file.exists() is False + read_cache.assert_called_once() write_cache.assert_not_called() - read_cache.assert_not_called() @pytest.mark.parametrize("color", [False, True], ids=["no-color", "with-color"]) @event_loop() @@ -2035,8 +2139,7 @@ def test_output_locking_when_writeback_diff(self, color: bool) -> None: with cache_dir() as workspace: for tag in range(0, 4): src = (workspace / f"test{tag}.py").resolve() - with src.open("w") as fobj: - fobj.write("print('hello')") + src.write_text("print('hello')", encoding="utf-8") with patch( "black.concurrency.Manager", wraps=multiprocessing.Manager ) as mgr: @@ -2061,18 +2164,19 @@ def test_no_cache_when_stdin(self) -> None: def test_read_cache_no_cachefile(self) -> None: mode = DEFAULT_MODE with cache_dir(): - assert black.read_cache(mode) == {} + assert black.Cache.read(mode).file_data == {} def test_write_cache_read_cache(self) -> None: mode = DEFAULT_MODE with cache_dir() as workspace: src = (workspace / "test.py").resolve() src.touch() - black.write_cache({}, [src], mode) - cache = black.read_cache(mode) - assert str(src) in cache - assert cache[str(src)] == black.get_cache_info(src) + write_cache = black.Cache.read(mode) + write_cache.write([src]) + read_cache = black.Cache.read(mode) + assert not read_cache.is_changed(src) + @pytest.mark.incompatible_with_mypyc def test_filter_cached(self) -> None: with TemporaryDirectory() as workspace: path = Path(workspace) @@ -2082,21 +2186,67 @@ def test_filter_cached(self) -> None: uncached.touch() cached.touch() cached_but_changed.touch() - cache = { - str(cached): black.get_cache_info(cached), - str(cached_but_changed): (0.0, 0), - } - todo, done = black.cache.filter_cached( - cache, {uncached, cached, cached_but_changed} - ) + cache = black.Cache.read(DEFAULT_MODE) + + orig_func = black.Cache.get_file_data + + def wrapped_func(path: Path) -> FileData: + if path == cached: + return orig_func(path) + if path == cached_but_changed: + return FileData(0.0, 0, "") + raise AssertionError + + with patch.object(black.Cache, "get_file_data", side_effect=wrapped_func): + cache.write([cached, cached_but_changed]) + todo, done = cache.filtered_cached({uncached, cached, cached_but_changed}) assert todo == {uncached, cached_but_changed} assert done == {cached} + def test_filter_cached_hash(self) -> None: + with TemporaryDirectory() as workspace: + path = Path(workspace) + src = (path / "test.py").resolve() + src.write_text("print('hello')", encoding="utf-8") + st = src.stat() + cache = black.Cache.read(DEFAULT_MODE) + cache.write([src]) + cached_file_data = cache.file_data[str(src)] + + todo, done = cache.filtered_cached([src]) + assert todo == set() + assert done == {src} + assert cached_file_data.st_mtime == st.st_mtime + + # Modify st_mtime + cached_file_data = cache.file_data[str(src)] = FileData( + cached_file_data.st_mtime - 1, + cached_file_data.st_size, + cached_file_data.hash, + ) + todo, done = cache.filtered_cached([src]) + assert todo == set() + assert done == {src} + assert cached_file_data.st_mtime < st.st_mtime + assert cached_file_data.st_size == st.st_size + assert cached_file_data.hash == black.Cache.hash_digest(src) + + # Modify contents + src.write_text("print('hello world')", encoding="utf-8") + new_st = src.stat() + todo, done = cache.filtered_cached([src]) + assert todo == {src} + assert done == set() + assert cached_file_data.st_mtime < new_st.st_mtime + assert cached_file_data.st_size != new_st.st_size + assert cached_file_data.hash != black.Cache.hash_digest(src) + def test_write_cache_creates_directory_if_needed(self) -> None: mode = DEFAULT_MODE with cache_dir(exists=False) as workspace: assert not workspace.exists() - black.write_cache({}, [], mode) + cache = black.Cache.read(mode) + cache.write([]) assert workspace.exists() @event_loop() @@ -2106,21 +2256,21 @@ def test_failed_formatting_does_not_get_cached(self) -> None: "concurrent.futures.ProcessPoolExecutor", new=ThreadPoolExecutor ): failing = (workspace / "failing.py").resolve() - with failing.open("w") as fobj: - fobj.write("not actually python") + failing.write_text("not actually python", encoding="utf-8") clean = (workspace / "clean.py").resolve() - with clean.open("w") as fobj: - fobj.write('print("hello")\n') + clean.write_text('print("hello")\n', encoding="utf-8") invokeBlack([str(workspace)], exit_code=123) - cache = black.read_cache(mode) - assert str(failing) not in cache - assert str(clean) in cache + cache = black.Cache.read(mode) + assert cache.is_changed(failing) + assert not cache.is_changed(clean) def test_write_cache_write_fail(self) -> None: mode = DEFAULT_MODE - with cache_dir(), patch.object(Path, "open") as mock: - mock.side_effect = OSError - black.write_cache({}, [], mode) + with cache_dir(): + cache = black.Cache.read(mode) + with patch.object(Path, "open") as mock: + mock.side_effect = OSError + cache.write([]) def test_read_cache_line_lengths(self) -> None: mode = DEFAULT_MODE @@ -2128,18 +2278,19 @@ def test_read_cache_line_lengths(self) -> None: with cache_dir() as workspace: path = (workspace / "file.py").resolve() path.touch() - black.write_cache({}, [path], mode) - one = black.read_cache(mode) - assert str(path) in one - two = black.read_cache(short_mode) - assert str(path) not in two + cache = black.Cache.read(mode) + cache.write([path]) + one = black.Cache.read(mode) + assert not one.is_changed(path) + two = black.Cache.read(short_mode) + assert two.is_changed(path) def assert_collected_sources( src: Sequence[Union[str, Path]], expected: Sequence[Union[str, Path]], *, - ctx: Optional[FakeContext] = None, + root: Optional[Path] = None, exclude: Optional[str] = None, include: Optional[str] = None, extend_exclude: Optional[str] = None, @@ -2155,7 +2306,7 @@ def assert_collected_sources( ) gs_force_exclude = None if force_exclude is None else compile_pattern(force_exclude) collected = black.get_sources( - ctx=ctx or FakeContext(), + root=root or THIS_DIR, src=gs_src, quiet=False, verbose=False, @@ -2191,9 +2342,7 @@ def test_gitignore_used_as_default(self) -> None: base / "b/.definitely_exclude/a.pyi", ] src = [base / "b/"] - ctx = FakeContext() - ctx.obj["root"] = base - assert_collected_sources(src, expected, ctx=ctx, extend_exclude=r"/exclude/") + assert_collected_sources(src, expected, root=base, extend_exclude=r"/exclude/") def test_gitignore_used_on_multiple_sources(self) -> None: root = Path(DATA_DIR / "gitignore_used_on_multiple_sources") @@ -2201,10 +2350,8 @@ def test_gitignore_used_on_multiple_sources(self) -> None: root / "dir1" / "b.py", root / "dir2" / "b.py", ] - ctx = FakeContext() - ctx.obj["root"] = root src = [root / "dir1", root / "dir2"] - assert_collected_sources(src, expected, ctx=ctx) + assert_collected_sources(src, expected, root=root) @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) def test_exclude_for_issue_1572(self) -> None: @@ -2310,9 +2457,7 @@ def test_gitignore_that_ignores_subfolders(self) -> None: # If gitignore with */* is in root root = Path(DATA_DIR / "ignore_subfolders_gitignore_tests" / "subdir") expected = [root / "b.py"] - ctx = FakeContext() - ctx.obj["root"] = root - assert_collected_sources([root], expected, ctx=ctx) + assert_collected_sources([root], expected, root=root) # If .gitignore with */* is nested root = Path(DATA_DIR / "ignore_subfolders_gitignore_tests") @@ -2320,17 +2465,13 @@ def test_gitignore_that_ignores_subfolders(self) -> None: root / "a.py", root / "subdir" / "b.py", ] - ctx = FakeContext() - ctx.obj["root"] = root - assert_collected_sources([root], expected, ctx=ctx) + assert_collected_sources([root], expected, root=root) # If command is executed from outer dir root = Path(DATA_DIR / "ignore_subfolders_gitignore_tests") target = root / "subdir" expected = [target / "b.py"] - ctx = FakeContext() - ctx.obj["root"] = root - assert_collected_sources([target], expected, ctx=ctx) + assert_collected_sources([target], expected, root=root) def test_empty_include(self) -> None: path = DATA_DIR / "include_exclude_tests" @@ -2351,6 +2492,27 @@ def test_empty_include(self) -> None: # Setting exclude explicitly to an empty string to block .gitignore usage. assert_collected_sources(src, expected, include="", exclude="") + def test_include_absolute_path(self) -> None: + path = DATA_DIR / "include_exclude_tests" + src = [path] + expected = [ + Path(path / "b/dont_exclude/a.pie"), + ] + assert_collected_sources( + src, expected, root=path, include=r"^/b/dont_exclude/a\.pie$", exclude="" + ) + + def test_exclude_absolute_path(self) -> None: + path = DATA_DIR / "include_exclude_tests" + src = [path] + expected = [ + Path(path / "b/dont_exclude/a.py"), + Path(path / "b/.definitely_exclude/a.py"), + ] + assert_collected_sources( + src, expected, root=path, include=r"\.py$", exclude=r"^/b/exclude/a\.py$" + ) + def test_extend_exclude(self) -> None: path = DATA_DIR / "include_exclude_tests" src = [path] @@ -2363,38 +2525,72 @@ def test_extend_exclude(self) -> None: ) @pytest.mark.incompatible_with_mypyc - def test_symlink_out_of_root_directory(self) -> None: - path = MagicMock() + def test_symlinks(self) -> None: root = THIS_DIR.resolve() - child = MagicMock() include = re.compile(black.DEFAULT_INCLUDES) exclude = re.compile(black.DEFAULT_EXCLUDES) report = black.Report() gitignore = PathSpec.from_lines("gitwildmatch", []) - # `child` should behave like a symlink which resolved path is clearly - # outside of the `root` directory. - path.iterdir.return_value = [child] - child.resolve.return_value = Path("/a/b/c") - child.as_posix.return_value = "/a/b/c" - try: - list( - black.gen_python_files( - path.iterdir(), - root, - include, - exclude, - None, - None, - report, - {path: gitignore}, - verbose=False, - quiet=False, - ) + + regular = MagicMock() + regular.absolute.return_value = root / "regular.py" + regular.resolve.return_value = root / "regular.py" + regular.is_dir.return_value = False + regular.is_file.return_value = True + + outside_root_symlink = MagicMock() + outside_root_symlink.absolute.return_value = root / "symlink.py" + outside_root_symlink.resolve.return_value = Path("/nowhere") + outside_root_symlink.is_dir.return_value = False + outside_root_symlink.is_file.return_value = True + + ignored_symlink = MagicMock() + ignored_symlink.absolute.return_value = root / ".mypy_cache" / "symlink.py" + ignored_symlink.is_dir.return_value = False + ignored_symlink.is_file.return_value = True + + # A symlink that has an excluded name, but points to an included name + symlink_excluded_name = MagicMock() + symlink_excluded_name.absolute.return_value = root / "excluded_name" + symlink_excluded_name.resolve.return_value = root / "included_name.py" + symlink_excluded_name.is_dir.return_value = False + symlink_excluded_name.is_file.return_value = True + + # A symlink that has an included name, but points to an excluded name + symlink_included_name = MagicMock() + symlink_included_name.absolute.return_value = root / "included_name.py" + symlink_included_name.resolve.return_value = root / "excluded_name" + symlink_included_name.is_dir.return_value = False + symlink_included_name.is_file.return_value = True + + path = MagicMock() + path.iterdir.return_value = [ + regular, + outside_root_symlink, + ignored_symlink, + symlink_excluded_name, + symlink_included_name, + ] + + files = list( + black.gen_python_files( + path.iterdir(), + root, + include, + exclude, + None, + None, + report, + {path: gitignore}, + verbose=False, + quiet=False, ) - except ValueError as ve: - pytest.fail(f"`get_python_files_in_dir()` failed: {ve}") + ) + assert files == [regular, symlink_included_name] + path.iterdir.assert_called_once() - child.resolve.assert_called_once() + outside_root_symlink.resolve.assert_called_once() + ignored_symlink.resolve.assert_not_called() @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) def test_get_sources_with_stdin(self) -> None: @@ -2459,6 +2655,58 @@ def test_get_sources_with_stdin_filename_and_force_exclude(self) -> None: stdin_filename=stdin_filename, ) + @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) + def test_get_sources_with_stdin_filename_and_force_exclude_and_symlink( + self, + ) -> None: + # Force exclude should exclude a symlink based on the symlink, not its target + path = THIS_DIR / "data" / "include_exclude_tests" + stdin_filename = str(path / "symlink.py") + expected = [f"__BLACK_STDIN_FILENAME__{stdin_filename}"] + target = path / "b/exclude/a.py" + with patch("pathlib.Path.resolve", return_value=target): + assert_collected_sources( + src=["-"], + expected=expected, + force_exclude=r"exclude/a\.py", + stdin_filename=stdin_filename, + ) + + +class TestDeFactoAPI: + """Test that certain symbols that are commonly used externally keep working. + + We don't (yet) formally expose an API (see issue #779), but we should endeavor to + keep certain functions that external users commonly rely on working. + + """ + + def test_format_str(self) -> None: + # format_str and Mode should keep working + assert ( + black.format_str("print('hello')", mode=black.Mode()) == 'print("hello")\n' + ) + + # you can pass line length + assert ( + black.format_str("print('hello')", mode=black.Mode(line_length=42)) + == 'print("hello")\n' + ) + + # invalid input raises InvalidInput + with pytest.raises(black.InvalidInput): + black.format_str("syntax error", mode=black.Mode()) + + def test_format_file_contents(self) -> None: + # You probably should be using format_str() instead, but let's keep + # this one around since people do use it + assert ( + black.format_file_contents("x=1", fast=True, mode=black.Mode()) == "x = 1\n" + ) + + with pytest.raises(black.NothingChanged): + black.format_file_contents("x = 1\n", fast=True, mode=black.Mode()) + try: with open(black.__file__, "r", encoding="utf-8") as _bf: diff --git a/tests/test_blackd.py b/tests/test_blackd.py index 5b6461f7685..59703036dc0 100644 --- a/tests/test_blackd.py +++ b/tests/test_blackd.py @@ -31,7 +31,7 @@ def unittest_run_loop(func, *args, **kwargs): @pytest.mark.blackd -class BlackDTestCase(AioHTTPTestCase): # type: ignore[misc] +class BlackDTestCase(AioHTTPTestCase): def test_blackd_main(self) -> None: with patch("blackd.web.run_app"): result = CliRunner().invoke(blackd.main, []) @@ -104,7 +104,7 @@ async def check(header_value: str, expected_status: int = 400) -> None: @unittest_run_loop async def test_blackd_pyi(self) -> None: - source, expected = read_data("miscellaneous", "stub.pyi") + source, expected = read_data("cases", "stub.py") response = await self.client.post( "/", data=source, headers={blackd.PYTHON_VARIANT_HEADER: "pyi"} ) @@ -114,7 +114,7 @@ async def test_blackd_pyi(self) -> None: @unittest_run_loop async def test_blackd_diff(self) -> None: diff_header = re.compile( - r"(In|Out)\t\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d\.\d\d\d\d\d\d \+\d\d\d\d" + r"(In|Out)\t\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d\.\d\d\d\d\d\d\+\d\d:\d\d" ) source, _ = read_data("miscellaneous", "blackd_diff") @@ -240,3 +240,9 @@ async def test_normalizes_line_endings(self) -> None: response = await self.client.post("/", data=data) self.assertEqual(await response.text(), expected) self.assertEqual(response.status, 200) + + @unittest_run_loop + async def test_single_character(self) -> None: + response = await self.client.post("/", data="1") + self.assertEqual(await response.text(), "1\n") + self.assertEqual(response.status, 200) diff --git a/tests/test_format.py b/tests/test_format.py index ab849aac9a3..9162c585c08 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -1,4 +1,3 @@ -import re from dataclasses import replace from typing import Any, Iterator from unittest.mock import patch @@ -6,13 +5,13 @@ import pytest import black +from black.mode import TargetVersion from tests.util import ( - DEFAULT_MODE, - PY36_VERSIONS, all_data_cases, assert_format, dump_to_stderr, read_data, + read_data_with_mode, ) @@ -22,53 +21,59 @@ def patch_dump_to_file(request: Any) -> Iterator[None]: yield -def check_file( - subdir: str, filename: str, mode: black.Mode, *, data: bool = True -) -> None: - source, expected = read_data(subdir, filename, data=data) - assert_format(source, expected, mode, fast=False) +def check_file(subdir: str, filename: str, *, data: bool = True) -> None: + args, source, expected = read_data_with_mode(subdir, filename, data=data) + assert_format( + source, + expected, + args.mode, + fast=args.fast, + minimum_version=args.minimum_version, + lines=args.lines, + no_preview_line_length_1=args.no_preview_line_length_1, + ) + if args.minimum_version is not None: + major, minor = args.minimum_version + target_version = TargetVersion[f"PY{major}{minor}"] + mode = replace(args.mode, target_versions={target_version}) + assert_format( + source, + expected, + mode, + fast=args.fast, + minimum_version=args.minimum_version, + lines=args.lines, + no_preview_line_length_1=args.no_preview_line_length_1, + ) @pytest.mark.filterwarnings("ignore:invalid escape sequence.*:DeprecationWarning") -@pytest.mark.parametrize("filename", all_data_cases("simple_cases")) +@pytest.mark.parametrize("filename", all_data_cases("cases")) def test_simple_format(filename: str) -> None: - magic_trailing_comma = filename != "skip_magic_trailing_comma" - check_file( - "simple_cases", filename, black.Mode(magic_trailing_comma=magic_trailing_comma) - ) - - -@pytest.mark.parametrize("filename", all_data_cases("preview")) -def test_preview_format(filename: str) -> None: - check_file("preview", filename, black.Mode(preview=True)) - - -def test_preview_context_managers_targeting_py38() -> None: - source, expected = read_data("preview_context_managers", "targeting_py38.py") - mode = black.Mode(preview=True, target_versions={black.TargetVersion.PY38}) - assert_format(source, expected, mode, minimum_version=(3, 8)) - - -def test_preview_context_managers_targeting_py39() -> None: - source, expected = read_data("preview_context_managers", "targeting_py39.py") - mode = black.Mode(preview=True, target_versions={black.TargetVersion.PY39}) - assert_format(source, expected, mode, minimum_version=(3, 9)) - - -@pytest.mark.parametrize( - "filename", all_data_cases("preview_context_managers/auto_detect") -) -def test_preview_context_managers_auto_detect(filename: str) -> None: - match = re.match(r"features_3_(\d+)", filename) - assert match is not None, "Unexpected filename format: %s" % filename - source, expected = read_data("preview_context_managers/auto_detect", filename) - mode = black.Mode(preview=True) - assert_format(source, expected, mode, minimum_version=(3, int(match.group(1)))) + check_file("cases", filename) + + +@pytest.mark.parametrize("filename", all_data_cases("line_ranges_formatted")) +def test_line_ranges_line_by_line(filename: str) -> None: + args, source, expected = read_data_with_mode("line_ranges_formatted", filename) + assert ( + source == expected + ), "Test cases in line_ranges_formatted must already be formatted." + line_count = len(source.splitlines()) + for line in range(1, line_count + 1): + assert_format( + source, + expected, + args.mode, + fast=args.fast, + minimum_version=args.minimum_version, + lines=[(line, line)], + ) # =============== # -# Complex cases -# ============= # +# Unusual cases +# =============== # def test_empty() -> None: @@ -76,48 +81,6 @@ def test_empty() -> None: assert_format(source, expected) -@pytest.mark.parametrize("filename", all_data_cases("py_36")) -def test_python_36(filename: str) -> None: - source, expected = read_data("py_36", filename) - mode = black.Mode(target_versions=PY36_VERSIONS) - assert_format(source, expected, mode, minimum_version=(3, 6)) - - -@pytest.mark.parametrize("filename", all_data_cases("py_37")) -def test_python_37(filename: str) -> None: - source, expected = read_data("py_37", filename) - mode = black.Mode(target_versions={black.TargetVersion.PY37}) - assert_format(source, expected, mode, minimum_version=(3, 7)) - - -@pytest.mark.parametrize("filename", all_data_cases("py_38")) -def test_python_38(filename: str) -> None: - source, expected = read_data("py_38", filename) - mode = black.Mode(target_versions={black.TargetVersion.PY38}) - assert_format(source, expected, mode, minimum_version=(3, 8)) - - -@pytest.mark.parametrize("filename", all_data_cases("py_39")) -def test_python_39(filename: str) -> None: - source, expected = read_data("py_39", filename) - mode = black.Mode(target_versions={black.TargetVersion.PY39}) - assert_format(source, expected, mode, minimum_version=(3, 9)) - - -@pytest.mark.parametrize("filename", all_data_cases("py_310")) -def test_python_310(filename: str) -> None: - source, expected = read_data("py_310", filename) - mode = black.Mode(target_versions={black.TargetVersion.PY310}) - assert_format(source, expected, mode, minimum_version=(3, 10)) - - -@pytest.mark.parametrize("filename", all_data_cases("py_310")) -def test_python_310_without_target_version(filename: str) -> None: - source, expected = read_data("py_310", filename) - mode = black.Mode() - assert_format(source, expected, mode, minimum_version=(3, 10)) - - def test_patma_invalid() -> None: source, expected = read_data("miscellaneous", "pattern_matching_invalid") mode = black.Mode(target_versions={black.TargetVersion.PY310}) @@ -125,68 +88,3 @@ def test_patma_invalid() -> None: assert_format(source, expected, mode, minimum_version=(3, 10)) exc_info.match("Cannot parse: 10:11") - - -@pytest.mark.parametrize("filename", all_data_cases("py_311")) -def test_python_311(filename: str) -> None: - source, expected = read_data("py_311", filename) - mode = black.Mode(target_versions={black.TargetVersion.PY311}) - assert_format(source, expected, mode, minimum_version=(3, 11)) - - -@pytest.mark.parametrize("filename", all_data_cases("fast")) -def test_fast_cases(filename: str) -> None: - source, expected = read_data("fast", filename) - assert_format(source, expected, fast=True) - - -def test_python_2_hint() -> None: - with pytest.raises(black.parsing.InvalidInput) as exc_info: - assert_format("print 'daylily'", "print 'daylily'") - exc_info.match(black.parsing.PY2_HINT) - - -@pytest.mark.filterwarnings("ignore:invalid escape sequence.*:DeprecationWarning") -def test_docstring_no_string_normalization() -> None: - """Like test_docstring but with string normalization off.""" - source, expected = read_data("miscellaneous", "docstring_no_string_normalization") - mode = replace(DEFAULT_MODE, string_normalization=False) - assert_format(source, expected, mode) - - -def test_docstring_line_length_6() -> None: - """Like test_docstring but with line length set to 6.""" - source, expected = read_data("miscellaneous", "linelength6") - mode = black.Mode(line_length=6) - assert_format(source, expected, mode) - - -def test_preview_docstring_no_string_normalization() -> None: - """ - Like test_docstring but with string normalization off *and* the preview style - enabled. - """ - source, expected = read_data( - "miscellaneous", "docstring_preview_no_string_normalization" - ) - mode = replace(DEFAULT_MODE, string_normalization=False, preview=True) - assert_format(source, expected, mode) - - -def test_long_strings_flag_disabled() -> None: - """Tests for turning off the string processing logic.""" - source, expected = read_data("miscellaneous", "long_strings_flag_disabled") - mode = replace(DEFAULT_MODE, experimental_string_processing=False) - assert_format(source, expected, mode) - - -def test_stub() -> None: - mode = replace(DEFAULT_MODE, is_pyi=True) - source, expected = read_data("miscellaneous", "stub.pyi") - assert_format(source, expected, mode) - - -def test_power_op_newline() -> None: - # requires line_length=0 - source, expected = read_data("miscellaneous", "power_op_newline") - assert_format(source, expected, mode=black.Mode(line_length=0)) diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 7aa2e91dd00..59897190304 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -77,7 +77,7 @@ def test_trailing_semicolon_noop() -> None: [ pytest.param(JUPYTER_MODE, id="default mode"), pytest.param( - replace(JUPYTER_MODE, python_cell_magics={"cust1", "cust1"}), + replace(JUPYTER_MODE, python_cell_magics={"cust1", "cust2"}), id="custom cell magics mode", ), ], @@ -100,7 +100,7 @@ def test_cell_magic_noop() -> None: [ pytest.param(JUPYTER_MODE, id="default mode"), pytest.param( - replace(JUPYTER_MODE, python_cell_magics={"cust1", "cust1"}), + replace(JUPYTER_MODE, python_cell_magics={"cust1", "cust2"}), id="custom cell magics mode", ), ], @@ -183,7 +183,7 @@ def test_cell_magic_with_magic() -> None: id="No change when cell magic not registered", ), pytest.param( - replace(JUPYTER_MODE, python_cell_magics={"cust1", "cust1"}), + replace(JUPYTER_MODE, python_cell_magics={"cust1", "cust2"}), "%%custom_python_magic -n1 -n2\nx=2", pytest.raises(NothingChanged), id="No change when other cell magics registered", @@ -439,19 +439,14 @@ def test_cache_isnt_written_if_no_jupyter_deps_single( jupyter_dependencies_are_installed.cache_clear() nb = get_case_path("jupyter", "notebook_trailing_newline.ipynb") tmp_nb = tmp_path / "notebook.ipynb" - with open(nb) as src, open(tmp_nb, "w") as dst: - dst.write(src.read()) - monkeypatch.setattr( - "black.jupyter_dependencies_are_installed", lambda verbose, quiet: False - ) + tmp_nb.write_bytes(nb.read_bytes()) + monkeypatch.setattr("black.jupyter_dependencies_are_installed", lambda warn: False) result = runner.invoke( main, [str(tmp_path / "notebook.ipynb"), f"--config={EMPTY_CONFIG}"] ) assert "No Python files are present to be formatted. Nothing to do" in result.output jupyter_dependencies_are_installed.cache_clear() - monkeypatch.setattr( - "black.jupyter_dependencies_are_installed", lambda verbose, quiet: True - ) + monkeypatch.setattr("black.jupyter_dependencies_are_installed", lambda warn: True) result = runner.invoke( main, [str(tmp_path / "notebook.ipynb"), f"--config={EMPTY_CONFIG}"] ) @@ -465,16 +460,15 @@ def test_cache_isnt_written_if_no_jupyter_deps_dir( jupyter_dependencies_are_installed.cache_clear() nb = get_case_path("jupyter", "notebook_trailing_newline.ipynb") tmp_nb = tmp_path / "notebook.ipynb" - with open(nb) as src, open(tmp_nb, "w") as dst: - dst.write(src.read()) + tmp_nb.write_bytes(nb.read_bytes()) monkeypatch.setattr( - "black.files.jupyter_dependencies_are_installed", lambda verbose, quiet: False + "black.files.jupyter_dependencies_are_installed", lambda warn: False ) result = runner.invoke(main, [str(tmp_path), f"--config={EMPTY_CONFIG}"]) assert "No Python files are present to be formatted. Nothing to do" in result.output jupyter_dependencies_are_installed.cache_clear() monkeypatch.setattr( - "black.files.jupyter_dependencies_are_installed", lambda verbose, quiet: True + "black.files.jupyter_dependencies_are_installed", lambda warn: True ) result = runner.invoke(main, [str(tmp_path), f"--config={EMPTY_CONFIG}"]) assert "reformatted" in result.output @@ -483,8 +477,7 @@ def test_cache_isnt_written_if_no_jupyter_deps_dir( def test_ipynb_flag(tmp_path: pathlib.Path) -> None: nb = get_case_path("jupyter", "notebook_trailing_newline.ipynb") tmp_nb = tmp_path / "notebook.a_file_extension_which_is_definitely_not_ipynb" - with open(nb) as src, open(tmp_nb, "w") as dst: - dst.write(src.read()) + tmp_nb.write_bytes(nb.read_bytes()) result = runner.invoke( main, [ diff --git a/tests/test_no_ipynb.py b/tests/test_no_ipynb.py index b63ecde8896..12c820def39 100644 --- a/tests/test_no_ipynb.py +++ b/tests/test_no_ipynb.py @@ -27,8 +27,7 @@ def test_ipynb_diff_with_no_change_dir(tmp_path: pathlib.Path) -> None: runner = CliRunner() nb = get_case_path("jupyter", "notebook_trailing_newline.ipynb") tmp_nb = tmp_path / "notebook.ipynb" - with open(nb) as src, open(tmp_nb, "w") as dst: - dst.write(src.read()) + tmp_nb.write_bytes(nb.read_bytes()) result = runner.invoke(main, [str(tmp_path)]) expected_output = ( "Skipping .ipynb files as Jupyter dependencies are not installed.\n" diff --git a/tests/test_ranges.py b/tests/test_ranges.py new file mode 100644 index 00000000000..d9fa9171a7f --- /dev/null +++ b/tests/test_ranges.py @@ -0,0 +1,185 @@ +"""Test the black.ranges module.""" + +from typing import List, Tuple + +import pytest + +from black.ranges import adjusted_lines + + +@pytest.mark.parametrize( + "lines", + [[(1, 1)], [(1, 3)], [(1, 1), (3, 4)]], +) +def test_no_diff(lines: List[Tuple[int, int]]) -> None: + source = """\ +import re + +def func(): +pass +""" + assert lines == adjusted_lines(lines, source, source) + + +@pytest.mark.parametrize( + "lines", + [ + [(1, 0)], + [(-8, 0)], + [(-8, 8)], + [(1, 100)], + [(2, 1)], + [(0, 8), (3, 1)], + ], +) +def test_invalid_lines(lines: List[Tuple[int, int]]) -> None: + original_source = """\ +import re +def foo(arg): +'''This is the foo function. + +This is foo function's +docstring with more descriptive texts. +''' + +def func(arg1, +arg2, arg3): +pass +""" + modified_source = """\ +import re +def foo(arg): +'''This is the foo function. + +This is foo function's +docstring with more descriptive texts. +''' + +def func(arg1, arg2, arg3): +pass +""" + assert not adjusted_lines(lines, original_source, modified_source) + + +@pytest.mark.parametrize( + "lines,adjusted", + [ + ( + [(1, 1)], + [(1, 1)], + ), + ( + [(1, 2)], + [(1, 1)], + ), + ( + [(1, 6)], + [(1, 2)], + ), + ( + [(6, 6)], + [], + ), + ], +) +def test_removals( + lines: List[Tuple[int, int]], adjusted: List[Tuple[int, int]] +) -> None: + original_source = """\ +1. first line +2. second line +3. third line +4. fourth line +5. fifth line +6. sixth line +""" + modified_source = """\ +2. second line +5. fifth line +""" + assert adjusted == adjusted_lines(lines, original_source, modified_source) + + +@pytest.mark.parametrize( + "lines,adjusted", + [ + ( + [(1, 1)], + [(2, 2)], + ), + ( + [(1, 2)], + [(2, 5)], + ), + ( + [(2, 2)], + [(5, 5)], + ), + ], +) +def test_additions( + lines: List[Tuple[int, int]], adjusted: List[Tuple[int, int]] +) -> None: + original_source = """\ +1. first line +2. second line +""" + modified_source = """\ +this is added +1. first line +this is added +this is added +2. second line +this is added +""" + assert adjusted == adjusted_lines(lines, original_source, modified_source) + + +@pytest.mark.parametrize( + "lines,adjusted", + [ + ( + [(1, 11)], + [(1, 10)], + ), + ( + [(1, 12)], + [(1, 11)], + ), + ( + [(10, 10)], + [(9, 9)], + ), + ([(1, 1), (9, 10)], [(1, 1), (9, 9)]), + ([(9, 10), (1, 1)], [(1, 1), (9, 9)]), + ], +) +def test_diffs(lines: List[Tuple[int, int]], adjusted: List[Tuple[int, int]]) -> None: + original_source = """\ + 1. import re + 2. def foo(arg): + 3. '''This is the foo function. + 4. + 5. This is foo function's + 6. docstring with more descriptive texts. + 7. ''' + 8. + 9. def func(arg1, +10. arg2, arg3): +11. pass +12. # last line +""" + modified_source = """\ + 1. import re # changed + 2. def foo(arg): + 3. '''This is the foo function. + 4. + 5. This is foo function's + 6. docstring with more descriptive texts. + 7. ''' + 8. + 9. def func(arg1, arg2, arg3): +11. pass +12. # last line changed +""" + assert adjusted == adjusted_lines(lines, original_source, modified_source) diff --git a/tests/test_trans.py b/tests/test_trans.py index dce8a939677..784e852e12a 100644 --- a/tests/test_trans.py +++ b/tests/test_trans.py @@ -13,7 +13,7 @@ def check( # a glance than only spans assert len(spans) == len(expected_slices) for (i, j), slice in zip(spans, expected_slices): - assert len(string[i:j]) == j - i + assert 0 <= i <= j <= len(string) assert string[i:j] == slice assert spans == expected_spans diff --git a/tests/util.py b/tests/util.py index 967d576fafe..9ea30e62fe3 100644 --- a/tests/util.py +++ b/tests/util.py @@ -1,16 +1,23 @@ +import argparse +import functools import os +import shlex import sys import unittest from contextlib import contextmanager -from dataclasses import replace +from dataclasses import dataclass, field, replace from functools import partial from pathlib import Path -from typing import Any, Iterator, List, Optional, Tuple +from typing import Any, Collection, Iterator, List, Optional, Tuple import black +from black.const import DEFAULT_LINE_LENGTH from black.debug import DebugVisitor from black.mode import TargetVersion from black.output import diff, err, out +from black.ranges import parse_line_ranges + +from . import conftest PYTHON_SUFFIX = ".py" ALLOWED_SUFFIXES = (PYTHON_SUFFIX, ".pyi", ".out", ".diff", ".ipynb") @@ -33,23 +40,44 @@ fs = partial(black.format_str, mode=DEFAULT_MODE) +@dataclass +class TestCaseArgs: + mode: black.Mode = field(default_factory=black.Mode) + fast: bool = False + minimum_version: Optional[Tuple[int, int]] = None + lines: Collection[Tuple[int, int]] = () + no_preview_line_length_1: bool = False + + def _assert_format_equal(expected: str, actual: str) -> None: - if actual != expected and not os.environ.get("SKIP_AST_PRINT"): + if actual != expected and (conftest.PRINT_FULL_TREE or conftest.PRINT_TREE_DIFF): bdv: DebugVisitor[Any] - out("Expected tree:", fg="green") + actual_out: str = "" + expected_out: str = "" + if conftest.PRINT_FULL_TREE: + out("Expected tree:", fg="green") try: exp_node = black.lib2to3_parse(expected) - bdv = DebugVisitor() + bdv = DebugVisitor(print_output=conftest.PRINT_FULL_TREE) list(bdv.visit(exp_node)) + expected_out = "\n".join(bdv.list_output) except Exception as ve: err(str(ve)) - out("Actual tree:", fg="red") + if conftest.PRINT_FULL_TREE: + out("Actual tree:", fg="red") try: exp_node = black.lib2to3_parse(actual) - bdv = DebugVisitor() + bdv = DebugVisitor(print_output=conftest.PRINT_FULL_TREE) list(bdv.visit(exp_node)) + actual_out = "\n".join(bdv.list_output) except Exception as ve: err(str(ve)) + if conftest.PRINT_TREE_DIFF: + out("Tree Diff:") + out( + diff(expected_out, actual_out, "expected tree", "actual tree") + or "Trees do not differ" + ) if actual != expected: out(diff(expected, actual, "expected", "actual")) @@ -68,6 +96,8 @@ def assert_format( *, fast: bool = False, minimum_version: Optional[Tuple[int, int]] = None, + lines: Collection[Tuple[int, int]] = (), + no_preview_line_length_1: bool = False, ) -> None: """Convenience function to check that Black formats as expected. @@ -76,7 +106,7 @@ def assert_format( separate from TargetVerson Mode configuration. """ _assert_format_inner( - source, expected, mode, fast=fast, minimum_version=minimum_version + source, expected, mode, fast=fast, minimum_version=minimum_version, lines=lines ) # For both preview and non-preview tests, ensure that Black doesn't crash on @@ -88,6 +118,7 @@ def assert_format( replace(mode, preview=not mode.preview), fast=fast, minimum_version=minimum_version, + lines=lines, ) except Exception as e: text = "non-preview" if mode.preview else "preview" @@ -95,20 +126,28 @@ def assert_format( f"Black crashed formatting this case in {text} mode." ) from e # Similarly, setting line length to 1 is a good way to catch - # stability bugs. But only in non-preview mode because preview mode - # currently has a lot of line length 1 bugs. - try: - _assert_format_inner( - source, - None, - replace(mode, preview=False, line_length=1), - fast=fast, - minimum_version=minimum_version, - ) - except Exception as e: - raise FormatFailure( - "Black crashed formatting this case with line-length set to 1." - ) from e + # stability bugs. Some tests are known to be broken in preview mode with line length + # of 1 though, and have marked that with a flag --no-preview-line-length-1 + preview_modes = [False] + if not no_preview_line_length_1: + preview_modes.append(True) + + for preview_mode in preview_modes: + + try: + _assert_format_inner( + source, + None, + replace(mode, preview=preview_mode, line_length=1), + fast=fast, + minimum_version=minimum_version, + lines=lines, + ) + except Exception as e: + text = "preview" if preview_mode else "non-preview" + raise FormatFailure( + f"Black crashed formatting this case in {text} mode with line-length=1." + ) from e def _assert_format_inner( @@ -118,8 +157,9 @@ def _assert_format_inner( *, fast: bool = False, minimum_version: Optional[Tuple[int, int]] = None, + lines: Collection[Tuple[int, int]] = (), ) -> None: - actual = black.format_str(source, mode=mode) + actual = black.format_str(source, mode=mode, lines=lines) if expected is not None: _assert_format_equal(expected, actual) # It's not useful to run safety checks if we're expecting no changes anyway. The @@ -131,7 +171,7 @@ def _assert_format_inner( # when checking modern code on older versions. if minimum_version is None or sys.version_info >= minimum_version: black.assert_equivalent(source, actual) - black.assert_stable(source, actual, mode=mode) + black.assert_stable(source, actual, mode=mode, lines=lines) def dump_to_stderr(*output: str) -> str: @@ -164,18 +204,111 @@ def get_case_path( return case_path +def read_data_with_mode( + subdir_name: str, name: str, data: bool = True +) -> Tuple[TestCaseArgs, str, str]: + """read_data_with_mode('test_name') -> Mode(), 'input', 'output'""" + return read_data_from_file(get_case_path(subdir_name, name, data)) + + def read_data(subdir_name: str, name: str, data: bool = True) -> Tuple[str, str]: """read_data('test_name') -> 'input', 'output'""" - return read_data_from_file(get_case_path(subdir_name, name, data)) + _, input, output = read_data_with_mode(subdir_name, name, data) + return input, output + + +def _parse_minimum_version(version: str) -> Tuple[int, int]: + major, minor = version.split(".") + return int(major), int(minor) + + +@functools.lru_cache() +def get_flags_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + parser.add_argument( + "--target-version", + action="append", + type=lambda val: TargetVersion[val.upper()], + default=(), + ) + parser.add_argument("--line-length", default=DEFAULT_LINE_LENGTH, type=int) + parser.add_argument( + "--skip-string-normalization", default=False, action="store_true" + ) + parser.add_argument("--pyi", default=False, action="store_true") + parser.add_argument("--ipynb", default=False, action="store_true") + parser.add_argument( + "--skip-magic-trailing-comma", default=False, action="store_true" + ) + parser.add_argument("--preview", default=False, action="store_true") + parser.add_argument("--fast", default=False, action="store_true") + parser.add_argument( + "--minimum-version", + type=_parse_minimum_version, + default=None, + help=( + "Minimum version of Python where this test case is parseable. If this is" + " set, the test case will be run twice: once with the specified" + " --target-version, and once with --target-version set to exactly the" + " specified version. This ensures that Black's autodetection of the target" + " version works correctly." + ), + ) + parser.add_argument("--line-ranges", action="append") + parser.add_argument( + "--no-preview-line-length-1", + default=False, + action="store_true", + help=( + "Don't run in preview mode with --line-length=1, as that's known to cause a" + " crash" + ), + ) + return parser + + +def parse_mode(flags_line: str) -> TestCaseArgs: + parser = get_flags_parser() + args = parser.parse_args(shlex.split(flags_line)) + mode = black.Mode( + target_versions=set(args.target_version), + line_length=args.line_length, + string_normalization=not args.skip_string_normalization, + is_pyi=args.pyi, + is_ipynb=args.ipynb, + magic_trailing_comma=not args.skip_magic_trailing_comma, + preview=args.preview, + ) + if args.line_ranges: + lines = parse_line_ranges(args.line_ranges) + else: + lines = [] + return TestCaseArgs( + mode=mode, + fast=args.fast, + minimum_version=args.minimum_version, + lines=lines, + no_preview_line_length_1=args.no_preview_line_length_1, + ) -def read_data_from_file(file_name: Path) -> Tuple[str, str]: +def read_data_from_file(file_name: Path) -> Tuple[TestCaseArgs, str, str]: with open(file_name, "r", encoding="utf8") as test: lines = test.readlines() _input: List[str] = [] _output: List[str] = [] result = _input + mode = TestCaseArgs() for line in lines: + if not _input and line.startswith("# flags: "): + mode = parse_mode(line[len("# flags: ") :]) + if mode.lines: + # Retain the `# flags: ` line when using --line-ranges=. This requires + # the `# output` section to also include this line, but retaining the + # line is important to make the line ranges match what you see in the + # test file. + result.append(line) + continue line = line.replace(EMPTY_LINE, "") if line.rstrip() == "# output": result = _output @@ -185,7 +318,7 @@ def read_data_from_file(file_name: Path) -> Tuple[str, str]: if _input and not _output: # If there's no output marker, treat the entire file as already pre-formatted. _output = _input[:] - return "".join(_input).strip() + "\n", "".join(_output).strip() + "\n" + return mode, "".join(_input).strip() + "\n", "".join(_output).strip() + "\n" def read_jupyter_notebook(subdir_name: str, name: str, data: bool = True) -> str: diff --git a/tox.ini b/tox.ini index 4934514264b..8b4175d23fe 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,11 @@ [tox] isolated_build = true -envlist = {,ci-}py{37,38,39,310,311,py3},fuzz,run_self +envlist = {,ci-}py{38,39,310,311,py3},fuzz,run_self [testenv] -setenv = PYTHONPATH = {toxinidir}/src +setenv = + PYTHONPATH = {toxinidir}/src + PYTHONWARNDEFAULTENCODING = 1 skip_install = True # We use `recreate=True` because otherwise, on the second run of `tox -e py`, # the `no_jupyter` tests would run with the jupyter extra dependencies installed. @@ -37,19 +39,15 @@ deps = ; remove this when pypy releases the bugfix commands = pip install -e .[d] - coverage erase pytest tests \ --run-optional no_jupyter \ !ci: --numprocesses auto \ - ci: --numprocesses 1 \ - --cov {posargs} + ci: --numprocesses 1 pip install -e .[jupyter] pytest tests --run-optional jupyter \ -m jupyter \ !ci: --numprocesses auto \ - ci: --numprocesses 1 \ - --cov --cov-append {posargs} - coverage report + ci: --numprocesses 1 [testenv:{,ci-}311] setenv = @@ -96,5 +94,5 @@ commands = setenv = PYTHONPATH = {toxinidir}/src skip_install = True commands = - pip install -e .[d] + pip install -e . black --check {toxinidir}/src {toxinidir}/tests