diff --git a/.flake8 b/.flake8 index 3ba454a9d5..46776f0502 100644 --- a/.flake8 +++ b/.flake8 @@ -10,8 +10,11 @@ ignore = # ambiguous variable name, e.g. 'l' E741, # Various codes from flake8-docstrings we don't care for - D1,D205,D209,D213,D400,D401,D402,D412,D413,D999,D202, + D1,D205,D209,D213,D301,D400,D401,D402,D412,D413,D999,D202, # flake8-bugbear options we disagree with B008,B011, # flake8-bandit security warnings we disagree with or don't mind S101,S102,S105,S110,S307,S311,S404,S6 +select = + # enable checks for self-or-cls param name, use of raise-from + B902,B904 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ebcf1e348e..cf9bfc9cfd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,5 +1,10 @@ name: Hypothesis CI +env: + # Tell pytest and other tools to produce coloured terminal output. + # Make sure this is also in the "passenv" section of the tox config. + PY_COLORS: 1 + on: push: branches: [ master ] @@ -7,6 +12,12 @@ on: branches: [ master ] workflow_dispatch: +# Cancel in-progress PR builds if another commit is pushed. +# On non-PR builds, fall back to the globally-unique run_id and don't cancel. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: test: runs-on: ubuntu-latest @@ -18,9 +29,16 @@ jobs: - check-format - check-coverage - check-conjecture-coverage - - check-py38 - - check-pypy36 + - check-py37 - check-pypy37 + - check-py38 + - check-pypy38 + - check-py39 + - check-pypy39 + - check-py310 + # - check-py310-pyjion # see notes in tox.ini + - check-py311 + # - check-py312 # enable when alpha1 is released, ~Nov 2022 - check-quality - lint-ruby - check-ruby-tests @@ -31,23 +49,28 @@ jobs: - check-rust-tests - audit-conjecture-rust - lint-conjecture-rust - python-version: ["3.8"] - include: - - task: check-py36 - python-version: "3.6" - - task: check-py37 - python-version: "3.7" - - task: check-py39 - python-version: "3.9" + - check-nose + - check-pytest46 + - check-pytest54 + - check-pytest62 + - check-django41 + - check-django40 + - check-django32 + - check-pandas15 + - check-pandas14 + - check-pandas13 + - check-pandas12 + - check-pandas11 + - check-pandas10 fail-fast: false steps: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python 3.8 uses: actions/setup-python@v2 with: - python-version: ${{ matrix.python-version }} + python-version: "3.8" - name: Restore cache uses: actions/cache@v2 with: @@ -56,12 +79,32 @@ jobs: ~/wheelhouse ~/.local vendor/bundle - key: deps-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements/tools.txt') }}-${{ matrix.task }} + key: deps-${{ runner.os }}-${{ hashFiles('requirements/*.txt') }}-${{ matrix.task }} restore-keys: | - deps-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements/tools.txt') }} - deps-${{ runner.os }}-${{ matrix.python-version }} + deps-${{ runner.os }}-${{ hashFiles('requirements/*.txt') }} + deps-${{ runner.os }} + - name: Install dotnet6 for Pyjion + if: ${{ endsWith(matrix.task, '-pyjion') }} + run: | + wget https://packages.microsoft.com/config/ubuntu/21.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + rm packages-microsoft-prod.deb + sudo apt-get update + sudo apt-get install -y apt-transport-https && \ + sudo apt-get update && \ + sudo apt-get install -y dotnet-sdk-6.0 - name: Run tests run: TASK=${{ matrix.task }} ./build.sh + - name: Upload coverage data + uses: actions/upload-artifact@v2 + # Invoke the magic `always` function to run on both success and failure. + if: ${{ always() && endsWith(matrix.task, '-coverage') }} + with: + name: ${{ matrix.task }}-data + path: | + hypothesis-python/.coverage* + !hypothesis-python/.coveragerc + hypothesis-python/branch-check test-win: runs-on: windows-latest @@ -69,20 +112,18 @@ jobs: matrix: include: - task: check-py38-x64 - python.version: "3.8" python.architecture: "x64" - task: check-py38-x86 - python.version: "3.8" python.architecture: "x86" fail-fast: false steps: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python 3.8 uses: actions/setup-python@v2 with: - python-version: ${{ matrix.python-version }} + python-version: "3.8" architecture: ${{ matrix.python-architecture }} - name: Restore cache uses: actions/cache@v2 @@ -90,17 +131,17 @@ jobs: path: | ~\appdata\local\pip\cache vendor\bundle - key: deps-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.python-architecture }}-${{ hashFiles('requirements/tools.txt') }}-${{ matrix.task }} + key: deps-${{ runner.os }}-${{ matrix.python-architecture }}-${{ hashFiles('requirements/*.txt') }}-${{ matrix.task }} restore-keys: | - deps-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.python-architecture }}-${{ hashFiles('requirements/tools.txt') }} - deps-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.python-architecture }} + deps-${{ runner.os }}-${{ matrix.python-architecture }}-${{ hashFiles('requirements/*.txt') }} + deps-${{ runner.os }}-${{ matrix.python-architecture }} - name: Install dependencies run: | pip install --upgrade setuptools pip wheel - pip install -r requirements/test.txt fakeredis typing-extensions + pip install -r requirements/coverage.txt pip install hypothesis-python/[all] - name: Run tests - run: python -m pytest --numprocesses auto hypothesis-python/tests/ + run: python -m pytest --numprocesses auto hypothesis-python/tests/ --ignore=hypothesis-python/tests/quality/ --ignore=hypothesis-python/tests/ghostwriter/ test-osx: runs-on: macos-latest @@ -108,76 +149,43 @@ jobs: matrix: task: - check-py38 - python-version: ["3.8"] - fail-fast: false - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Run tests - run: TASK=${{ matrix.task }} ./build.sh - - specific-deps: - runs-on: ubuntu-latest - strategy: - matrix: - task: - - check-nose - - check-pytest46 - - check-django31 - - check-django30 - - check-django22 - - check-pandas111 - - check-pandas100 - - check-pandas25 - python-version: ["3.8"] fail-fast: false steps: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python 3.8 uses: actions/setup-python@v2 with: - python-version: ${{ matrix.python-version }} + python-version: "3.8" - name: Restore cache uses: actions/cache@v2 with: path: | ~/.cache - ~/wheelhouse - ~/.local - vendor/bundle - key: deps-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements/tools.txt') }}-${{ matrix.task }} - restore-keys: | - deps-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements/tools.txt') }} - deps-${{ runner.os }}-${{ matrix.python-version }} + ~/Library/Caches/pip + key: deps-${{ runner.os }}-${{ hashFiles('requirements/*.txt') }}-${{ matrix.task }} - name: Run tests run: TASK=${{ matrix.task }} ./build.sh deploy: if: "github.event_name == 'push' && github.repository == 'HypothesisWorks/hypothesis'" runs-on: ubuntu-latest - needs: [test, test-win, test-osx, specific-deps] + needs: [test, test-win, test-osx] strategy: matrix: task: - deploy - python-version: ["3.8"] fail-fast: false steps: - uses: actions/checkout@v2 with: fetch-depth: 0 token: ${{ secrets.GH_TOKEN }} - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python 3.8 uses: actions/setup-python@v2 with: - python-version: ${{ matrix.python-version }} + python-version: "3.8" - name: Restore cache uses: actions/cache@v2 with: @@ -186,10 +194,10 @@ jobs: ~/wheelhouse ~/.local vendor/bundle - key: deps-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements/tools.txt') }}-${{ matrix.task }} + key: deps-${{ runner.os }}-${{ hashFiles('requirements/*.txt') }}-${{ matrix.task }} restore-keys: | - deps-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements/tools.txt') }} - deps-${{ runner.os }}-${{ matrix.python-version }} + deps-${{ runner.os }}-${{ hashFiles('requirements/*.txt') }} + deps-${{ runner.os }} - name: Deploy package env: GH_TOKEN: ${{ secrets.GH_TOKEN }} diff --git a/.gitignore b/.gitignore index 6cf2757167..71c438a78e 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ htmlcov build dist .doctrees/ +.venv/ # encrypted files secrets.tar @@ -39,3 +40,65 @@ secrets # Rust build targets target + +_site/ +.sass-cache/ +.docker + +# ========================= +# Operating System Files +# ========================= + +# OSX +# ========================= + +.AppleDouble +.LSOverride + +# Thumbnails +._* + +# Files that might appear on external disk +.Spotlight-V100 +.Trashes + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +# Windows +# ========================= + +# Windows image file caches +Thumbs.db +ehthumbs.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msm +*.msp + +# Windows shortcuts +*.lnk +sftp-config.json + +# Vim files + +*.sw* + +__pycache__ + +.jekyll-metadata + +HypothesisWorks.github.io.iml +jekyll.log diff --git a/.pyup.yml b/.pyup.yml deleted file mode 100644 index 8c7b095885..0000000000 --- a/.pyup.yml +++ /dev/null @@ -1,3 +0,0 @@ -# Disable pyup.io spam until we can turn it off upstream -update: False -schedule: "every month" diff --git a/AUTHORS.rst b/AUTHORS.rst index 267cb0e0c8..34f86e2052 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -8,32 +8,40 @@ their individual contributions. .. NOTE - this list is in alphabetical order by first name (or handle). +* `Aaron Meurer `_ * `Adam Johnson `_ * `Adam Sven Johnson `_ * `Afrida Tabassum `_ (afrida@gmail.com) +* `Afonso Silva `_ (ajcerejeira@gmail.com) * `Akash Suresh `_ (akashsuresh36@gmail.com) * `Alex Gaynor `_ * `Alex Stapleton `_ * `Alex Willmer `_ (alex@moreati.org.uk) * `Andrea Pierré `_ +* `Ben Anhalt `_ * `Ben Peterson `_ (killthrush@hotmail.com) * `Benjamin Lee `_ (benjamindlee@me.com) * `Benjamin Palmer `_ * `Bex Dunn `_ (bex.dunn@gmail.com) * `Bill Tucker `_ (imbilltucker@gmail.com) +* `Brandon Chinn `_ * `Bryant Eisenbach `_ * `Buck Evan, copyright Google LLC `_ * `Cameron McGill `_ * `Charles O'Farrell `_ * `Charlie Tanksley `_ * `Chase Garner `_ (chase@garner.red) +* `Cheuk Ting Ho `_ * `Chris Down `_ +* `Chris van Dronkelaar `_ * `Christopher Martin `_ (ch.martin@gmail.com) +* `Claudio Jolowicz `_ * `Conrad Ho `_ (conrad.alwin.ho@gmail.com) * `Cory Benfield `_ * `Cristi Cobzarenco `_ (cristi@reinfer.io) * `Damon Francisco `_ (damontfrancisco@yahoo.com) * `Daniel J. West `_ +* `Daniel Knell `_ (contact@danielknell.co.uk) * `David Bonner `_ (dbonner@gmail.com) * `David Chudzicki `_ (dchudz@gmail.com) * `David Mascharka `_ @@ -41,22 +49,33 @@ their individual contributions. * `Derek Gustafson `_ * `Dion Misic `_ (dion.misic@gmail.com) * `Dmitry Dygalo `_ +* `Ed Rogers `- * `Eduardo Enriquez `_ (eduardo.a.enriquez@gmail.com) * `El Awbery `_ * `Emmanuel Leblond `_ +* `Evan Tey `_ +* `Felix Divo `_ * `Felix Grünewald `_ * `Felix Sheldon `_ * `Florian Bruhin `_ * `follower `_ +* `Gabe Joseph `_ * `Gary Donovan `_ +* `George Macon `_ +* `Glenn Lehman `_ * `Graham Williamson `_ * `Grant David Bachman `_ (grantbachman@gmail.com) * `Gregory Petrosyan `_ +* `Grzegorz Zieba `_ (g.zieba@erax.pl) * `Grigorios Giannakopoulos `_ * `Hugo van Kemenade `_ +* `Humberto Rocha `_ +* `Ilya Lebedev `_ (melevir@gmail.com) * `Israel Fruchter `_ +* `Ivan Tham `_ * `Jack Massey `_ * `Jakub Nabaglo `_ (j@nab.gl) +* `James Lamb `_ * `Jenny Rouleau `_ * `Jeremy Thurgood `_ * `J.J. Green `_ @@ -71,14 +90,17 @@ their individual contributions. * `jwg4 `_ * `Kai Chen `_ (kaichen120@gmail.com) * `Karthikeyan Singaravelan `_ (tir.karthi@gmail.com) +* `Katelyn Gigante `_ * `Katrina Durance `_ * `kbara `_ +* `Keeri Tramm `_ * `Kristian Glass `_ * `Krzysztof Przybyła `_ * `Kyle Reeve `_ (krzw92@gmail.com) * `Lampros Mountrakis `_ * `Lea Provenzano `_ * `Lee Begg `_ +* `Libor Martínek `_ * `Lisa Goeller `_ * `Louis Taylor `_ * `Luke Barone-Adesi `_ @@ -89,46 +111,64 @@ their individual contributions. * `Markus Unterwaditzer `_ (markus@unterwaditzer.net) * `Mathieu Paturel `_ (mathieu.paturel@gmail.com) * `Matt Bachmann `_ (bachmann.matt@gmail.com) +* `Matthew Barber `_ (quitesimplymatt@gmail.com) * `Max Nordlund `_ (max.nordlund@gmail.com) * `Maxim Kulkin `_ (maxim.kulkin@gmail.com) +* `Mel Seto `_ * `Michel Alexandre Salim `_ (michel@michel-slm.name) * `mulkieran `_ +* `Munir Abdinur `_ * `Nicholas Chammas `_ * `Nick Anyos `_ * `Nikita Sobolev `_ (mail@sobolevn.me) +* `Oleg Höfling `_ (oleg.hoefling@gmail.com) * `Paul Ganssle `_ (paul@ganssle.io) * `Paul Kehrer `_ * `Paul Lorett Amazona `_ * `Paul Stiverson `_ +* `Pax (R. Margret) W. `_ * `Peadar Coyle `_ (peadarcoyle@gmail.com) * `Petr Viktorin `_ +* `Phillip Schanely `_ (pschanely@gmail.com) * `Pierre-Jean Campigotto `_ * `Przemek Konopko `_ * `Richard Boulton `_ (richard@tartarus.org) +* `Richard Scholtens `_ (richardscholtens2@gmail.com) +* `Robert Howlett `_ * `Robert Knight `_ (robertknight@gmail.com) +* `Rodrigo Girão Serrão `_ (rodrigo@mathspp.com) * `Rónán Carrigan `_ (rcarriga@tcd.ie) +* `Ruben Opdebeeck `_ * `Ryan Soklaski `_ (rsoklaski@gmail.com) * `Ryan Turner `_ (ryan.turner@uber.com) * `Sam Bishop (TechDragon) `_ (sam@techdragon.io) +* `Sam Clamons `_ (sclamons@gmail.com) * `Sam Hames `_ +* `Sam Watts `_ * `Sangarshanan `_ (sangarshanan1998@gmail.com) * `Sanyam Khurana `_ * `Saul Shanabrook `_ (s.shanabrook@gmail.com) +* `Sebastiaan Zeeff `_ (sebastiaan.zeeff@ordina.nl) +* `Shlok Gandhi `_ (shlok.gandhi@gmail.com) +* `Sogata Ray `_ (rayardinanda@gmail.com) * `Stuart Cook `_ * `SuperStormer `_ * `Sushobhit `_ (sushobhitsolanki@gmail.com) * `Tariq Khokhar `_ (tariq@khokhar.net) * `Tessa Bradbury `_ * `Thea Koutsoukis `_ +* `Thomas Ball `_ (bomtall1@hotmail.com) * `Thomas Grainge `_ -* `Tim Martin `_ (tim@asymptotic.co.uk) * `Thomas Kluyver `_ (thomas@kluyver.me.uk) +* `Tim Martin `_ (tim@asymptotic.co.uk) * `Tom McDermott `_ (sponster@gmail.com) * `Tom Milligan `_ (code@tommilligan.net) * `Tyler Gibbons `_ (tyler.gibbons@flexport.com) * `Tyler Nickerson `_ * `Vidya Rani `_ (vidyarani.d.g@gmail.com) * `Vincent Michel `_ (vxgmichel@gmail.com) +* `Viorel Pluta `_ (viopluta@gmail.com) +* `Vytautas Strimaitis `_ * `Will Hall `_ (wrsh07@gmail.com) * `Will Thompson `_ (will@willthompson.co.uk) * `Wilfred Hughes `_ diff --git a/CITATION b/CITATION deleted file mode 100644 index 5733322709..0000000000 --- a/CITATION +++ /dev/null @@ -1,34 +0,0 @@ -If you use Hypothesis as part of a published research project, -please cite our paper in the Journal of Open Source Software: - -Text: - -MacIver et al., (2019). Hypothesis: A new approach to property-based testing. -Journal of Open Source Software, 4(43), 1891, https://doi.org/10.21105/joss.01891 - -BibTeX: - -@article{MacIver2019Hypothesis, - journal = {Journal of Open Source Software}, - doi = {10.21105/joss.01891}, - issn = {2475-9066}, - number = {43}, - publisher = {The Open Journal}, - title = {Hypothesis: A new approach to property-based testing}, - url = {http://dx.doi.org/10.21105/joss.01891}, - volume = {4}, - author = {MacIver, David and Hatfield-Dodds, Zac and Contributors, Many}, - pages = {1891}, - date = {2019-11-21}, - year = {2019}, - month = {11}, - day = {21}, -} - - -To reference a particular version of Hypothesis as a software artifact, -you can use the version-specific DOIs we create for each release under -https://doi.org/10.5281/zenodo.1412597 - -If you are unsure about which version of Hypothesis you are using run: -`pip show hypothesis` for the Python version. diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000000..e8a6765af6 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,65 @@ +cff-version: 1.2.0 +message: | + If you use Hypothesis as part of a published research project, + please cite our paper in the Journal of Open Source Software: + + Text: + + MacIver et al., (2019). Hypothesis: A new approach to property-based testing. + Journal of Open Source Software, 4(43), 1891, https://doi.org/10.21105/joss.01891 + + BibTeX: + + @article{MacIver2019Hypothesis, + journal = {Journal of Open Source Software}, + doi = {10.21105/joss.01891}, + issn = {2475-9066}, + number = {43}, + publisher = {The Open Journal}, + title = {Hypothesis: A new approach to property-based testing}, + url = {http://dx.doi.org/10.21105/joss.01891}, + volume = {4}, + author = {MacIver, David and Hatfield-Dodds, Zac and Contributors, Many}, + pages = {1891}, + date = {2019-11-21}, + year = {2019}, + month = {11}, + day = {21}, + } + + To reference a particular version of Hypothesis as a software artifact, + you can use the version-specific DOIs we create for each release under + https://doi.org/10.5281/zenodo.1412597 + + +preferred-citation: + title: 'Hypothesis: A new approach to property-based testing' + date-released: 2019-11-21 + type: article + doi: 10.21105/joss.01891 + authors: + - family-names: MacIver + given-names: David R. + orcid: https://orcid.org/0000-0002-8635-3223 + affiliation: Imperial College London + - family-names: Hatfield-Dodds + given-names: Zac + orcid: https://orcid.org/0000-0002-8646-8362 + affiliation: Australian National University + - name: "many other contributors" + +# Citation metadata for the software itself, as required by the CFF spec +doi: 10.5281/zenodo.1412597 # Version-independent DOI for the software archive +title: 'Hypothesis: Property-Based Testing for Python' +repository-code: https://github.com/HypothesisWorks/hypothesis +license: MPL-2.0 +authors: + - family-names: MacIver + given-names: David R. + orcid: https://orcid.org/0000-0002-8635-3223 + affiliation: Imperial College London + - family-names: Hatfield-Dodds + given-names: Zac + orcid: https://orcid.org/0000-0002-8646-8362 + affiliation: Australian National University + - name: "many other contributors" diff --git a/CODEOWNERS b/CODEOWNERS index cda93afbf9..ac74a335b6 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,6 +1,6 @@ # Engine changes need to be approved by DRMacIver, as per # https://github.com/HypothesisWorks/hypothesis/blob/master/guides/review.rst#engine-changes -/hypothesis-python/src/hypothesis/internal/conjecture/ @DRMacIver +/hypothesis-python/src/hypothesis/internal/conjecture/ @DRMacIver @Zac-HD # Changes to the paper also need to be approved by DRMacIver /paper.md @DRMacIver diff --git a/CODE_OF_CONDUCT.rst b/CODE_OF_CONDUCT.rst index ff64175f4b..4a027d6bf3 100644 --- a/CODE_OF_CONDUCT.rst +++ b/CODE_OF_CONDUCT.rst @@ -28,8 +28,7 @@ Resolution of Violations ~~~~~~~~~~~~~~~~~~~~~~~~ David R. MacIver (the project lead) acts as the main point of contact and enforcer for code of conduct violations. -You can email him at david@drmaciver.com, message him as DRMacIver on irc.freenode.net, or for violations on GitHub -that you want to draw his attention to you can also mention him as @DRMacIver. +You can email him at david@drmaciver.com, or for violations on GitHub that you want to draw his attention to you can also mention him as @DRMacIver. Other people (especially Hypothesis team members) should feel free to call people on code of conduct violations when they see them, and it is appreciated but not required (especially if doing so would make you feel uncomfortable or unsafe). diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index ea4d755991..959aa7feb3 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -4,6 +4,17 @@ Contributing First off: It's great that you want to contribute to Hypothesis! Thanks! +--------------------------------------- +Just tell me how to make a pull request +--------------------------------------- + +1. Make you change and ensure it has adequate tests +2. Create ``hypothesis-python/RELEASE.rst`` with ``RELEASE_TYPE: patch`` + for small bugfixes, or ``minor`` for new features. See recent PRs for examples. +3. Add yourself to the list in ``AUTHORS.rst`` and open a PR! + +For more detail, read on; for even more, continue to the ``guides/`` directory! + ------------------ Ways to Contribute ------------------ @@ -42,8 +53,8 @@ make changes and install the changed version) you can do this with: .. code:: bash - pip install -r requirements/test.txt - pip install -r requirements/tools.txt + pip install -r requirements/test.in + pip install -r requirements/tools.in pip install -e hypothesis-python/ # You don't need to run the tests, but here's the command: @@ -87,7 +98,7 @@ The actual contribution OK, so you want to make a contribution and have sorted out the legalese. What now? First off: If you're planning on implementing a new feature, talk to us -first! Come `join us on IRC `_, +first! Come `join us on the mailing list `_, or open an issue. If it's really small feel free to open a work in progress pull request sketching out the idea, but it's best to get feedback from the Hypothesis maintainers before sinking a bunch of work into it. @@ -165,7 +176,7 @@ Some notable commands: ``./build.sh check-coverage`` will verify 100% code coverage by running a curated subset of the test suite. -``./build.sh check-py36`` (etc.) will run most of the test suite against a +``./build.sh check-py37`` (etc.) will run most of the test suite against a particular python version. ``./build.sh format`` will reformat your code according to the Hypothesis coding style. You should use this before each @@ -180,3 +191,53 @@ Run ``./build.sh tasks`` for a list of all supported build task names. Note: The build requires a lot of different versions of python, so rather than have you install them yourself, the build system will install them itself in a local directory. This means that the first time you run a task you may have to wait a while as the build downloads and installs the right version of python for you. + +~~~~~~~~~~~~~ +Running Tests +~~~~~~~~~~~~~ + +The tasks described above will run all of the tests (e.g. ``check-py37``). But +the ``tox`` task will give finer-grained control over the test runner. At a +high level, the task takes the form: + +.. code-block:: + + ./build.sh tox py37-custom 3.7.13 [tox args] -- [pytest args] + +Namely, first provide the tox environment (see ``tox.ini``), then the python +version to test with, then any ``tox`` or ``pytest`` args as needed. For +example, to run all of the tests in the file +``tests/nocover/test_conjecture_engine.py`` with python 3.8: + +.. code-block:: + + ./build.sh tox py38-custom 3.8.13 -- tests/nocover/test_conjecture_engine.py + +See the ``tox`` docs and ``pytest`` docs for more information: +* https://docs.pytest.org/en/latest/how-to/usage.html +* https://tox.wiki/en/latest/config.html#cli + +^^^^^^^^^^^ +Test Layout +^^^^^^^^^^^ + +See ``hypothesis-python/tests/README.rst`` + +^^^^^^^^^^^^^^^^ +Useful Arguments +^^^^^^^^^^^^^^^^ + +Some useful arguments to pytest include: + +* You can pass ``-n 0`` to turn off ``pytest-xdist``'s parallel test execution. + Sometimes for running just a small number of tests its startup time is longer + than the time it saves (this will vary from system to system), so this can + be helpful if you find yourself waiting on test runners to start a lot. +* You can use ``-k`` to select a subset of tests to run. This matches on substrings + of the test names. For example ``-kfoo`` will only run tests that have "foo" as + a substring of their name. You can also use composite expressions here. + e.g. ``-k'foo and not bar'`` will run anything containing foo that doesn't + also contain bar. `More information on how to select tests to run can be found + in the pytest documentation `__. + + diff --git a/HypothesisWorks.github.io/.ruby-version b/HypothesisWorks.github.io/.ruby-version new file mode 100644 index 0000000000..8e8299dcc0 --- /dev/null +++ b/HypothesisWorks.github.io/.ruby-version @@ -0,0 +1 @@ +2.4.2 diff --git a/HypothesisWorks.github.io/CNAME b/HypothesisWorks.github.io/CNAME new file mode 100644 index 0000000000..e93475be88 --- /dev/null +++ b/HypothesisWorks.github.io/CNAME @@ -0,0 +1 @@ +hypothesis.works diff --git a/HypothesisWorks.github.io/Dockerfile b/HypothesisWorks.github.io/Dockerfile new file mode 100644 index 0000000000..18b39185d9 --- /dev/null +++ b/HypothesisWorks.github.io/Dockerfile @@ -0,0 +1,15 @@ +FROM ruby:2.4-alpine3.6 + +LABEL maintainer "Alex Chan " +LABEL description "Local build image for HypothesisWorks.github.io" + +COPY Gemfile . +COPY Gemfile.lock . + +RUN apk update && \ + apk add build-base git make nodejs +RUN bundle install + +WORKDIR /site + +ENTRYPOINT ["bundle", "exec", "jekyll"] diff --git a/HypothesisWorks.github.io/Gemfile b/HypothesisWorks.github.io/Gemfile new file mode 100644 index 0000000000..34ea81879e --- /dev/null +++ b/HypothesisWorks.github.io/Gemfile @@ -0,0 +1,4 @@ +source 'https://rubygems.org' + +gem 'github-pages', group: :jekyll_plugins +gem 'jekyll-redirect-from', group: :jekyll_plugins diff --git a/HypothesisWorks.github.io/Gemfile.lock b/HypothesisWorks.github.io/Gemfile.lock new file mode 100644 index 0000000000..b59308e4f5 --- /dev/null +++ b/HypothesisWorks.github.io/Gemfile.lock @@ -0,0 +1,240 @@ +GEM + remote: https://rubygems.org/ + specs: + activesupport (4.2.9) + i18n (~> 0.7) + minitest (~> 5.1) + thread_safe (~> 0.3, >= 0.3.4) + tzinfo (~> 1.1) + addressable (2.5.2) + public_suffix (>= 2.0.2, < 4.0) + coffee-script (2.4.1) + coffee-script-source + execjs + coffee-script-source (1.11.1) + colorator (1.1.0) + commonmarker (0.17.7.1) + ruby-enum (~> 0.5) + concurrent-ruby (1.0.5) + ethon (0.11.0) + ffi (>= 1.3.0) + execjs (2.7.0) + faraday (0.14.0) + multipart-post (>= 1.2, < 3) + ffi (1.9.21) + forwardable-extended (2.6.0) + gemoji (3.0.0) + github-pages (177) + activesupport (= 4.2.9) + github-pages-health-check (= 1.3.5) + jekyll (= 3.6.2) + jekyll-avatar (= 0.5.0) + jekyll-coffeescript (= 1.0.2) + jekyll-commonmark-ghpages (= 0.1.5) + jekyll-default-layout (= 0.1.4) + jekyll-feed (= 0.9.2) + jekyll-gist (= 1.4.1) + jekyll-github-metadata (= 2.9.3) + jekyll-mentions (= 1.2.0) + jekyll-optional-front-matter (= 0.3.0) + jekyll-paginate (= 1.1.0) + jekyll-readme-index (= 0.2.0) + jekyll-redirect-from (= 0.12.1) + jekyll-relative-links (= 0.5.2) + jekyll-remote-theme (= 0.2.3) + jekyll-sass-converter (= 1.5.0) + jekyll-seo-tag (= 2.3.0) + jekyll-sitemap (= 1.1.1) + jekyll-swiss (= 0.4.0) + jekyll-theme-architect (= 0.1.0) + jekyll-theme-cayman (= 0.1.0) + jekyll-theme-dinky (= 0.1.0) + jekyll-theme-hacker (= 0.1.0) + jekyll-theme-leap-day (= 0.1.0) + jekyll-theme-merlot (= 0.1.0) + jekyll-theme-midnight (= 0.1.0) + jekyll-theme-minimal (= 0.1.0) + jekyll-theme-modernist (= 0.1.0) + jekyll-theme-primer (= 0.5.2) + jekyll-theme-slate (= 0.1.0) + jekyll-theme-tactile (= 0.1.0) + jekyll-theme-time-machine (= 0.1.0) + jekyll-titles-from-headings (= 0.5.0) + jemoji (= 0.8.1) + kramdown (= 1.16.2) + liquid (= 4.0.0) + listen (= 3.0.6) + mercenary (~> 0.3) + minima (= 2.1.1) + nokogiri (>= 1.8.1, < 2.0) + rouge (= 2.2.1) + terminal-table (~> 1.4) + github-pages-health-check (1.3.5) + addressable (~> 2.3) + net-dns (~> 0.8) + octokit (~> 4.0) + public_suffix (~> 2.0) + typhoeus (~> 0.7) + html-pipeline (2.7.1) + activesupport (>= 2) + nokogiri (>= 1.4) + i18n (0.9.4) + concurrent-ruby (~> 1.0) + jekyll (3.6.2) + addressable (~> 2.4) + colorator (~> 1.0) + jekyll-sass-converter (~> 1.0) + jekyll-watch (~> 1.1) + kramdown (~> 1.14) + liquid (~> 4.0) + mercenary (~> 0.3.3) + pathutil (~> 0.9) + rouge (>= 1.7, < 3) + safe_yaml (~> 1.0) + jekyll-avatar (0.5.0) + jekyll (~> 3.0) + jekyll-coffeescript (1.0.2) + coffee-script (~> 2.2) + coffee-script-source (~> 1.11.1) + jekyll-commonmark (1.1.0) + commonmarker (~> 0.14) + jekyll (>= 3.0, < 4.0) + jekyll-commonmark-ghpages (0.1.5) + commonmarker (~> 0.17.6) + jekyll-commonmark (~> 1) + rouge (~> 2) + jekyll-default-layout (0.1.4) + jekyll (~> 3.0) + jekyll-feed (0.9.2) + jekyll (~> 3.3) + jekyll-gist (1.4.1) + octokit (~> 4.2) + jekyll-github-metadata (2.9.3) + jekyll (~> 3.1) + octokit (~> 4.0, != 4.4.0) + jekyll-mentions (1.2.0) + activesupport (~> 4.0) + html-pipeline (~> 2.3) + jekyll (~> 3.0) + jekyll-optional-front-matter (0.3.0) + jekyll (~> 3.0) + jekyll-paginate (1.1.0) + jekyll-readme-index (0.2.0) + jekyll (~> 3.0) + jekyll-redirect-from (0.12.1) + jekyll (~> 3.3) + jekyll-relative-links (0.5.2) + jekyll (~> 3.3) + jekyll-remote-theme (0.2.3) + jekyll (~> 3.5) + rubyzip (>= 1.2.1, < 3.0) + typhoeus (>= 0.7, < 2.0) + jekyll-sass-converter (1.5.0) + sass (~> 3.4) + jekyll-seo-tag (2.3.0) + jekyll (~> 3.3) + jekyll-sitemap (1.1.1) + jekyll (~> 3.3) + jekyll-swiss (0.4.0) + jekyll-theme-architect (0.1.0) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-theme-cayman (0.1.0) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-theme-dinky (0.1.0) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-theme-hacker (0.1.0) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-theme-leap-day (0.1.0) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-theme-merlot (0.1.0) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-theme-midnight (0.1.0) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-theme-minimal (0.1.0) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-theme-modernist (0.1.0) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-theme-primer (0.5.2) + jekyll (~> 3.5) + jekyll-github-metadata (~> 2.9) + jekyll-seo-tag (~> 2.2) + jekyll-theme-slate (0.1.0) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-theme-tactile (0.1.0) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-theme-time-machine (0.1.0) + jekyll (~> 3.5) + jekyll-seo-tag (~> 2.0) + jekyll-titles-from-headings (0.5.0) + jekyll (~> 3.3) + jekyll-watch (1.5.1) + listen (~> 3.0) + jemoji (0.8.1) + activesupport (~> 4.0, >= 4.2.9) + gemoji (~> 3.0) + html-pipeline (~> 2.2) + jekyll (>= 3.0) + kramdown (1.16.2) + liquid (4.0.0) + listen (3.0.6) + rb-fsevent (>= 0.9.3) + rb-inotify (>= 0.9.7) + mercenary (0.3.6) + mini_portile2 (2.3.0) + minima (2.1.1) + jekyll (~> 3.3) + minitest (5.11.3) + multipart-post (2.0.0) + net-dns (0.8.0) + nokogiri (1.8.2) + mini_portile2 (~> 2.3.0) + octokit (4.8.0) + sawyer (~> 0.8.0, >= 0.5.3) + pathutil (0.16.1) + forwardable-extended (~> 2.6) + public_suffix (2.0.5) + rb-fsevent (0.10.2) + rb-inotify (0.9.10) + ffi (>= 0.5.0, < 2) + rouge (2.2.1) + ruby-enum (0.7.1) + i18n + rubyzip (1.2.1) + safe_yaml (1.0.4) + sass (3.5.5) + sass-listen (~> 4.0.0) + sass-listen (4.0.0) + rb-fsevent (~> 0.9, >= 0.9.4) + rb-inotify (~> 0.9, >= 0.9.7) + sawyer (0.8.1) + addressable (>= 2.3.5, < 2.6) + faraday (~> 0.8, < 1.0) + terminal-table (1.8.0) + unicode-display_width (~> 1.1, >= 1.1.1) + thread_safe (0.3.6) + typhoeus (0.8.0) + ethon (>= 0.8.0) + tzinfo (1.2.5) + thread_safe (~> 0.1) + unicode-display_width (1.3.0) + +PLATFORMS + ruby + +DEPENDENCIES + github-pages + jekyll-redirect-from + +BUNDLED WITH + 1.16.0 diff --git a/HypothesisWorks.github.io/LICENSE.md b/HypothesisWorks.github.io/LICENSE.md new file mode 100644 index 0000000000..e387b3b2ef --- /dev/null +++ b/HypothesisWorks.github.io/LICENSE.md @@ -0,0 +1,25 @@ +Parts of this site are based on rifyll by Moinul Hossain. These parts (and +only these parts) arereleased under the following license: + + +The MIT License (MIT) + +Copyright (c) [2015] [Moinul Hossain] + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/HypothesisWorks.github.io/Makefile b/HypothesisWorks.github.io/Makefile new file mode 100644 index 0000000000..aa2c780323 --- /dev/null +++ b/HypothesisWorks.github.io/Makefile @@ -0,0 +1,37 @@ +BUILD_IMAGE = hypothesisworks/hypothesisworks.github.io +SERVE_CONTAINER = server + +ROOT = $(shell git rev-parse --show-toplevel) + + +.docker/build: Dockerfile Gemfile Gemfile.lock + docker build --tag $(BUILD_IMAGE) . + mkdir -p .docker + touch .docker/build + + +build: .docker/build + docker run --volume $(SRC):/site $(BUILD_IMAGE) build + +serve: .docker/build + @# Clean up old running containers + @docker stop $(SERVE_CONTAINER) >/dev/null 2>&1 || true + @docker rm $(SERVE_CONTAINER) >/dev/null 2>&1 || true + + docker run \ + --publish 5858:5858 \ + --volume $(ROOT):/site \ + --name $(SERVE_CONTAINER) \ + --hostname $(SERVE_CONTAINER) \ + --tty $(BUILD_IMAGE) \ + serve --host $(SERVE_CONTAINER) --port 5858 --watch + +Gemfile.lock: Gemfile + docker run \ + --volume $(ROOT):/site \ + --workdir /site \ + --tty $(shell cat Dockerfile | grep FROM | awk '{print $$2}') \ + bundle lock --update + + +.PHONY: build serve diff --git a/HypothesisWorks.github.io/README.md b/HypothesisWorks.github.io/README.md new file mode 100644 index 0000000000..5fe1dd1ed2 --- /dev/null +++ b/HypothesisWorks.github.io/README.md @@ -0,0 +1,25 @@ +# README # + +This is the main hypothesis.works site. It is originally based off the [rifyll](https://github.com/itsrifat/rifyll) Jekyll template. + +## Getting started + +You need Git, make and Docker installed. + +To run a local copy of the site: + +```console +$ git clone git@github.com:HypothesisWorks/HypothesisWorks.github.io.git +$ make serve +``` + +The site should be running on . +If you make changes to the source files, it will automatically update. + +To build a one-off set of static HTML files: + +```console +$ make build +``` + +When you push to master, the site is automatically built and served by GitHub Pages. diff --git a/HypothesisWorks.github.io/Vagrantfile b/HypothesisWorks.github.io/Vagrantfile new file mode 100644 index 0000000000..9f6dd7147e --- /dev/null +++ b/HypothesisWorks.github.io/Vagrantfile @@ -0,0 +1,14 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + + + +Vagrant.configure(2) do |config| + config.vm.provider "vmware_workstation" do |v| + v.memory = 1024 + end + + config.vm.box = "bento/ubuntu-14.04" + config.vm.network "forwarded_port", guest: 4000, host: 4000 + config.vm.provision "shell", path: "provision.sh", privileged: false +end diff --git a/HypothesisWorks.github.io/_config.yml b/HypothesisWorks.github.io/_config.yml new file mode 100644 index 0000000000..9d46319e2f --- /dev/null +++ b/HypothesisWorks.github.io/_config.yml @@ -0,0 +1,39 @@ +# Site settings +title: Hypothesis +email: hello@hypothesis.works +description: "Test faster, fix more" + +#baseurl: "" # the subpath of your site, e.g. /blog/ +url: http://hypothesis.works + +# Build settings +markdown: kramdown +kramdown: + input: GFM + +permalink: /articles/:title/ + +highlighter: rouge + +#Other Global Configuration +exclude: + - README.md + - LICENSE.md + - CNAME + - Gemfile + - Gemfile.lock + - Makefile + - provision.sh + - Vagrantfile + - Dockerfile + +excerpt_separator: + +#show the jumbotron header on index page +show_blog_header: true + +tag_descriptions: + python: "Articles specific to the Python implementation of Hypothesis" + +gems: + - jekyll-redirect-from diff --git a/HypothesisWorks.github.io/_data/authors.yml b/HypothesisWorks.github.io/_data/authors.yml new file mode 100644 index 0000000000..756d62ccfa --- /dev/null +++ b/HypothesisWorks.github.io/_data/authors.yml @@ -0,0 +1,15 @@ +drmaciver: + name: David R. MacIver + url: http://www.drmaciver.com +jml: + name: Jonathan M. Lange + url: https://jml.io +giorgiosironi: + name: Giorgio Sironi + url: http://giorgiosironi.com +nchammas: + name: Nicholas Chammas + url: http://nchammas.com +alexwlchan: + name: Alex Chan + url: https://alexwlchan.net diff --git a/HypothesisWorks.github.io/_data/menu.yml b/HypothesisWorks.github.io/_data/menu.yml new file mode 100644 index 0000000000..30b2443bc6 --- /dev/null +++ b/HypothesisWorks.github.io/_data/menu.yml @@ -0,0 +1,26 @@ + +mainmenu: + home: + url: "/" + text: "Home" + icon_class: fa-home + products: + url: "/products/" + text: "Products" + icon_class: fa-wrench + services: + url: "/services/" + text: "Services" + icon_class: fa-briefcase + training: + url: "/training/" + text: "Training" + icon_class: "fa-graduation-cap" + articles: + url: "/articles/" + text: "Articles" + icon_class: "fa-book" + contact: + url: "mailto:hello@hypothesis.works" + text: "Contact" + icon_class: "fa-envelope" diff --git a/HypothesisWorks.github.io/_includes/blog_header.html b/HypothesisWorks.github.io/_includes/blog_header.html new file mode 100644 index 0000000000..874fddf37a --- /dev/null +++ b/HypothesisWorks.github.io/_includes/blog_header.html @@ -0,0 +1,10 @@ +
+
+
+
+

{{site.title}}

+

{{site.description}}

+
+
+
+
\ No newline at end of file diff --git a/HypothesisWorks.github.io/_includes/footer.html b/HypothesisWorks.github.io/_includes/footer.html new file mode 100644 index 0000000000..be3981394c --- /dev/null +++ b/HypothesisWorks.github.io/_includes/footer.html @@ -0,0 +1,10 @@ + diff --git a/HypothesisWorks.github.io/_includes/head.html b/HypothesisWorks.github.io/_includes/head.html new file mode 100644 index 0000000000..e034ed25c3 --- /dev/null +++ b/HypothesisWorks.github.io/_includes/head.html @@ -0,0 +1,21 @@ + + + + + + {% if page.title %} {{ page.title }} - {{ site.title }} {% else %} {{ site.title }} {% endif %} + + + + + + + + + + {% if page.has_feed %} + + {% endif %} + + diff --git a/HypothesisWorks.github.io/_includes/header.html b/HypothesisWorks.github.io/_includes/header.html new file mode 100644 index 0000000000..cb907203db --- /dev/null +++ b/HypothesisWorks.github.io/_includes/header.html @@ -0,0 +1,30 @@ +
+ + +
diff --git a/HypothesisWorks.github.io/_layouts/blog_feed.xml b/HypothesisWorks.github.io/_layouts/blog_feed.xml new file mode 100644 index 0000000000..eb9f8d8060 --- /dev/null +++ b/HypothesisWorks.github.io/_layouts/blog_feed.xml @@ -0,0 +1,53 @@ +--- +layout: null +--- + + +{% if page.tag %} + +{% assign posts = site.tags[page.tag] %} + +{% else %} + +{% assign posts = site.posts %} + +{% endif %} + + + + + {% if page.tag %} + {{ site.title | xml_escape }} articles tagged "{{page.tag}}" + {{ site.url }}/articles/{{page.tag}}/ + {% else %} + {{ site.title | xml_escape }} articles + {{ site.url }}/articles/ + {% endif %} + {{ site.description | xml_escape }} + + {% if page.tag %} + + {% else %} + + {% endif %} + {{ site.time | date_to_rfc822 }} + {{ site.time | date_to_rfc822 }} + {% for post in posts limit:10 %} + + {{ post.title | xml_escape }} + {{ post.excerpt | xml_escape }} + <a href="{{ post.url | prepend: site.baseurl | prepend: site.url }}">Read more...</a> + + {{ post.date | date_to_rfc822 }} + {{ post.url | prepend: site.baseurl | prepend: site.url }} + {{ post.url | prepend: site.baseurl | prepend: site.url }} + {% for tag in post.tags %} + {{ tag | xml_escape }} + {% endfor %} + {% for cat in post.categories %} + {{ cat | xml_escape }} + {% endfor %} + + {% endfor %} + + diff --git a/HypothesisWorks.github.io/_layouts/blog_listing.html b/HypothesisWorks.github.io/_layouts/blog_listing.html new file mode 100644 index 0000000000..a70345a0c3 --- /dev/null +++ b/HypothesisWorks.github.io/_layouts/blog_listing.html @@ -0,0 +1,75 @@ +--- +layout: default +has_feed: true +--- + +{% if page.tag %} + +{% assign posts = site.tags[page.tag] %} + +{% else %} + +{% assign posts = site.posts %} + +{% endif %} + +
+
+
+
+
+ {% if page.tag %} + {{page.tag}} + {% else %} + All our articles + {% endif %} +
+
+ {{content}} +
+
+ + {% if page.tag %} + Subscribe to articles tagged “{{page.tag}}” via RSS + {% else %} + Subscribe to all articles from us via RSS + {% endif %} + + + +
+
+ + {% for post in posts %} +
+

{{post.title}}

+
+ + + + +
+
+ + + {% assign sorted_tags = post.tags | sort %} + {% for tag in sorted_tags %} + {{tag}} + {% endfor %} + +
+ {% if post.thumbnail %} +
+ {{post.title}} +
+ {% endif %} +
{{post.excerpt}}
+ Read More +
+ {% endfor %} +
+
+
diff --git a/HypothesisWorks.github.io/_layouts/default.html b/HypothesisWorks.github.io/_layouts/default.html new file mode 100644 index 0000000000..af95c95ec2 --- /dev/null +++ b/HypothesisWorks.github.io/_layouts/default.html @@ -0,0 +1,24 @@ + + + {% include head.html %} + + + {% include header.html %} + + {% if site.show_blog_header %} + {% include blog_header.html %} + {% endif %} + + + {{ content }} + + {% include footer.html%} + + + + + + + + + diff --git a/HypothesisWorks.github.io/_layouts/page.html b/HypothesisWorks.github.io/_layouts/page.html new file mode 100644 index 0000000000..a0f67d435c --- /dev/null +++ b/HypothesisWorks.github.io/_layouts/page.html @@ -0,0 +1,23 @@ +--- +layout: default +--- + +
+
+
+
+ +
+

{{ page.title }}

+ {% if page.subtitle %} +
{{ page.subtitle }}
+ {% endif %} +
+ +
+ {{ content }} +
+
+
+
+
diff --git a/HypothesisWorks.github.io/_layouts/post.html b/HypothesisWorks.github.io/_layouts/post.html new file mode 100644 index 0000000000..c4f225fb5b --- /dev/null +++ b/HypothesisWorks.github.io/_layouts/post.html @@ -0,0 +1,41 @@ +--- +layout: default +--- +
+
+
+
+ +
+

{{ page.title }}

+
+ + {% assign author = site.data.authors[page.author] %} + +
+ {{ author.name }} +
+ +
+ +
+ +
+ {% assign sorted_tags = page.tags | sort %} + {% for tag in sorted_tags %} + {{tag}} + {% endfor %} +
+
+ +
+ +
+ {{ content }} +
+
+
+
+
diff --git a/HypothesisWorks.github.io/_posts/2016-04-15-economics-of-software-correctness.md b/HypothesisWorks.github.io/_posts/2016-04-15-economics-of-software-correctness.md new file mode 100644 index 0000000000..23463fa668 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-04-15-economics-of-software-correctness.md @@ -0,0 +1,70 @@ +--- +layout: post +tags: writing-good-software non-technical +date: 2016-04-15 15:00 +title: The Economics of Software Correctness +published: true +author: drmaciver +--- + +You have probably never written a significant piece of correct software. + +That's not a value judgement. It's certainly not a criticism of your competence. I can say with almost complete confidence that every non-trivial piece of software I have written contains at least one bug. You *might* have written small libraries that are essentially bug free, but the chance that you have written a non-trivial bug free program is tantamount to zero. + +I don't even mean this in some pedantic academic sense. I'm talking about behaviour where if someone spotted it and pointed it out to you you would probably admit that it's a bug. It might even be a bug that you cared about. + +Why is this? + + + +Well, lets start with why it's not: It's not because we don't know how to write correct software. We've known how to write software that is more or less correct (or at least vastly closer to correct than the norm) for a while now. If you look at the NASA development process they're pretty much doing it. + +Also, if you look at the NASA development process you will probably conclude that we can't do that. It's orders of magnitude more work than we ever put into software development. It's process heavy, laborious, and does not adapt well to changing requirements or tight deadlines. + +The problem is not that we don't know how to write correct software. The problem is that correct software is too expensive. + +And "too expensive" doesn't mean "It will knock 10% off our profit margins, we couldn't possibly do that". It means "if our software cost this much to make, nobody would be willing to pay a price we could afford to sell it at". It may also mean "If our software took this long to make then someone else will release a competing product two years earlier than us, everyone will use that, and when ours comes along nobody will be interested in using it". + +("sell" and "release" here can mean a variety of things. It can mean that terribly unfashionable behaviour where people give you money and you give them a license to your software. It can mean subscriptions. It can mean ad space. It can even mean paid work. I'm just going to keep saying sell and release). + +NASA can do it because when they introduce a software bug they potentially lose some combination of billions of dollars, years of work and many lives. When that's the cost of a bug, spending that much time and money on correctness seems like a great deal. Safety critical industries like medical technology and aviation can do it for similar reasons +([buggy medical technology kills people](https://en.wikipedia.org/wiki/Therac-25) and [you don't want your engines power cycling themselves midflight](http://www.engadget.com/2015/05/01/boeing-787-dreamliner-software-bug/)). + +The rest of us aren't writing safety critical software, and as a result people aren't willing to pay for that level of correctness. + +So the result is that we write software with bugs in it, and we adopt a much cheaper software testing methodology: We ship it and see what happens. Inevitably some user will find a bug in our software. Probably many users will find many bugs in our software. + +And this means that we're turning our users into our QA department. + +Which, to be clear, is fine. Users have stated the price that they're willing to pay, and that price does not include correctness, so they're getting software that is not correct. I think we all feel bad about shipping buggy software, so let me emphasise this here: Buggy software is not a moral failing. The option to ship correct software is simply not on the table, so why on earth should we feel bad about not taking it? + +But in another sense, turning our users into a QA department is a terrible idea. + +Why? Because users are not actually good at QA. QA is a complicated professional skill which very few people can do well. Even skilled developers often don't know how to write a good bug report. How can we possibly expect our users to? + +The result is long and frustrating conversations with users in which you try to determine whether what they're seeing is actually a bug or a misunderstanding (although treating misunderstandings as bugs is a good idea too), trying to figure out what the actual bug is, etc. It's a time consuming process which ends up annoying the user and taking up a lot of expensive time from developers and customer support. + +And that's of course if the users tell you at all. Some users will just try your software, decide it doesn't work, and go away without ever saying anything to you. This is particularly bad for software where you can't easily tell who is using it. + +Also, some of our users are actually adversaries. They're not only not going to tell you about bugs they find, they're going to actively try to keep you from finding out because they're using it to steal money and/or data from you. + +So this is the problem with shipping buggy software: Bugs found by users are more expensive than bugs found before a user sees them. Bugs found by users may result in lost users, lost time and theft. These all hurt the bottom line. + +At the same time, your users are a lot more effective at finding bugs than you are due to sheer numbers if nothing else, and as we've established it's basically impossible to ship fully correct software, so we end up choosing some level of acceptable defect rate in the middle. This is generally determined by the point at which it is more expensive to find the next bug yourself than it is to let your users find it. Any higher or lower defect rate and you could just adjust your development process and make more money, and companies like making money so if they're competently run will generally do the things that cause them to do so. + +You can, and should, [cheaper to find bugs is to reduce the cost of when your users do find them](http://itamarst.org/softwaretesting/book/realworld.html), but it's always going to be expensive. + +This means that there are only two viable ways to improve software quality: + +* Make users angrier about bugs +* Make it cheaper to find bugs + +I think making users angrier about bugs is a good idea and I wish people cared more about software quality, but as a business plan it's a bit of a rubbish one. It creates higher quality software by making it more expensive to write software. + +Making it cheaper to find bugs though... that's a good one, because it increases the quality of the software by increasing your profit margins. Literally everyone wins: The developers win, the users win, the business's owners win. + +And so this is the lever we get to pull to change the world: If you want better software, make or find tools that reduce the effort of finding bugs. + +Obviously I think Hypothesis is an example of this, but it's neither the only one nor the only one you need. Better monitoring is another. Code review processes. Static analysis. Improved communication. There are many more. + +But one thing that *won't* improve your ability to find bugs is feeling bad about yourself and trying really hard to write correct software then feeling guilty when you fail. This seems to be the current standard, and it's deeply counter-productive. You can't fix systemic issues with individual action, and the only way to ship better software is to change the economics to make it viable to do so. diff --git a/HypothesisWorks.github.io/_posts/2016-04-15-getting-started-with-hypothesis.md b/HypothesisWorks.github.io/_posts/2016-04-15-getting-started-with-hypothesis.md new file mode 100644 index 0000000000..b269e909bc --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-04-15-getting-started-with-hypothesis.md @@ -0,0 +1,100 @@ +--- +layout: post +tags: intro python technical properties +date: 2016-04-15 13:00 +title: Getting started with Hypothesis +published: true +author: drmaciver +--- + +Hypothesis will speed up your testing process and improve your software quality, +but when first starting out people often struggle to figure out exactly how to use it. + +Until you're used to thinking in this style of testing, it's not always obvious what the invariants of your +code actually *are*, and people get stuck trying to come up with interesting ones to test. + +Fortunately, there's a simple invariant which every piece of software should satisfy, and which can be +remarkably powerful as a way to uncover surprisingly deep bugs in your software. + + + +That invariant is simple: The software shouldn't crash. Or sometimes, it should only crash in defined +ways. + +There is then a standard test you can write for most of your code that asserts this +invariant. + +It consists of two steps: + +1. Pick a function in your code base that you want to be better tested. +2. Call it with random data. + +This style of testing is usually called *fuzzing*. + +This will possibly require you to figure out how to generate your domain objects. Hypothesis +[has a pretty extensive library]({{site.url}}{% post_url 2016-05-11-generating-the-right-data %}) +of tools (called 'strategies' in Hypothesis terminology) for generating +custom types but if you can, try to start somewhere where the types you +need aren’t *too* complicated to generate. + +Chances are actually pretty good that you’ll find something wrong this way if you pick a +sufficiently interesting entry point. For example, there’s a long track record of people trying to +test interesting properties with their text handling and getting unicode errors when text() +gives them something that their code didn’t know how to handle. + +You’ll probably get exceptions here you don’t care about. e.g. some arguments to functions may not be valid. +Set up your test to ignore those. + +So at this point you’ll have something like this: + +```python +from hypothesis import given, reject +from hypothesis.strategies import integers, text + + +@given(integers(), text()) +def test_some_stuff(x, y): + try: + my_function(x, y) + except SomeExpectedException: + reject() +``` + +In this example we generate two values - one integer, one text - and +pass them to your test function. Hypothesis will repeatedly call the +test function with values drawn from these strategies, trying to find +one that produces an unexpected exception. + +When an exception we know is possible happens (e.g. a ValueError because +some argument was out of range) we call reject. This discards the +example, and Hypothesis won't count it towards the 'budget' of examples +it is allowed to run. + +This is already a pretty good starting point and does have a decent +tendency to flush out bugs. You’ll often find cases where you forgot some +boundary condition and your code misbehaves as a result. But there’s +still plenty of room to improve. + +There are now two directions you can go in from here: + +1. Try to assert some things about the function’s result. Anything at all. What type is it? + Can it be None? Does it have any relation at all to the input values that you can check? + It doesn’t have to be clever - even very trivial properties are useful here. +2. Start making your code more defensive. + +The second part is probably the most productive one. + +The goal is to turn faulty assumptions in your code into crashes instead of silent corruption of your application state. You can do this in a couple ways: + +1. Add argument checking to your functions (Hypothesis uses a dedicated InvalidArgumentException for this case, but you can raise whatever errors you find appropriate). +2. Add a whole bunch of assertions into your code itself. +Even when it’s hard to reason about formal properties of your code, it’s usually relatively easy to add local properties, and assertions are a great way to encode them. John Regehr has [a good post on this subject](http://blog.regehr.org/archives/1091) if you want to know more about it. + +This approach will make your code more robust even if you don’t find any bugs in it during testing (and you’ll probably find bugs in it during testing), and it gives you a nice easy route into property based testing by letting you focus on only one half of the problem at a time. + +Once you think you've got the hang of this, a good next step is to start looking for +[places with complex optimizations]({{site.url}}{% post_url 2016-04-29-testing-performance-optimizations %}) or +[Encode/Decode pairs]({{site.url}}{% post_url 2016-04-16-encode-decode-invariant %}) in +your code, as they're a fairly easy properties to test and are both rich sources of bugs. + +And, of course, if you’re still having trouble getting started with Hypothesis, the other easy way is to persuade your company [to hire us for a training course](/training/). Drop us an email at [training@hypothesis.works](mailto:training@hypothesis.works]) diff --git a/HypothesisWorks.github.io/_posts/2016-04-16-anatomy-of-a-test.md b/HypothesisWorks.github.io/_posts/2016-04-16-anatomy-of-a-test.md new file mode 100644 index 0000000000..30fba4a8ff --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-04-16-anatomy-of-a-test.md @@ -0,0 +1,203 @@ +--- +layout: post +tags: python details technical +date: 2016-04-16 06:00 +title: Anatomy of a Hypothesis Based Test +published: true +author: drmaciver +--- + +What happens when you run a test using Hypothesis? This article will help you understand. + + + +The Python version of Hypothesis uses *decorators* to transform a test function that +doesn't use Hypothesis into one that does. + +Consider the following example using [py.test](http://pytest.org/latest/) style testing: + +```python +from hypothesis import given +from hypothesis.strategies import floats + + +@given(floats(), floats()) +def test_floats_are_commutative(x, y): + assert x + y == y + x +``` + +The inner function takes two arguments, but the wrapping function defined by the @given +decorator takes none and may be invoked as a normal test: + +```bash +python -m pytest test_floats.py +``` + +And we see the following output from py.test: + +``` + @given(floats(), floats()) + def test_floats_are_commutative(x, y): +> assert x + y == y + x +E assert (0.0 + nan) == (nan + 0.0) + +test_floats.py:7: AssertionError + +Falsifying example: test_floats_are_commutative(x=0.0, y=nan) +``` + +The test fails, because [nan](https://en.wikipedia.org/wiki/NaN) is a valid floating +point number which is not equal to itself, and adding anything to nan yields nan. + +When we ran this, Hypothesis invoked our test function with a number of randomly chosen +values for the arguments until it found one that failed. It then attempted to *shrink* +those values to produce a simpler one that would also fail. + +If we wanted to see what it actually called our function with we can set the *verbosity +level*. This can either be done in code with settings, or by specifying an environment +variable: + + +```python +from hypothesis import Verbosity, given, settings +from hypothesis.strategies import floats + + +@settings(verbosity=Verbosity.verbose) +@given(floats(), floats()) +def test_floats_are_commutative(x, y): + assert x + y == y + x +``` + +```bash +HYPOTHESIS_VERBOSITY_LEVEL=verbose python -m pytest test_floats.py +``` + +Any verbosity values explicitly passed in settings will override whatever is set at +the environment level - the latter just provides a default. + +Whichever one we choose, running it we'll see output something like the following: + +``` + +Trying example: test_floats_are_commutative(x=-0.05851890381391768, y=-6.060045836901702e+300) +Trying example: test_floats_are_commutative(x=-0.06323690311413645, y=2.0324087421708266e-308) +Trying example: test_floats_are_commutative(x=-0.05738038380011458, y=-1.5993500302384265e-308) +Trying example: test_floats_are_commutative(x=-0.06598754758697359, y=-1.1412902232349034e-308) +Trying example: test_floats_are_commutative(x=-0.06472919559855002, y=1.7429441378277974e+35) +Trying example: test_floats_are_commutative(x=-0.06537123121982172, y=-8.136220566134233e-156) +Trying example: test_floats_are_commutative(x=-0.06016703321602157, y=1.9718842567475311e-215) +Trying example: test_floats_are_commutative(x=-0.055257588875432875, y=1.578407827448836e-308) +Trying example: test_floats_are_commutative(x=-0.06313031758042666, y=1.6749023021600297e-175) +Trying example: test_floats_are_commutative(x=-0.05886897920547916, y=1.213699633272585e+292) +Trying example: test_floats_are_commutative(x=-12.0, y=-0.0) +Trying example: test_floats_are_commutative(x=4.0, y=1.7976931348623157e+308) +Trying example: test_floats_are_commutative(x=-9.0, y=0.0) +Trying example: test_floats_are_commutative(x=-38.0, y=1.7976931348623157e+308) +Trying example: test_floats_are_commutative(x=-24.0, y=1.5686642754811104e+289) +Trying example: test_floats_are_commutative(x=-10.0, y=nan) +Traceback (most recent call last): + ... +AssertionError: assert (-10.0 + nan) == (nan + -10.0) + +Trying example: test_floats_are_commutative(x=10.0, y=nan) +Traceback (most recent call last): + ... +AssertionError: assert (10.0 + nan) == (nan + 10.0) + +Trying example: test_floats_are_commutative(x=0.0, y=nan) +Traceback (most recent call last): + ... +AssertionError: assert (0.0 + nan) == (nan + 0.0) + +Trying example: test_floats_are_commutative(x=0.0, y=0.0) +Trying example: test_floats_are_commutative(x=0.0, y=inf) +Trying example: test_floats_are_commutative(x=0.0, y=-inf) +Falsifying example: test_floats_are_commutative(x=0.0, y=nan) +``` + +Notice how the first failing example we got was -10.0, nan but Hypothesis was able +to turn that into 0.0, nan. That's the shrinking at work. For a simple case like this it +doesn't matter so much, but as your examples get complicated it's essential for making +Hypothesis's output easy to understand. + + +``` +Trying example: test_floats_are_commutative(x=nan, y=0.0) +Traceback (most recent call last): + ... +AssertionError: assert (nan + 0.0) == (0.0 + nan) + +Trying example: test_floats_are_commutative(x=0.0, y=0.0) +Trying example: test_floats_are_commutative(x=inf, y=0.0) +Trying example: test_floats_are_commutative(x=-inf, y=0.0) +Falsifying example: test_floats_are_commutative(x=nan, y=0.0) +``` + +Now lets see what happens when we rerun the test: + + +``` +Trying example: test_floats_are_commutative(x=0.0, y=nan) +Traceback (most recent call last): + ... +AssertionError: assert (0.0 + nan) == (nan + 0.0) + +Trying example: test_floats_are_commutative(x=0.0, y=0.0) +Trying example: test_floats_are_commutative(x=0.0, y=inf) +Trying example: test_floats_are_commutative(x=0.0, y=-inf) +Falsifying example: test_floats_are_commutative(x=0.0, y=nan) +``` + +Notice how the first example it tried was the failing example we had last time? That's +not an accident. Hypothesis has an example database where it saves failing examples. +When it starts up it looks for any examples it has seen failing previously and tries +them first before any random generation occurs. If any of them fail, we take that +failure as our starting point and move straight to the shrinking phase without any +generation. + +The database format is safe to check in to version control if you like and will merge +changes correctly out of the box, but it's often clearer to specify the examples you +want to run every time in the source code as follows: + + +```python +from hypothesis import example, given +from hypothesis.strategies import floats + + +@example(0.0, float("nan")) +@given(floats(), floats()) +def test_floats_are_commutative(x, y): + assert x + y == y + x +``` + +``` +Falsifying example: test_floats_are_commutative(x=0.0, y=nan) +``` + +If you run this in verbose mode it will print out +Falsifying example: test\_floats\_are\_commutative(x=0.0, y=nan) immediately and +not try to do any shrinks. Values you pass in via example will not be shrunk. +This is partly a technical limitation but it can often be useful as well. + +Explicitly provided examples are run before any generated examples. + +So, to recap and elaborate, when you use a test written using Hypothesis: + +1. Your test runner sees the decorated test as if it were a perfectly normal test function + and invokes it. +2. Hypothesis calls your test function with each explicitly provided @example. If one + of these fails it stops immediately and bubbles up the exception for the test runner to handle. +3. Hypothesis reads examples out of its database of previously failing examples. If any of them + fail, it stops there and proceeds to the shrinking step with that example. Otherwise it continues + to the generation step. +4. Hypothesis tries generating a number of examples. If any of these raises an exception, it stops + there and proceeds to the shrinking step. If none of them raise an exception, it silently returns + and the test passes. +5. Hypothesis takes the previously failing example it's seen and tries to produce a "Simpler" version + of it. Once it has found the simplest it possibly can, it saves that in the example database (in + actual fact it saves every failing example in the example database as it shrinks, but the reasons + why aren't important right now). +6. Hypothesis takes the simplest failing example and replays it, finally letting the test bubble up to + the test runner. diff --git a/HypothesisWorks.github.io/_posts/2016-04-16-encode-decode-invariant.md b/HypothesisWorks.github.io/_posts/2016-04-16-encode-decode-invariant.md new file mode 100644 index 0000000000..6b009db02c --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-04-16-encode-decode-invariant.md @@ -0,0 +1,156 @@ +--- +layout: post +tags: python intro technical properties +date: 2016-04-16 06:00 +title: The Encode/Decode invariant +published: true +author: drmaciver +--- + +One of the simplest types of invariant to find once you move past +[just fuzzing your code]({{site.url}}{% post_url 2016-04-15-getting-started-with-hypothesis %}) is asserting that two +different operations should produce the same result, and one of the simplest instances of +*that* is looking for encode/decode pairs. That is, you have some function that takes a +value and encodes it as another value, and another that is supposed to reverse the process. + +This is ripe for testing with Hypothesis because it has a natural completely defined +specification: Encoding and then decoding should be exactly the same as doing nothing. + +Lets look at a concrete example. + + + +The following code is a lightly reformatted version of +an implementation of [Run Length Encoding](https://en.wikipedia.org/wiki/Run-length_encoding) +taken [from Rosetta Code](http://rosettacode.org/wiki/Run-length_encoding). + +```python +def encode(input_string): + count = 1 + prev = "" + lst = [] + for character in input_string: + if character != prev: + if prev: + entry = (prev, count) + lst.append(entry) + count = 1 + prev = character + else: + count += 1 + else: + entry = (character, count) + lst.append(entry) + return lst + + +def decode(lst): + q = "" + for character, count in lst: + q += character * count + return q +``` + +We can test this using Hypothesis and py.test as follows: + + +```python +from hypothesis import given +from hypothesis.strategies import text + + +@given(text()) +def test_decode_inverts_encode(s): + assert decode(encode(s)) == s +``` + +This asserts what we described above: If we encode a string as run length encoded and then +decode it, we get back to where we started. + +This test finds a bug, not through the actual invariant. Instead it finds one through pure +fuzzing: The code does not correctly handle the empty string. + + +``` +Falsifying example: test_decode_inverts_encode(s='') + +UnboundLocalError: local variable 'character' referenced before assignment +``` + +One of the nice features of testing invariants is that they incorporate the fuzzing you +could be doing anyway, more or less for free, so even trivial invariants can often +find interesting problems. + +We can fix this bug by adding a guard to the encode function: + +```python +if not input_string: + return [] +``` + +The test now passes, which isn't very interesting, so lets break the code. We'll delete +a line from our implementation of encode which resets the count when the character changes: + + +```python +def encode(input_string): + if not input_string: + return [] + count = 1 + prev = "" + lst = [] + for character in input_string: + if character != prev: + if prev: + entry = (prev, count) + lst.append(entry) + # count = 1 # Missing reset operation + prev = character + else: + count += 1 + else: + entry = (character, count) + lst.append(entry) + return lst +``` + +Now the test fails: + +``` + @given(text()) + def test_decode_inverts_encode(s): +> assert decode(encode(s)) == s +E assert '1100' == '110' +E - 1100 +E ? - +E + 110 + +test_encoding.py:35: AssertionError +------------------------------------ Hypothesis ------------------------------------ +Falsifying example: test_decode_inverts_encode(s='110') + +``` + +Not resetting the count did indeed produce unintended data that doesn't translate back +to the original thing. Hypothesis has given us the shortest example that could trigger +it - two identical characters followed by one different one. It's not *quite* the +simplest example according to Hypothesis's preferred ordering - that would be '001' - +but it's still simple enough to be quite legible, which helps to rapidly diagnose +the problem when you see it in real code. + +Encode/decode loops like this are *very* common, because you will frequently want to +serialize your domain objects to other representations - into forms, into APIs, into +the database, and these are things that are so integral to your applications that it's +worth getting all the edge cases right. + +Other examples of this: + +* [This talk by Matt Bacchman](https://speakerdeck.com/bachmann1234/property-based-testing-hypothesis) + in which he discovers an eccentricity of formats for dates. +* Mercurial bugs [4927](https://bz.mercurial-scm.org/show_bug.cgi?id=4927) and [5031](https://bz.mercurial-scm.org/show_bug.cgi?id=5031) + were found by applying this sort of testing to their internal UTF8b encoding functions. +* [This test](https://github.com/The-Compiler/qutebrowser/blob/24a71e5c2ebbffd9021694f32fa9ec51d0046d5a/tests/unit/browser/test_webelem.py#L652). + Has caught three bugs in Qutebrowser's JavaScript escaping ([1](https://github.com/The-Compiler/qutebrowser/commit/73e9fd11188ce4dddd7626e39d691e0df649e87c), + [2](https://github.com/The-Compiler/qutebrowser/commit/93d27cbb5f49085dd5a7f5e05f2cc45cc84f94a4), + [3](https://github.com/The-Compiler/qutebrowser/commit/24a71e5c2ebbffd9021694f32fa9ec51d0046d5a)), which could have caused data loss if a user had run + into them. diff --git a/HypothesisWorks.github.io/_posts/2016-04-16-quickcheck-in-every-language.html b/HypothesisWorks.github.io/_posts/2016-04-16-quickcheck-in-every-language.html new file mode 100644 index 0000000000..6a27f4bc98 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-04-16-quickcheck-in-every-language.html @@ -0,0 +1,212 @@ +--- +layout: post +tags: alternatives technical +date: 2016-04-16 15:00 +title: QuickCheck in Every Language +published: true +author: drmaciver +--- + +

+There are a lot of ports of QuickCheck, +the original property based testing library, to a variety of different languages. +

+ +

+Some of them are good. Some of them are very good. Some of them are OK. Many are not. +

+ +

+I thought it would be worth keeping track of which are which, so I've put together a list. +

+ + + +

In order to make it onto this list, an implementation has to meet the following criteria:

+ +
    +
  1. Must support random generation of data to a test function. e.g. testing systems based on + smallcheck while interesting and related + don't fit on this list. +
    2. +
  2. It must be fairly straightforward to generate your own custom types.
    3. +
  3. It must support shrinking of falsifying examples.
    4. +
  4. It must be under active development, in the sense that bugs in it will get fixed.
    5. +
  5. Must be under an OSI approved license.
    6. +
+

+In this I've tried to to collect a list of what I think the best ones are for any given language. +I haven't used all of these, but I've used some and read and talked to people about the others. +

+ +

Uncontested Winners by Language

+ +

For many languages there is a clear winner that you should just use. Here are the ones I've +found and what I think of them.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LanguageLibraryOur rating
CtheftGood but does not come with a library of data generators.
C++CppQuickCheckUnsure
Clojuretest.checkVery good
CoqQuickChickUnsure
F#FsCheckVery Good
GogopterUnsure but looks promising.
HaskellHedgehogComparatively new, but looks solid. See below.
JavaQuickTheoriesUnsure. Extremely new but looks promising.
JavaScriptjsverifyGood
PHPErisUnsure. Looks promising.
PythonHypothesisI may be a bit biased on this one, but it's also unambiguously true.
RubyRantlyUnsure. We're not convinced, but the alternatives are definitely worse.
RustQuickcheckUnsure, but probably very good based on initial appearance and usage level.
ScalaScalaCheckVery Good
SwiftSwiftcheckUnsure
+ +

Where when I've said "Unsure" I really just mean that I think it looks good but +I haven't put in the depth of in time to be sure, not that I have doubts.

+ +

Special case: Haskell

+ +

+ The original QuickCheck + was of course written in Haskell, so it may seem odd that it's not the property based testing + library I recommend for Haskell! +

+ +

+ The reason is that I feel that the design of classic QuickCheck is fundamentally limiting, + and that Hedgehog takes it in the direction that the rest of the property-based testing + world is moving (and where most of the implementations for dynamic languages, Hypothesis + included, already are). In particular its approach starts from generators rather than + type classes, and it has integrated shrinking, + and a fairly comprehensive library of generators. +

+ +

Special case: Erlang

+ +

+ Erlang is a special case because they have QuviQ's QuickCheck. + Their QuickCheck implementation is by all accounts extremely good, but it is also proprietary + and fairly expensive. Nevertheless, if you find yourselves in the right language, circumstance and + financial situation to use it, I would strongly recommend doing so. +

+ +

+ In particular, QuviQ's QuickCheck is really the only implementation in this article I think is + simply better than Hypothesis. Hypothesis is significantly more user friendly, especially if the + users in question are less than familiar with Erlang, but there are things QuviQ can do that + Hypothesis can't, and the data generation has had a great deal more engineering effort put into it. +

+ +

+ If you're using Erlang but not able to pay for QuickCheck, apparently the one to use is + PropEr. If you're also unable to use GPLed software + there's triq. I know very little about either. +

+ +

Special case: OCaml

+ +

+ OCaml seems to be suffering from a problem of being close enough to Haskell that people try to do a + straight port of Quickcheck but far enough from Haskell that this doesn't work. The result is that + there is a "mechanical port" of Quickcheck + which is completely abandoned and a fork + of it that uses more idiomatic OCaml. I'm insufficiently familiar with OCaml or its community + to know if either is used or whether there is another one that is. +

+ +

What does this have to do with Hypothesis?

+ +

+ In some sense these are all "competitors" to Hypothesis, but we're perfectly happy not to compete. +

+ +

+ In the case of Erlang, I wouldn't even try. In the case of Scala, F#, or Clojure, I might at some + point work with them to try to bring the best parts of Hypothesis to their existing implementation, + but I don't consider them a priority - they're well served by what they have right now, and there + are many languages that are not. +

+ +

+ For the rest though? I'm glad they exist! I care about testing about about high quality software, + and they're doing their part to make it possible. +

+ +

+ But I feel they're being badly served by their underlying model, and that they feel quite unnatural + to use in the context of a more conventional test setting. I think Hypothesis is the way forward, + and I'll be doing my best to make it + possible for everyone to use it in their language of choice. +

diff --git a/HypothesisWorks.github.io/_posts/2016-04-16-the-purpose-of-hypothesis.md b/HypothesisWorks.github.io/_posts/2016-04-16-the-purpose-of-hypothesis.md new file mode 100644 index 0000000000..c8892eca2d --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-04-16-the-purpose-of-hypothesis.md @@ -0,0 +1,60 @@ +--- +layout: post +tags: writing-good-software principles non-technical +date: 2016-04-16 12:00 +title: The Purpose of Hypothesis +published: true +author: drmaciver +--- + +What is Hypothesis for? + +From the perspective of a user, the purpose of Hypothesis is to make it easier for you +to write better tests. + +From my perspective as the primary author, that is of course also *a* purpose of Hypothesis. +I write a lot of code, it needs testing, and the idea of trying to do that without Hypothesis +has become nearly unthinkable. + +But, on a large scale, the true purpose of Hypothesis is to drag the world kicking and screaming +into a new and terrifying age of high quality software. + + + +Software is everywhere. We have built a civilization on it, and it's only getting more prevalent +as more services move online and embedded and "internet of things" devices become cheaper and +more common. + +Software is also terrible. It’s buggy, it's insecure, and it's rarely well thought out. + +This combination is clearly a recipe for disaster. + +The state of software testing is even worse. It’s uncontroversial at this point that you *should* +be testing your code, but it's a rare codebase whose authors could honestly claim that they feel +its testing is sufficient. + +Much of the problem here is that it’s too hard to write good tests. Tests take up a vast quantity +of development time, but they mostly just laboriously encode exactly the same assumptions and +fallacies that the authors had when they wrote the code, so they miss exactly the same bugs that +you missed when they wrote the code. + +Meanwhile, there are all sorts of tools for making testing better that are basically unused, or +used in only specialised contexts. The original Quickcheck is from 1999 and the majority of +developers have not even heard of it, let alone used it. There are a bunch of half-baked +implementations for most languages, but very few of them are worth using. More recently, there +are many good tools applied to specialized problems, but very little that even attempts, let +alone succeeds, to help general purpose testing. + +The goal of Hypothesis is to fix this, by taking research level ideas and applying solid +engineering to them to produce testing tools that are both powerful *and* practical, and +are accessible to everyone.. + +Many of the ideas that Hypothesis is built on are new. Many of them are not. It doesn't matter. +The purpose of Hypothesis is not to produce research level ideas. The purpose of Hypothesis is +to produce high quality software by any means necessary. Where the ideas we need exist, we +will use them. Where they do not, we will invent them. + +If people aren't using advanced testing tools, that's a bug. We should find it and fix it. + +Fortunately, we have this tool called Hypothesis. It's very good at finding bugs. But this +one it can also fix. diff --git a/HypothesisWorks.github.io/_posts/2016-04-19-rule-based-stateful-testing.md b/HypothesisWorks.github.io/_posts/2016-04-19-rule-based-stateful-testing.md new file mode 100644 index 0000000000..6abcfe1e58 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-04-19-rule-based-stateful-testing.md @@ -0,0 +1,350 @@ +--- +layout: post +tags: python technical intro +date: 2016-04-19 07:00 +title: Rule Based Stateful Testing +published: true +author: drmaciver +--- + +Hypothesis's standard testing mechanisms are very good for testing things that can be +considered direct functions of data. But supposed you have some complex stateful +system or object that you want to test. How can you do that? + +In this article we'll see how to use Hypothesis's *rule based state machines* to define +tests that generate not just simple data, but entire programs using some stateful +object. These will give the same level of boost to testing the behaviour of the +object as you get to testing the data it accepts. + + + +The model of a stateful system we'll be using is a [priority queue](https://en.wikipedia.org/wiki/Priority_queue) +implemented as a binary heap. + +We have the following operations: + +* newheap() - returns a new heap +* heappush(heap, value) - place a new value into the heap +* heappop(heap) - remove and return the smallest value currently on the heap. Error if heap is empty. +* heapempty(heap) - return True if the heap has no elements, else False. + +We'll use the following implementation of these: + +```python +def heapnew(): + return [] + + +def heapempty(heap): + return not heap + + +def heappush(heap, value): + heap.append(value) + index = len(heap) - 1 + while index > 0: + parent = (index - 1) // 2 + if heap[parent] > heap[index]: + heap[parent], heap[index] = heap[index], heap[parent] + index = parent + else: + break + + +def heappop(heap): + return heap.pop(0) +``` + +(Note that this implementation is *wrong*. heappop as implemented will return the smallest element +if the heap currently satisfies the heap property, but it will not rebalance the heap afterwards +so it may leave the heap in an invalid state) + +We could test this readily enough using @given with something like the following: + + +```python +from hypothesis import given +from hypothesis.strategies import integers, lists + + +@given(lists(integers())) +def test_pop_in_sorted_order(ls): + h = heapnew() + for l in ls: + heappush(h, l) + r = [] + while not heapempty(h): + r.append(heappop(h)) + assert r == sorted(ls) +``` + +And this indeed finds the bug: + +``` +> assert r == sorted(ls) +E assert [0, 1, 0] == [0, 0, 1] +E At index 1 diff: 1 != 0 +E Use -v to get the full diff + +binheap.py:74: AssertionError +----- Hypothesis ----- +Falsifying example: test_pop_in_sorted_order(ls=[0, 1, 0]) +``` + +So we replace heappop with a correct implementation which rebalances the heap: + +```python +def heappop(heap): + if len(heap) == 0: + raise ValueError("Empty heap") + if len(heap) == 1: + return heap.pop() + result = heap[0] + heap[0] = heap.pop() + index = 0 + while index * 2 + 1 < len(heap): + children = [index * 2 + 1, index * 2 + 2] + children = [i for i in children if i < len(heap)] + assert children + children.sort(key=lambda x: heap[x]) + for c in children: + if heap[index] > heap[c]: + heap[index], heap[c] = heap[c], heap[index] + index = c + break + else: + break + return result +``` + +But how do we know this is enough? Might some combination of mixing pushes and pops break the +invariants of the heap in a way that this simple pattern of pushing everything then popping +everything cannot witness? + +This is where the rule based state machines come in. Instead of just letting Hypothesis give +us data which we feed into a fixed structure of test, we let Hypothesis choose which operations +to perform on our data structure: + +```python +from hypothesis.stateful import RuleBasedStateMachine, precondition, rule + + +class HeapMachine(RuleBasedStateMachine): + def __init__(self): + super().__init__() + self.heap = [] + + @rule(value=integers()) + def push(self, value): + heappush(self.heap, value) + + @rule() + @precondition(lambda self: self.heap) + def pop(self): + correct = min(self.heap) + result = heappop(self.heap) + assert correct == result +``` + +@rule is a slightly restricted version of @given that only works for methods on a RuleBasedStateMachine. + +However it has one *major* difference from @given, which is that multiple rules can be chained together: +A test using this state machine doesn't just run each rule in isolation, it instantiates an instance of +the machine and then runs multiple rules in succession. + +The @precondition decorator constrains when a rule is allowed to fire: We are not allowed to pop from +an empty heap, so the pop rule may only fire when there is data to be popped. + +We can run this by getting a standard unit test TestCase object out of it to be picked up by unittest +or py.test as normal: + +```python +TestHeaps = HeapMachine.TestCase +``` + +With our original broken heappop we find the same bug as before: + +``` +E AssertionError: assert 0 == 1 + +binheap.py:90: AssertionError +----- Captured stdout call ----- +Step #1: push(value=1) +Step #2: push(value=0) +Step #3: push(value=0) +Step #4: pop() +Step #5: pop() +``` + +With the fixed implementation the test passes. + +As it currently stands, this is already very useful. It's particularly good for testing single standalone +objects or services like storage systems. + +But one limitation of it as we have written it is that it only concerns ourselves with a single heap. What +if we wanted to combine two heaps? For example, suppose we wanted a heap merging operation that takes two +heaps and returns a new heap containing the values in either of the original two. + +As before, we'll start with a broken implementation: + +```python +def heapmerge(x, y): + x, y = sorted((x, y)) + return x + y +``` + +We can't just write a strategy for heaps, because each heap would be a fresh object and thus it would not +preserve the stateful aspect. + +What we instead do is use the other big feature of Hypothesis's rule bases state machines: Bundles. + +Bundles allow rules to return as well as accept values. A bundle is a strategy which generates anything +a rule has previously provided to it. Using them is as follows: + + +```python +class HeapMachine(RuleBasedStateMachine): + Heaps = Bundle("heaps") + + @rule(target=Heaps) + def newheap(self): + return [] + + @rule(heap=Heaps, value=integers()) + def push(self, heap, value): + heappush(heap, value) + + @rule(heap=Heaps.filter(bool)) + def pop(self, heap): + correct = min(heap) + result = heappop(heap) + assert correct == result + + @rule(target=Heaps, heap1=Heaps, heap2=Heaps) + def merge(self, heap1, heap2): + return heapmerge(heap1, heap2) +``` + +So now instead of a single heap we manage a collection of heaps. All of our previous operations become +constrained by an instance of a heap. + +Note the use of filter: A bundle is a strategy you can use like any other. In this case the filter replaces +our use of a precondition because we now only care about whether this *specific* heap is empty. + +This is sufficient to find the fact that our implementation is wrong: + +``` + @rule(heap=Heaps.filter(bool)) + def pop(self, heap): + correct = min(heap) + result = heappop(heap) +> assert correct == result +E AssertionError: assert 0 == 1 + +binheap.py:105: AssertionError + +----- Captured stdout call ----- + +Step #1: v1 = newheap() +Step #2: push(heap=v1, value=0) +Step #3: push(heap=v1, value=1) +Step #4: push(heap=v1, value=1) +Step #5: v2 = merge(heap2=v1, heap1=v1) +Step #6: pop(heap=v2) +Step #7: pop(heap=v2) +``` + +We create a small heap, merge it with itself, and rapidly discover that it has become unbalanced. + +We can fix this by fixing our heapmerge to be correct: + +```python +def heapmerge(x, y): + result = list(x) + for v in y: + heappush(result, v) + return result +``` + +But that's boring. Lets introduce a more *interestingly* broken implementation instead: + +```python +def heapmerge(x, y): + result = [] + i = 0 + j = 0 + while i < len(x) and j < len(y): + if x[i] <= y[j]: + result.append(x[i]) + i += 1 + else: + result.append(y[j]) + j += 1 + result.extend(x[i:]) + result.extend(y[j:]) + return result +``` + +This merge operation selectively splices two heaps together as if we were merging two +sorted lists (heaps aren't actually sorted, but the code still works regardless it +just doesn't do anything very meaningful). + +This is wrong, but it turns out to work surprisingly well for small heaps and it's +not completely straightforward to find an example showing that it's wrong. + +Here's what Hypothesis comes up with: + +``` +Step #1: v1 = newheap() +Step #2: push(heap=v1, value=0) +Step #3: v2 = merge(heap1=v1, heap2=v1) +Step #4: v3 = merge(heap1=v2, heap2=v2) +Step #5: push(heap=v3, value=-1) +Step #6: v4 = merge(heap1=v1, heap2=v2) +Step #7: pop(heap=v4) +Step #8: push(heap=v3, value=-1) +Step #9: v5 = merge(heap1=v1, heap2=v2) +Step #10: v6 = merge(heap1=v5, heap2=v4) +Step #11: v7 = merge(heap1=v6, heap2=v3) +Step #12: pop(heap=v7) +Step #13: pop(heap=v7) +``` + +Through a careful set of heap creation and merging, Hypothesis manages to find a series +of merges that produce an unbalanced heap. Every heap prior to v7 is balanced, but v7 looks +like this: + +``` +>>> v7 +[-1, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0] +``` + +Which doesn't satisfy the heap property because of that -1 far down in the list. + +I don't know about you, but I would never have come up with that example. There's probably +a simpler one given a different set of operations - e.g. one thing that would probably improve +the quality of this test is to let Hypothesis instantiate a new heap with a list of elements +which it pops onto it. + +But the nice thing about rule based stateful testing is that I don't *have* to come up with +the example. Instead Hypothesis is able to guarantee that every combination of operations +on my objects works, and can flush out some remarkably subtle bugs in the process. + +Because after all, if it takes this complicated an example to demonstrate that a completely +wrong implementation is wrong, how hard can it sometimes be to demonstrate subtle bugs? + +### Real world usage + +This feature is currently somewhat under-documented so hasn't seen as widespread adoption as +it could. However, there are at least two interesting real world examples: + +1. Hypothesis uses it to test itself. Hypothesis has [tests of its example database]( + https://github.com/HypothesisWorks/hypothesis-python/blob/master/tests/cover/test_database_agreement.py) + which work very much like the above, and [a small model of its test API]( + https://github.com/HypothesisWorks/hypothesis-python/blob/master/tests/nocover/test_strategy_state.py) + which generates random strategies and runs tests using them. +2. It's being used to [test Mercurial](https://www.mercurial-scm.org/pipermail/mercurial-devel/2016-February/080037.html) + generating random. So far it's found [bug 5112](https://bz.mercurial-scm.org/show_bug.cgi?id=5112) and + [bug 5113](https://bz.mercurial-scm.org/show_bug.cgi?id=5113). The usage pattern on Mercurial is one + such that the stateful testing probably needs more resources, more rules and more work on deployment + before it's going to find much more than that though. diff --git a/HypothesisWorks.github.io/_posts/2016-04-29-testing-performance-optimizations.md b/HypothesisWorks.github.io/_posts/2016-04-29-testing-performance-optimizations.md new file mode 100644 index 0000000000..8f180dc3ff --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-04-29-testing-performance-optimizations.md @@ -0,0 +1,129 @@ +--- +layout: post +tags: technical intro python properties +date: 2016-04-29 11:00 +title: Testing performance optimizations +published: true +author: drmaciver +--- + +Once you've +[flushed out the basic crashing bugs]({{site.url}}{% post_url 2016-04-15-getting-started-with-hypothesis %}) +in your code, you're going to want to look for more interesting things to test. + +The next easiest thing to test is code where you know what the right answer is for every input. + +Obviously in theory you think you know what the right answer is - you can just run the code. That's not very +helpful though, as that's the answer you're trying to verify. + +But sometimes there is more than one way to get the right answer, and you choose the one you run in production +not because it gives a different answer but because it gives the same answer *faster*. + + + +For example: + +* There might be a fancy but fast version of an algorithm and a simple but slow version of an algorithm. +* You might have a caching layer and be able to run the code with and without caching turned on, or with a + different cache timeout. +* You might be moving to a new database backend to improve your scalability, but you still have the code for + the old backend until you've completed your migration. + +There are plenty of other ways this can crop up, but those are the ones that seem the most common. + +Anyway, this creates an *excellent* use case for property based testing, because if two functions are supposed +to always return the same answer, you can test that: Just call both functions with the same data and assert +that their answer is the same. + +Lets look at this in the fancy algorithm case. Suppose we implemented [merge sort]( +https://en.wikipedia.org/wiki/Merge_sort): + +```python +def merge_sort(ls): + if len(ls) <= 1: + return ls + else: + k = len(ls) // 2 + return merge_sorted_lists(merge_sort(ls[:k]), merge_sort(ls[k:])) + + +def merge_sorted_lists(x, y): + result = [] + i = 0 + j = 0 + while i < len(x) and j < len(y): + if x[i] <= y[j]: + result.append(x[i]) + i += 1 + else: + result.append(y[j]) + j += 1 + return result +``` + +We want a reference implementation to test it against, so lets also implement [bubble sort]( +https://en.wikipedia.org/wiki/Bubble_sort): + +```python +def bubble_sort(ls): + ls = list(ls) + needs_sorting = True + while needs_sorting: + needs_sorting = False + for i in range(1, len(ls)): + if ls[i - 1] > ls[i]: + needs_sorting = True + ls[i - 1], ls[i] = ls[i], ls[i - 1] + return ls +``` + +These *should* always give the same answer, so lets test that: + +```python +@given(lists(integers())) +def test_bubble_sorting_is_same_as_merge_sorting(ls): + assert bubble_sort(ls) == merge_sort(ls) +``` + +This gives us an error: + +``` + @given(lists(integers())) + def test_bubble_sorting_is_same_as_merge_sorting(ls): +> assert bubble_sort(ls) == merge_sort(ls) +E assert [0, 0] == [0] +E Left contains more items, first extra item: 0 +E Use -v to get the full diff + +foo.py:43: AssertionError +----- Hypothesis ----- +Falsifying example: test_bubble_sorting_is_same_as_merge_sorting(ls=[0, 0]) +``` + +What's happened is that we messed up our implementation of merge\_sorted\_lists, because we forgot +to include the elements left over in the other list once we've reached the end of one of them. As a +result we ended up losing elements from the list, a problem that our simpler implementation lacks. +We can fix this as follows and then the test passes: + +```python +def merge_sorted_lists(x, y): + result = [] + i = 0 + j = 0 + while i < len(x) and j < len(y): + if x[i] <= y[j]: + result.append(x[i]) + i += 1 + else: + result.append(y[j]) + j += 1 + result.extend(x[i:]) + result.extend(y[j:]) + return result +``` + +This technique combines especially well with +[Hypothesis's stateful testing]({{site.url}}{% post_url 2016-04-19-rule-based-stateful-testing %}), because +you can use it to then test different implementations of complex APIs. For example, Hypothesis uses this +property together with stateful testing to [verify that the different implementations of its example database +behave identically](https://github.com/HypothesisWorks/hypothesis/blob/master/hypothesis-python/tests/nocover/test_database_agreement.py). diff --git a/HypothesisWorks.github.io/_posts/2016-05-02-referential-transparency.md b/HypothesisWorks.github.io/_posts/2016-05-02-referential-transparency.md new file mode 100644 index 0000000000..e45dd6a876 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-05-02-referential-transparency.md @@ -0,0 +1,25 @@ +--- +layout: post +tags: non-technical +date: 2016-05-02 08:00 +title: You Don't Need Referential Transparency +published: true +author: drmaciver +--- + +It's a common belief that in order for property based testing to be useful, your code must be [referentially transparent](https://en.wikipedia.org/wiki/Referential_transparency). That is, it must be a pure function with no side effects that just takes input data and produces output data and is solely defined by what input data produces what output data. + +This is, bluntly, complete and utter nonsense with no basis in reality. + + + + +The idea comes from the fact that it was true of very early versions of [the original Haskell QuickCheck](https://hackage.haskell.org/package/QuickCheck) - it was designed to look more like formal methods than unit testing, and it was designed for a language where referential transparency was the norm. + +But that was the *original* version of Haskell QuickCheck. It's not even true for the latest version of it, let alone for ports to other languages! The Haskell version has full support for testing properties in IO (if you don't know Haskell, this means "tests which may have side effects"). It works really well. Hypothesis doesn't even consider this a question - testing code with side effects works the same way as testing code without side effects. + +The *only* requirement that property based testing has on the side effects your tests may perform is that if your test has *global* side effects then it must be able to roll them back at the end. + +If that sounds familiar, it's because it's *exactly the same requirement every other test has*. Tests that have global side effects are not repeatable and may interfere with other tests, so they must keep their side effects to themselves by rolling them back at the end of the test. + +Property based testing is just normal testing, run multiple times, with a source of data to fill in some of the blanks. There is no special requirement on it beyond that, and the myth that there is causes great harm and keeps many people from adopting more powerful testing tools. diff --git a/HypothesisWorks.github.io/_posts/2016-05-11-generating-the-right-data.md b/HypothesisWorks.github.io/_posts/2016-05-11-generating-the-right-data.md new file mode 100644 index 0000000000..fe6aeab0fd --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-05-11-generating-the-right-data.md @@ -0,0 +1,372 @@ +--- +layout: post +tags: technical python intro +date: 2016-05-11 10:00 +title: Generating the right data +published: true +author: drmaciver +--- + +One thing that often causes people problems is figuring out how to generate +the right data to fit their data +model. You can start with just generating strings and integers, but eventually you want +to be able to generate +objects from your domain model. Hypothesis provides a lot of tools to help you build the +data you want, but sometimes the choice can be a bit overwhelming. + +Here's a worked example to walk you through some of the details and help you get to grips with how to use +them. + + + +Suppose we have the following class: + +```python +class Project: + def __init__(self, name, start, end): + self.name = name + self.start = start + self.end = end + + def __repr__(self): + return "Project {} from {} to {}".format( + self.name, self.start.isoformat(), self.end.isoformat() + ) +``` + +A project has a name, a start date, and an end date. + +How do we generate such a thing? + +The idea is to break the problem down into parts, and then use the tools +Hypothesis provides to assemble those parts into a strategy for generating +our projects. + +We'll start by generating the data we need for each field, and then at the end +we'll see how to put it all together to generate a Project. + +### Names + +First we need to generate a name. We'll use Hypothesis's standard text +strategy for that: + +```pycon +>>> from hypothesis.strategies import text +>>> text().example() +'' +>>> text().example() +'\nŁ昘迥' +``` + +Lets customize this a bit: First off, lets say project names have to be +non-empty. + +```pycon +>>> text(min_size=1).example() +'w\nC' +>>> text(min_size=1).example() +'ሚಃJ»' +``` + +Now, lets avoid the high end unicode for now (of course, your system *should* +handle the full range of unicode, but this is just an example, right?). + +To do this we need to pass an alphabet to the text strategy. This can either be +a range of characters or another strategy. We're going to use the *characters* +strategy, which gives you a flexible way of describing a strategy for single-character +text strings, to do that. + +```pycon +i>>> characters(min_codepoint=1, max_codepoint=1000, blacklist_categories=('Cc', 'Cs')).example() +'²' +>>> characters(min_codepoint=1, max_codepoint=1000, blacklist_categories=('Cc', 'Cs')).example() +'E' +>>> characters(min_codepoint=1, max_codepoint=1000, blacklist_categories=('Cc', 'Cs')).example() +'̺' + +``` + +The max and min codepoint parameters do what you'd expect: They limit the range of +permissible codepoints. We've blocked off the 0 codepoint (it's not really useful and +tends to just cause trouble with C libraries) and anything with a codepoint above +1000 - so we're considering non-ASCII characters but nothing really high end. + +The blacklist\_categories parameter uses the notion of [unicode category](https://en.wikipedia.org/wiki/Unicode_character_property#General_Category) +to limit the range of acceptable characters. If you want to see what category a +character has you can use Python's unicodedata module to find out: + +```pycon +>>> from unicodedata import category +>>> category('\n') +'Cc' +>>> category('\t') +'Cc' +>>> category(' ') +'Zs' +>>> category('a') +'Ll' +``` + +The categories we've excluded are *control characters* and *surrogates*. Surrogates +are excluded by default but when you explicitly pass in blacklist categories you +need to exclude them yourself. + +So we can put that together with text() to get a name matching our requirements: + +```pycon +>>> names = text(characters(max_codepoint=1000, blacklist_categories=('Cc', 'Cs')), min_size=1) +``` + +But this is still not quite right: We've allowed spaces in names, but we don't really want +a name to start with or end with a space. You can see that this is currently allowed by +asking Hypothesis for a more specific example: + +```pycon +>>> find(names, lambda x: x[0] == ' ') +' ' +``` + +So lets fix it so that they can't by stripping the spaces off it. + +To do this we're going to use the strategy's *map* method which lets you compose it with +an arbitrary function to post-process the results into the for you want: + +```pycon +>>> names = text(characters(max_codepoint=1000, blacklist_categories=('Cc', 'Cs')), min_size=1).map( +... lambda x: x.strip()) +``` + +Now lets check that we can no longer have the above problem: + +```pycon +>>> find(names, lambda x: x[0] == ' ') +Traceback (most recent call last): + File "", line 1, in + File "/usr/lib/python3.5/site-packages/hypothesis/core.py", line 648, in find + runner.run() + File "/usr/lib/python3.5/site-packages/hypothesis/internal/conjecture/engine.py", line 168, in run + self._run() + File "/usr/lib/python3.5/site-packages/hypothesis/internal/conjecture/engine.py", line 262, in _run + self.test_function(data) + File "/usr/lib/python3.5/site-packages/hypothesis/internal/conjecture/engine.py", line 68, in test_function + self._test_function(data) + File "/usr/lib/python3.5/site-packages/hypothesis/core.py", line 616, in template_condition + success = condition(result) + File "", line 1, in +IndexError: string index out of range +``` + +Whoops! + +The problem is that our initial test worked because the strings we were generating were always +non-empty because of the min\_size parameter. We're still only generating non-empty strings, +but if we generate a string which is all spaces then strip it, the result will be empty +*after* our map. + +We can fix this using the strategy's *filter* function, which restricts to only generating +things which satisfy some condition: + +```pycon +>>> names = text(characters(max_codepoint=1000, blacklist_categories=('Cc', 'Cs')), min_size=1).map( +... lambda s: s.strip()).filter(lambda s: len(s) > 0) +``` + +And repeating the check: + +```pycon +>>> find(names, lambda x: x[0] == ' ') +Traceback (most recent call last): + File "", line 1, in + File "/usr/lib/python3.5/site-packages/hypothesis/core.py", line 670, in find + raise NoSuchExample(get_pretty_function_description(condition)) +hypothesis.errors.NoSuchExample: No examples found of condition lambda x: +``` + +Hypothesis raises NoSuchExample to indicate that... well, that there's no such example. + +In general you should be a little bit careful with filter and only use it to filter +to conditions that are relatively hard to happen by accident. In this case it's fine +because the filter condition only fails if our initial draw was a string consisting +entirely of spaces, but if we'd e.g. tried the opposite and tried to filter to strings +that *only* had spaces, we'd have had a bad time of it and got a very slow and not +very useful test. + +Anyway, we now really do have a strategy that produces decent names for our projects. +Lets put this all together into a test that demonstrates that our names now have the +desired properties: + +```python +from unicodedata import category + +from hypothesis import given +from hypothesis.strategies import characters, text + +names = ( + text(characters(max_codepoint=1000, blacklist_categories=("Cc", "Cs")), min_size=1) + .map(lambda s: s.strip()) + .filter(lambda s: len(s) > 0) +) + + +@given(names) +def test_names_match_our_requirements(name): + assert len(name) > 0 + assert name == name.strip() + for c in name: + assert 1 <= ord(c) <= 1000 + assert category(c) not in ("Cc", "Cs") +``` + +It's not common practice to write tests for your strategies, but it can be helpful +when trying to figure things out. + +### Dates and times + +Hypothesis has date and time generation in a hypothesis.extra subpackage because it +relies on pytz to generate them, but other than that it works in exactly the same +way as before: + +```pycon +>>> from hypothesis.extra.datetime import datetimes +>>> datetimes().example() +datetime.datetime(1642, 1, 23, 2, 34, 28, 148985, tzinfo=) +``` + +Lets constrain our dates to be UTC, because the sensible thing to do is to use UTC +internally and convert on display to the user: + +```pycon +>>> datetimes(timezones=('UTC',)).example() +datetime.datetime(6820, 2, 4, 19, 16, 27, 322062, tzinfo=) +``` + +We can also constrain our projects to start in a reasonable range of years, +as by default Hypothesis will cover the whole of representable history: + +```pycon +>>> datetimes(timezones=('UTC',), min_year=2000, max_year=2100).example() +datetime.datetime(2084, 6, 9, 11, 48, 14, 213208, tzinfo=) +``` + +Again we can put together a test that checks this behaviour (though we have +less code here so it's less useful): + +```python +from hypothesis import given +from hypothesis.extra.datetime import datetimes + +project_date = datetimes(timezones=("UTC",), min_year=2000, max_year=2100) + + +@given(project_date) +def test_dates_are_in_the_right_range(date): + assert 2000 <= date.year <= 2100 + assert date.tzinfo._tzname == "UTC" +``` + +### Putting it all together + +We can now generate all the parts for our project definitions, but how do we +generate a project? + +The first thing to reach for is the *builds* function. + +```pycon + +>>> from hypothesis.strategies import builds +>>> projects = builds(Project, name=names, start=project_date, end=project_date) +>>> projects.example() +Project 'd!#ñcJν' from 2091-06-22T06:57:39.050162+00:00 to 2057-06-11T02:41:43.889510+00:00 +``` + +builds lets you take a set of strategies and feed their results as arguments to a +function (or, in this case, class. Anything callable really) to create a new +strategy that works by drawing those arguments then passing them to the function +to give you that example. + +Unfortunately, this isn't quite right: + +```pycon +>>> find(projects, lambda x: x.start > x.end) +Project '0' from 2000-01-01T00:00:00.000001+00:00 to 2000-01-01T00:00:00+00:00 +``` + +Projects can start after they end when we use builds this way. One way to fix this would be +to use filter(): + +```pycon +>>> projects = builds(Project, name=names, start=project_date, end=project_date).filter( +... lambda p: p.start < p.end) +>>> find(projects, lambda x: x.start > x.end) +Traceback (most recent call last): + File "", line 1, in + File "/usr/lib/python3.5/site-packages/hypothesis/core.py", line 670, in find + raise NoSuchExample(get_pretty_function_description(condition)) +hypothesis.errors.NoSuchExample: No examples found of condition lambda x: +``` + +This will work, but it starts to edge into the territory of where filter should be +avoided - about half of the initially generated examples will fail the filter. + +What we'll do instead is draw two dates and use whichever one is smallest as the +start, and whatever is largest at the end. This is hard to do with builds because +of the dependence between the arguments, so instead we'll use builds' more advanced +cousin, *composite*: + +```python +from hypothesis import assume +from hypothesis.strategies import composite + + +@composite +def projects(draw): + name = draw(names) + date1 = draw(project_date) + date2 = draw(project_date) + assume(date1 != date2) + start = min(date1, date2) + end = max(date1, date2) + return Project(name, start, end) +``` + +The idea of composite is you get passed a magic first argument 'draw' that you can +use to get examples out of a strategy. You then make as many draws as you want and +use these to return the desired data. + +You can also use the *assume* function to discard the current call if you get yourself +into a state where you can't proceed or where it's easier to start again. In this case +we do that when we draw the same data twice. + +```pycon +>>> projects().example() +Project 'rĂ5ĠǓ#' from 2000-05-14T07:21:12.282521+00:00 to 2026-05-12T13:20:43.225796+00:00 +>>> find(projects(), lambda x: x.start > x.end) +Traceback (most recent call last): + File "", line 1, in + File "/usr/lib/python3.5/site-packages/hypothesis/core.py", line 670, in find + raise NoSuchExample(get_pretty_function_description(condition)) +hypothesis.errors.NoSuchExample: No examples found of condition lambda x: +``` + +Note that in all of our examples we're now writing projects() instead of projects. That's +because composite returns a function rather than a strategy. Any arguments to your +defining function other than the first are also arguments to the one produced by composite. + +We can now put together one final test that we got this bit right too: + +```python +@given(projects()) +def test_projects_end_after_they_started(project): + assert project.start < project.end +``` + +### Wrapping up + +There's a lot more to Hypothesis's data generation than this, but hopefully it gives you +a flavour of the sort of things to try and the sort of things that are possible. + +It's worth having a read of [the documentation](https://hypothesis.readthedocs.io/en/latest/data.html) +for this, and if you're still stuck then try asking [the community](https://hypothesis.readthedocs.io/en/latest/community.html) +for some help. We're pretty friendly. + + diff --git a/HypothesisWorks.github.io/_posts/2016-05-13-what-is-property-based-testing.md b/HypothesisWorks.github.io/_posts/2016-05-13-what-is-property-based-testing.md new file mode 100644 index 0000000000..c3760a4bba --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-05-13-what-is-property-based-testing.md @@ -0,0 +1,136 @@ +--- +layout: post +tags: non-technical philosophy +date: 2016-05-14 09:00 +title: What is Property Based Testing? +published: true +author: drmaciver +--- + +I get asked this a lot, and I write property based testing tools for a living, so you'd think +I have a good answer to this, but historically I haven't. This is my attempt to fix that. + +Historically the definition of property based testing has been "The thing that +[QuickCheck]({{site.url}}{% post_url 2016-04-16-quickcheck-in-every-language %}) does". As +a working definition this has served pretty well, but the problem is that it makes us unable +to distinguish what the essential features of property-based testing are and what are just +[accidental features that appeared in the implementations that we're used to]( + {{site.url}}{% post_url 2016-05-02-referential-transparency %}). + +As the author of a property based testing system which diverges quite a lot from QuickCheck, +this troubles me more than it would most people, so I thought I'd set out some of my thoughts on +what property based testing is and isn't. + +This isn't intended to be definitive, and it will evolve over time as my thoughts do, but +it should provide a useful grounding for further discussion. + + + +There are essentially two ways we can draw a boundary for something like this: We can go +narrow or we can go wide. i.e. we can restrict our definitions to things that look exactly +like QuickCheck, or things that are in the same general family of behaviour. My inclination +is always to go wide, but I'm going to try to rein that in for the purpose of this piece. + +But I'm still going to start by planting a flag. The following are *not* essential features +of property based testing: + +1. [Referential Transparency]({{site.url}}{% post_url 2016-05-02-referential-transparency %}). +2. Types +3. Randomization +4. The use of any particular tool or library + +As evidence I present the following: + +1. Almost every property based testing library, including but not limited to Hypothesis and + QuickCheck (both Erlang and Haskell). +2. The many successful property based testing systems for dynamic languages. e.g. Erlang + QuickCheck, test.check, Hypothesis. +3. [SmallCheck](https://hackage.haskell.org/package/smallcheck). I have mixed feelings about + its effectiveness, but it's unambiguously property-based testing. +4. It's very easy to hand-roll your own testing protocols for property-based testing of a + particular result. For example, I've [previously done this for testing a code formatter]( + http://www.drmaciver.com/2015/03/27-bugs-in-24-hours/): Run it over a corpus (more on + whether running over a corpus "really" counts in a second) of Python files, check whether + the resulting formatted code satisfies PEP8. It's classic property-based testing with an + oracle. + +So that provides us with a useful starting point of things that are definitely property based +testing. But you're never going to find a good definition by looking at only positive examples, +so lets look at some cases where it's more arguable. + +First off, lets revisit that parenthetical question: Does just testing against a large corpus count? + +I'm going to go with "probably". I think if we're counting SmallCheck we need to count testing +against a large corpus: If you take the first 20k outputs that would be generated by SmallCheck +and just replay the test using those the first N of those each time, you're doing exactly the +same sort of testing. Similarly if you draw 20k outputs using Hypothesis and then just randomly +sample from them each time. + +I think drawing from a small, fixed, corpus probably *doesn't* count. If you could feasibly +write a property based test as 10 example based tests in line in your source code, it's +probably really just example based testing. This boundary is a bit, um, fuzzy though. + +On which note, what about fuzzing? + +I have previously argued that fuzzing is just a form of property-based testing - you're testing +the property "it doesn't crash". I think I've reversed my opinion on this. In particular, I think +[the style of testing I advocate for getting started with Hypothesis]( + {{site.url}}{% post_url 2016-04-15-getting-started-with-hypothesis %}), *probably* doesn't +count as property based testing. + +I'm unsure about this boundary. The main reason I'm drawing it here is that they do feel like +they have a different character - property based testing requires you to reason about how your +program should behave, while fuzzing can just be applied to arbitrary programs with minimal +understanding of their behaviour - and also that fuzzing feels somehow more fundamental. + +But you can certainly do property based testing using fuzzing tools, in the same way that you +can do it with hand-rolled property based testing systems - I could have taken my Python formatting +test above, added [python-afl](http://jwilk.net/software/python-afl) to the mix, and +that would still be property based testing. + +Conversely, you can do fuzzing with property-based testing tools: If fuzzing is not property +based testing then not all tests using Hypothesis, QuickCheck, etc. are property based tests. +I'm actually OK with that. There's a long tradition of testing tools being used outside their +domain - e.g. most test frameworks originally designed as unit testing tools end up getting +being used for the whole gamut of testing. + +So with that in mind, lets provide a definition of fuzzing that I'd like to use: + +> Fuzzing is feeding a piece of code (function, program, etc.) data from a large corpus, possibly +> dynamically generated, possibly dependent on the results of execution on previous data, in +> order to see whether it fails. + +The definitions of "data" and "whether it fails" will vary from fuzzer to fuzzer - some fuzzers +will generate only binary data, some fuzzers will generate more structured data. Some fuzzers +will look for a process crash, some might just look for a function returning false. + +(Often definitions of fuzzing focus on "malformed" data. I think this is misguided and fails +to consider a lot of things that people would obviously consider fuzzers. e.g. [CSmith](https://embed.cs.utah.edu/csmith/) +is certainly a type of fuzzer but deliberately only generates well formed C programs). + +And given that definition, I think I can now provide a definition of property-based testing: + +> Property based testing is the construction of tests such that, when these tests are fuzzed, +> failures in the test reveal problems with the system under test that could not have been +> revealed by direct fuzzing of that system. + +(If you feel strongly that fuzzing *should* count as property-based testing you can just drop +the 'that could not have been etc.' part. I'm on the fence about it myself.) + +These extra modes of failure then constitute the properties that we're testing. + +I think this works pretty well at capturing what we're doing with property based testing. It's +not perfect, but I'm pretty happy with it. One of the things I particularly like is that it makes +it clear that property-based testing is what *you* do, not what the computer does. The part +that the computer does is "just fuzzing". + +Under this point of view, a property-based testing library is really two parts: + +1. A fuzzer. +2. A library of tools for making it easy to construct property-based tests using that fuzzer. + +Hypothesis is very explicitly designed along these lines - the core of it is a structured +fuzzing library called Conjecture - which suggests that I may have a bit of bias here, but +I still feel that it captures the behaviour of most other property based testing systems +quite well, and provides quite a nice middle ground between the wider definition that I +wanted and the more tightly focused definition that QuickCheck orients people around. diff --git a/HypothesisWorks.github.io/_posts/2016-05-19-announcing-hypothesis-legacy-support.md b/HypothesisWorks.github.io/_posts/2016-05-19-announcing-hypothesis-legacy-support.md new file mode 100644 index 0000000000..39ad915209 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-05-19-announcing-hypothesis-legacy-support.md @@ -0,0 +1,24 @@ +--- +layout: post +tags: news python +date: 2016-05-19 12:00 +title: Announcing Hypothesis Legacy Support +published: true +author: drmaciver +--- + +For a brief period, Python 2.6 was supported in [Hypothesis for Python]( + {{site.url}}/products/#hypothesis-for-python +). Because Python 2.6 has been end of lifed for some time, I decided this wasn't +a priority and support was dropped in Hypothesis 2.0. + +I've now added it back, but under a more restrictive license. + +If you want to use Hypothesis on Python 2.6, you can now do so by installing +the [hypothesislegacysupport]({{site.url}}/products/#hypothesis-legacy-support) +package. This will allow you to run Hypothesis on Python 2.6. + +Note that by default this is licensed under the [GNU Affero General Public License 3.0]( +https://www.gnu.org/licenses/agpl-3.0.en.html). If you want to use it in commercial +software you will likely want to buy a commercial license. Email us at [licensing@hypothesis.works](mailto:licensing@hypothesis.works) +to discuss details. diff --git a/HypothesisWorks.github.io/_posts/2016-05-26-exploring-voting-with-hypothesis.md b/HypothesisWorks.github.io/_posts/2016-05-26-exploring-voting-with-hypothesis.md new file mode 100644 index 0000000000..472705c165 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-05-26-exploring-voting-with-hypothesis.md @@ -0,0 +1,180 @@ +--- +layout: post +tags: python technical example +date: 2016-05-26 11:00 +title: Exploring Voting Systems with Hypothesis +published: true +author: drmaciver +--- + +Hypothesis is, of course, a library for writing tests. + +But from an *implementation* point of view this is hardly noticeable. +Really it's a library for constructing and exploring data and using it +to prove or disprove hypotheses about it. It then has a small testing +library built on top of it. + +It's far more widely used as a testing library, and that's really where +the focus of its development lies, but with the *find* function you can +use it just as well to explore your data interactively. + +In this article we'll go through an example of doing this, by using it +to take a brief look at one of my other favourite subjects: Voting +systems. + + + +We're going to focus entirely on single winner preferential voting +systems: You have a set of candidates, and every voter gives a complete +ordering of the candidates from their favourite to their least +favourite. The voting system then tries to select a single candidate and +declare them the winner. + +The general Python interface for a voting system we'll use is things +that look like the following: + +```python +def plurality_winner(election): + counts = Counter(vote[0] for vote in election) + alternatives = candidates_for_election(election) + winning_score = max(counts.values()) + winners = [c for c, v in counts.items() if v == winning_score] + if len(winners) > 1: + return None + else: + return winners[0] +``` + +That is, they take a list of individual votes, each expressed +as a list putting the candidates in order, and return a candidate that +is an unambiguous winner or None in the event of a tie. + +The above implements plurality voting, what most people might think of +as "normal voting": The candidate with the most first preference votes +wins. + +The other main voting system we'll consider is Instant Runoff Voting ( +which you might know under the name "Alternative Vote" if you follow +British politics): + +```python +def irv_winner(election): + candidates = candidates_for_election(election) + while len(candidates) > 1: + scores = Counter() + for vote in election: + for c in vote: + if c in candidates: + scores[c] += 1 + break + losing_score = min(scores[c] for c in candidates) + candidates = [c for c in candidates if scores[c] > losing_score] + if not candidates: + return None + else: + return candidates[0] +``` + +In IRV, we run the vote in multiple rounds until we've eliminated all +but one candidate. In each round, we give each candidate a score which +is the number of voters who have ranked that candidate highest amongst +all the ones remaining. The candidates with the joint lowest score +drop out. + +At the end, we'll either have either zero or one candidates remaining ( +we can have zero if all candidates are tied for joint lowest score at +some point). If we have zero, that's a draw. If we have one, that's a +victory. + +It seems pretty plausible that these would not produce the same answer +all the time (it would be surpising if they did!), but it's maybe not +obvious how you would go about constructing an example that shows it. + +Fortunately, we don't have to because Hypothesis can do it for us! + +We first create a strategy which generates elections, using Hypothesis's +composite decorator: + +```python +import hypothesis.strategies as st + + +@st.composite +def election(draw): + candidates = list(range(draw(st.integers(2, 10)))) + return draw(st.lists(st.permutations(candidates), min_size=1)) +``` + +This first draws the set of candidates as a list of integers of size +between 2 and 10 (it doesn't really matter what our candidates are as +long as they're distinct, so we use integers for simplicity). It then +draws an election as lists of permutations of those candidates, as we +defined it above. + +We now write a condition to look for: + +```python +def differing_without_ties(election): + irv = irv_winner(election) + if irv is None: + return False + plurality = plurality_winner(election) + if plurality is None: + return False + return irv != plurality +``` + +That is, we're interested in elections where neither plurality nor IRV +resulted in a tie, but they resulted in distinct candidates winning. + +We can now run this in the console: + +``` +>>> from hypothesis import find +>>> import voting as v +>>> distinct = find(v.election(), v.differing_without_ties) +>>> distinct +[[0, 1, 2], + [0, 1, 2], + [1, 0, 2], + [2, 1, 0], + [0, 1, 2], + [0, 1, 2], + [1, 0, 2], + [1, 0, 2], + [2, 1, 0]] +``` + +The example is a bit large, mostly because we insisted on there being +no ties: If we'd broken ties arbitrarily (e.g. preferring the lower +numbered candidates) we could have found a smaller one. Also, in some +runs Hypothesis ends up finding a slightly smaller election but with +four candidates instead of three. + +We can check to make sure that these really do give different results: + +``` +>>> v.irv_winner(distinct) +1 + +>>> v.plurality_winner(distinct) +0 +``` + +There are a lot of other interesting properties of voting systems to +explore, but this is an article about Hypothesis rather than one about +voting, so I'll stop here. However the interested reader might want to +try to build on this to: + +1. Find an election which has a [Condorcet Cycle](https://en.wikipedia.org/wiki/Voting_paradox) +2. Find elections in which the majority prefers the plurality winner to + the IRV winner and vice versa. +3. Use @given rather than find and write some tests verifying some of + [the classic properties of election systems](https://en.wikipedia.org/wiki/Voting_system#Evaluating_voting_systems_using_criteria). + +And the reader who isn't that interested in voting systems might still +want to think about how this could be useful in other areas: Development +is often a constant series of small experiments and, while testing is +often a good way to perform them, sometimes you just have a more +exploratory "I wonder if...?" question to answer, and it can be +extremely helpful to be able to bring Hypothesis to bear there too. diff --git a/HypothesisWorks.github.io/_posts/2016-05-29-testing-optimizers-with-hypothesis.md b/HypothesisWorks.github.io/_posts/2016-05-29-testing-optimizers-with-hypothesis.md new file mode 100644 index 0000000000..781abc9991 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-05-29-testing-optimizers-with-hypothesis.md @@ -0,0 +1,160 @@ +--- +layout: post +tags: technical python example properties +date: 2016-05-29 21:00 +title: Testing Optimizers +published: true +author: drmaciver +redirect_from: /articles/testing-optimizers +--- + +We've [previously looked into testing performance optimizations]({{site.url}}{% post_url 2016-04-29-testing-performance-optimizations %}) +using Hypothesis, but this +article is about something quite different: It's about testing code +that is designed to optimize a value. That is, you have some function +and you want to find arguments to it that maximize (or minimize) its +value. + +As well as being an interesting subject in its own right, this will also +nicely illustrate the use of Hypothesis's data() functionality, which +allows you to draw more data after the test has started, and will +introduce a useful general property that can improve your testing in +a much wider variety of settings. + + + +We'll use [the Knapsack Packing Problem](https://en.wikipedia.org/wiki/Knapsack_problem) +as our example optimizer. We'll use the greedy approximation algorithm +described in the link, and see if Hypothesis can show us that it's +merely an approximation and not in fact optimal. + +```python +def pack_knapsack(items, capacity): + """Given items as a list [(value, weight)], with value and weight + strictly positive integers, try to find a maximum value subset of + items with total weight <= capacity""" + remaining_capacity = capacity + result = [] + + # Sort in order of decreasing value per unit weight, breaking + # ties by taking the lowest weighted items first. + items = sorted(items, key=lambda x: (x[1] / x[0], x[1])) + for value, weight in items: + if weight <= remaining_capacity: + result.append((value, weight)) + remaining_capacity -= weight + return result +``` + +So how are we going to test this? + +If we had another optimizer we could test by comparing the two results, +but we don't, so we need to figure out properties it should satisfy in +the absence of that. + +The trick we will used to test this is to look for responses to change. + +That is, we will run the function, we will make a change to the data +that should cause the function's output to change in a predictable way, +and then we will run the function again and see if it did. + +But how do we figure out what changes to make? + +The key idea is that we will look at the output of running the optimizer +and use that to guide what changes we make. In particular we will test +the following two properties: + +1. If we remove an item that was previously chosen as part of the + optimal solution, this should not improve the score. +2. If we add an extra copy of an item that was previously chosen as part + of the optimal solution, this should not make the score worse. + +In the first case, any solution that is found when running with one +fewer item would also be a possible solution when running with the full +set, so if the optimizer is working correctly then it should have found +that one if it were an improvement. + +In the second case, the opposite is true: Any solution that was +previously available is still available, so if the optimizer is working +correctly it can't find a worse one than it previously found. + +The two tests look very similar: + +```python +from hypothesis import Verbosity, assume, given, settings, strategies as st + + +def score_items(items): + return sum(value for value, _ in items) + + +PositiveIntegers = st.integers(min_value=1, max_value=10) +Items = st.lists(st.tuples(PositiveIntegers, PositiveIntegers), min_size=1) +Capacities = PositiveIntegers + + +@given(Items, Capacities, st.data()) +def test_cloning_an_item(items, capacity, data): + original_solution = pack_knapsack(items, capacity) + assume(original_solution) + items.append(data.draw(st.sampled_from(original_solution))) + new_solution = pack_knapsack(items, capacity) + assert score_items(new_solution) >= score_items(original_solution) + + +@given(Items, Capacities, st.data()) +def test_removing_an_item(items, capacity, data): + original_solution = pack_knapsack(items, capacity) + assume(original_solution) + item = data.draw(st.sampled_from(original_solution)) + items.remove(item) + new_solution = pack_knapsack(items, capacity) + assert score_items(new_solution) <= score_items(original_solution) +``` + +(The max_value parameter for integers is inessential but results in +nicer example quality). + +The *data* strategy simply provides an object you can use for drawing +more data interactively during the test. This allows us to make our +choices dependent on the output of the function when we run it. The +draws made will be printed as additional information in the case of a +failing example. + +In fact, both of these tests fail: + +``` + +Falsifying example: test_cloning_an_item(items=[(1, 1), (1, 1), (2, 5)], capacity=7, data=data(...)) +Draw 1: (1, 1) + +``` + +In this case what happens is that when Hypothesis clones an item of +weight and value 1, the algorithm stuffs its knapsack with all three +(1, 1) items, at which point it has spare capacity but no remaining +items that are small enough to fit in it. + +``` + +Falsifying example: test_removing_a_chosen_item(items=[(1, 1), (2, 4), (1, 2)], capacity=6, data=data(...)) +Draw 1: (1, 1) + +``` + +In this case what happens is the opposite: Previously the greedy +algorithm was reaching for the (1, 1) item as the most appealing because +it had the highest value to weight ratio, but by including it it only +had space for one of the remaining two. When Hypothesis removed that +option, it could fit the remaining two items into its knapsack and thus +scored a higher point. + +In this case these failures were more or less expected: As described in +the Wikipedia link, for the relatively small knapsacks we're exploring +here the greedy approximation algorithm turns out to in fact be quite +bad, and Hypothesis can easily expose that. + +This technique however can be more widely applied: e.g. You can try +changing permissions and settings on a user and asserting that they +always have more options, or increasing the capacity of a subsystem and +seeing that it is always allocated more tasks. diff --git a/HypothesisWorks.github.io/_posts/2016-05-31-looking-for-guest-posts.md b/HypothesisWorks.github.io/_posts/2016-05-31-looking-for-guest-posts.md new file mode 100644 index 0000000000..18d709f671 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-05-31-looking-for-guest-posts.md @@ -0,0 +1,36 @@ +--- +layout: post +tags: non-technical news +date: 2016-05-29 21:00 +title: Guest Posts Welcome +published: true +author: drmaciver +--- + +I would like to see more posters on the hypothesis.works blog. I'm +particularly interested in experience reports from people who use +Hypothesis in the wild. Could that be you? + + + +Details of how to guest post on here: + +1. This site is [a Jekyll site on GitHub](https://github.com/HypothesisWorks/HypothesisWorks.github.io). + To add a post, create a markdown file in the _posts directory with the + appropriate structure and send a pull request. +2. You will want to add an entry for yourself to [the authors data file](https://github.com/HypothesisWorks/HypothesisWorks.github.io/blob/master/_data/authors.yml) +3. You of course retain all copyright to your work. All you're granting is the right to publish it on this site. + +I'd particularly like to hear from: + +* People who work in QA +* People using the Django support +* People using Hypothesis for heavily numerical work +* People whose first experience of property based testing was via Hypothesis +* People who would like to write about another property based testing system ( + and ideally to compare it to Hypothesis) + +But I'd also like to hear from anyone else who would like to write +something about Hypothesis, or property based testing in general: Whether +it's an experience report, a cool trick you figured out, an introductory +article to Hypothesis, etc. diff --git a/HypothesisWorks.github.io/_posts/2016-06-05-incremental-property-based-testing.md b/HypothesisWorks.github.io/_posts/2016-06-05-incremental-property-based-testing.md new file mode 100644 index 0000000000..22e68d7c63 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-06-05-incremental-property-based-testing.md @@ -0,0 +1,281 @@ +--- +layout: post +tags: intro python technical properties +date: 2016-06-05 16:00 +title: Evolving toward property-based testing with Hypothesis +published: true +author: jml +--- + +Many people are quite comfortable writing ordinary unit tests, but feel a bit +confused when they start with property-based testing. This post shows how two +ordinary programmers started with normal Python unit tests and nudged them +incrementally toward property-based tests, gaining many advantages on the way. + + + +Background +---------- + +I used to work on a command-line tool with an interface much like git's. It +had a repository, and within that repository you could create branches and +switch between them. Let's call the tool `tlr`. + +It was supposed to behave something like this: + +List branches: + + $ tlr branch + foo + * master + +Switch to an existing branch: + + $ tlr checkout foo + * foo + master + +Create a branch and switch to it: + + $ tlr checkout -b new-branch + $ tlr branch + foo + master + * new-branch + +Early on, my colleague and I found a bug: when you created a new branch with +`checkout -b` it wouldn't switch to it. The behavior looked something like +this: + + $ tlr checkout -b new-branch + $ tlr branch + foo + * master + new-branch + +The previously active branch (in this case, `master`) stayed active, rather +than switching to the newly-created branch (`new-branch`). + +Before we fixed the bug, we decided to write a test. I thought this would be a +good chance to start using Hypothesis. + +Writing a simple test +--------------------- + +My colleague was less familiar with Hypothesis than I was, so we started with +a plain old Python unit test: + +```python +def test_checkout_new_branch(self): + """Checking out a new branch makes it the current active branch.""" + tmpdir = FilePath(self.mktemp()) + tmpdir.makedirs() + repo = Repository.initialize(tmpdir.path) + repo.checkout("new-branch", create=True) + self.assertEqual("new-branch", repo.get_active_branch()) +``` + +The first thing to notice here is that the string `"new-branch"` is not +actually relevant to the test. It's just a value we picked to exercise the +buggy code. The test should be able to pass with *any valid branch name*. + +Even before we started to use Hypothesis, we made this more explicit by making +the branch name a parameter to the test: + +```python +def test_checkout_new_branch(self, branch_name="new-branch"): + tmpdir = FilePath(self.mktemp()) + tmpdir.makedirs() + repo = Repository.initialize(tmpdir.path) + repo.checkout(branch_name, create=True) + self.assertEqual(branch_name, repo.get_active_branch()) +``` + +(For brevity, I'll elide the docstring from the rest of the code examples) + +We never manually provided the `branch_name` parameter, but this change made +it more clear that the test ought to pass regardless of the branch name. + +Introducing Hypothesis +---------------------- + +Once we had a parameter, the next thing was to use Hypothesis to provide the +parameter for us. First, we imported Hypothesis: + +```python +from hypothesis import given, strategies as st +``` + +And then made the simplest change to our test to actually use it: + +```python +@given(branch_name=st.just("new-branch")) +def test_checkout_new_branch(self, branch_name): + tmpdir = FilePath(self.mktemp()) + tmpdir.makedirs() + repo = Repository.initialize(tmpdir.path) + repo.checkout(branch_name, create=True) + self.assertEqual(branch_name, repo.get_active_branch()) +``` + +Here, rather than providing the branch name as a default argument value, we +are telling Hypothesis to come up with a branch name for us using the +`just("new-branch")` +[strategy](https://hypothesis.readthedocs.io/en/latest/data.html). This +strategy will always come up with `"new-branch"`, so it's actually no +different from what we had before. + +What we actually wanted to test is that any valid branch name worked. We +didn't yet know how to generate any valid branch name, but using a +time-honored tradition we pretended that we did: + +```python +def valid_branch_names(): + """Hypothesis strategy to generate arbitrary valid branch names.""" + # TODO: Improve this strategy. + return st.just("new-branch") + + +@given(branch_name=valid_branch_names()) +def test_checkout_new_branch(self, branch_name): + tmpdir = FilePath(self.mktemp()) + tmpdir.makedirs() + repo = Repository.initialize(tmpdir.path) + repo.checkout(branch_name, create=True) + self.assertEqual(branch_name, repo.get_active_branch()) +``` + +Even if we had stopped here, this would have been an improvement. Although the +Hypothesis version of the test doesn't have any extra power over the vanilla +version, it is more explicit about what it's testing, and the +`valid_branch_names()` strategy can be re-used by future tests, giving us a +single point for improving the coverage of many tests at once. + +Expanding the strategy +---------------------- + +It's only when we get Hypothesis to start generating our data for us that we +really get to take advantage of its bug finding power. + +The first thing my colleague and I tried was: + +```python +def valid_branch_names(): + return st.text() +``` + +But that failed pretty hard-core. + +Turns out branch names were implemented as symlinks on disk, so valid branch +name has to be a valid file name on whatever filesystem the tests are running +on. This at least rules out empty names, `"."`, `".."`, very long names, names +with slashes in them, and probably others (it's actually +[really complicated](https://en.wikipedia.org/wiki/Filename#Comparison_of_filename_limitations)). + +Hypothesis had made something very clear to us: neither my colleague nor I +actually knew what a valid branch name should be. None of our interfaces +documented it, we had no validators, no clear ideas for rendering & display, +nothing. We had just been assuming that people would pick good, normal, +sensible names. + +It was as if we had suddenly gained the benefit of extensive real-world +end-user testing, just by calling the right function. This was: + + 1. Awesome. We've found bugs that our users won't. + 2. Annoying. We really didn't want to fix this bug right now. + +In the end, we compromised and implemented a relatively conservative strategy +to simulate the good, normal, sensible branch names that we expected: + +```python +from string import ascii_lowercase + +VALID_BRANCH_CHARS = ascii_lowercase + "_-." + + +def valid_branch_names(): + # TODO: Handle unicode / weird branch names by rejecting them early, raising nice errors + # TODO: How do we handle case-insensitive file systems? + return st.text(alphabet=VALID_BRANCH_CHARS, min_size=1, max_size=112) +``` + +Not ideal, but *much* more extensive than just hard-coding `"new-branch"`, and +much clearer communication of intent. + +Adding edge cases +----------------- + +There's one valid branch name that this strategy *could* generate, but +probably won't: `master`. If we left the test just as it is, then one time in +a hojillion the strategy would generate `"master"` and the test would fail. + +Rather than waiting on chance, we encoded this in the `valid_branch_names` +strategy, to make it more likely: + +```python +def valid_branch_names(): + return st.text(alphabet=letters, min_size=1, max_size=112).map( + lambda t: t.lower() + ) | st.just("master") +``` + +When we ran the tests now, they failed with an exception due to the branch +`master` already existing. To fix this, we used `assume`: + +```python +from hypothesis import assume + + +@given(branch_name=valid_branch_names()) +def test_checkout_new_branch(self, branch_name): + assume(branch_name != "master") + tmpdir = FilePath(self.mktemp()) + tmpdir.makedirs() + repo = Repository.initialize(tmpdir.path) + repo.checkout(branch_name, create=True) + self.assertEqual(branch_name, repo.get_active_branch()) +``` + +Why did we add `master` to the valid branch names if we were just going to +exclude it anyway? Because when other tests say "give me a valid branch name", +we want *them* to make the decision about whether `master` is appropriate or +not. Any future test author will be compelled to actually think about whether +handling `master` is a thing that they want to do. That's one of the great +benefits of Hypothesis: it's like having a rigorous design critic in your +team. + +Going forward +------------- + +We stopped there, but we need not have. Just as the test should have held for +any branch, it should also hold for any repository. We were just creating an +empty repository because it was convenient for us. + +If we were to continue, the next step would be to [write a `repositories()` +function to generate repositories]({% post_url 2016-05-11-generating-the-right-data %}) +with more varied contents, commit histories, and existing branches. +The test might then look something like this: + +```python +@given(repo=repositories(), branch_name=valid_branch_names()) +def test_checkout_new_branch(self, repo, branch_name): + """ + Checking out a new branch results in it being the current active + branch. + """ + assume(branch_name not in repo.get_branches()) + repo.checkout(branch_name, create=True) + self.assertEqual(branch_name, repo.get_active_branch()) +``` + +This is about as close to a bona fide "property" as you're likely to get in +code that isn't a straight-up computer science problem: if you create and +switch to a branch that doesn't already exist, the new active branch is the +newly created branch. + +We got there not by sitting down and thinking about the properties of our +software in the abstract, nor by necessarily knowing much about property-based +testing, but rather by incrementally taking advantage of features of Python +and Hypothesis. On the way, we discovered and, umm, contained a whole class of +bugs, and we made sure that all future tests would be heaps more powerful. +Win. diff --git a/HypothesisWorks.github.io/_posts/2016-06-13-testing-configuration-parameters.md b/HypothesisWorks.github.io/_posts/2016-06-13-testing-configuration-parameters.md new file mode 100644 index 0000000000..6fd7168c30 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-06-13-testing-configuration-parameters.md @@ -0,0 +1,151 @@ +--- +layout: post +tags: technical python properties +date: 2016-06-13 00:00 +title: Testing Configuration Parameters +published: true +author: drmaciver +--- + +A lot of applications end up growing a complex configuration system, +with a large number of different knobs and dials you can turn to change +behaviour. Some of these are just for performance tuning, some change +operational concerns, some have other functions. + +Testing these is tricky. As the number of parameters goes up, the number +of possible configuration goes up exponentially. Manual testing of the +different combinations quickly becomes completely unmanageable, not +to mention extremely tedious. + +Fortunately, this is somewhere where property-based testing in general +and Hypothesis in particular can help a lot. + + + +Configuration parameters almost all have one thing in common: For the +vast majority of things, they shouldn't change the behaviour. A +configuration parameter is rarely going to be a complete reskin of your +application. + +This means that they are relatively easy to test with property-based +testing. You take an existing test - either one that is already using +Hypothesis or a normal example based test test - and you vary some +configuration parameters and make sure the test still passes. + +This turns out to be remarkably effective. Here's an example where I +used this technique and found some bugs in the [Argon2]( +https://github.com/P-H-C/phc-winner-argon2) password hashing library, +using [Hynek](https://hynek.me/)'s +[CFFI based bindings](https://github.com/hynek/argon2_cffi). + +The idea of password hashing is straightforward: Given a password, you +can create a hash against which the password can be verified without +ever storing the password (after all, you're not storing passwords in +plain text on your servers, right?). Although straightforward to +describe, there's a lot of difficulty in making a good implementation +of this. Argon2 is a fairly recent one which won [the Password Hashing +Competition](https://password-hashing.net/) so should be fairly good. + +We can verify that hashing works correctly fairly immediately using +Hypothesis: + +```python +from argon2 import PasswordHasher + +from hypothesis import given, strategies as st + + +class TestPasswordHasherWithHypothesis: + @given(password=st.text()) + def test_a_password_verifies(self, password): + ph = PasswordHasher() + hash = ph.hash(password) + assert ph.verify(hash, password) +``` + +This takes an arbitrary text password, hashes it and verifies it against +the generated hash. + +This passes. So far, so good. + +But as you probably expected from its context here, argon2 has quite +a lot of different parameters to it. We can expand the test to vary +them and see what happens: + +```python +from argon2 import PasswordHasher + +from hypothesis import assume, given, strategies as st + + +class TestPasswordHasherWithHypothesis: + @given( + password=st.text(), + time_cost=st.integers(1, 10), + parallelism=st.integers(1, 10), + memory_cost=st.integers(8, 2048), + hash_len=st.integers(12, 1000), + salt_len=st.integers(8, 1000), + ) + def test_a_password_verifies( + self, + password, + time_cost, + parallelism, + memory_cost, + hash_len, + salt_len, + ): + assume(parallelism * 8 <= memory_cost) + ph = PasswordHasher( + time_cost=time_cost, + parallelism=parallelism, + memory_cost=memory_cost, + hash_len=hash_len, + salt_len=salt_len, + ) + hash = ph.hash(password) + assert ph.verify(hash, password) +``` + + +These parameters are mostly intended to vary the difficulty of +calculating the hash. Honestly I'm not entirely sure what all of them +do. Fortunately for the purposes of writing this test, understanding is +optional. + +In terms of how I chose the specific strategies to get there, I just +picked some plausible looking parameters ranges and adjusted them until +I wasn't getting validation errors (I did look for documentation, I +promise). The assume() call comes from reading the argon2 source to try +to find out what the valid range of parallelism was. + +This ended up finding +[two bugs](https://github.com/hynek/argon2_cffi/issues/4), which I duly +reported to Hynek, but they actually turned out to be upstream bugs! + +In both cases, a password would no longer validate against itself: + + +``` +Falsifying example: test_a_password_verifies( + password='', time_cost=1, parallelism=1, memory_cost=8, hash_len=4, + salt_len=8, +) +``` + +``` +Falsifying example: test_a_password_verifies( + password='', time_cost=1, parallelism=1, memory_cost=8, + hash_len=513, salt_len=8 +) +``` + +(I found the second one by manually determining that the first bug +happened whenever salt_len < 12 and manually ruling that case out). + +One interesting thing about both of these bugs is that they're actually +not bugs in the Python library but are both downstream bugs. I hadn't +set out to do that when I wrote these tests, but it nicely validates +that Hypothesis is rather useful for testing C libraries as well as +Python, given how easy they are to bind to with CFFI. diff --git a/HypothesisWorks.github.io/_posts/2016-06-30-tests-as-complete-specifications.md b/HypothesisWorks.github.io/_posts/2016-06-30-tests-as-complete-specifications.md new file mode 100644 index 0000000000..90d4e8307b --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-06-30-tests-as-complete-specifications.md @@ -0,0 +1,167 @@ +--- +layout: post +tags: technical python properties into +date: 2016-06-30 00:00 +title: Testing as a Complete Specification +published: true +author: drmaciver +--- + +Sometimes you're lucky enough to have problems where the result is +completely specified by a few simple properties. + +This doesn't necessarily correspond to them being easy! Many such +problems are actually extremely fiddly to implement. + +It does mean that they're easy to *test* though. Lets see how. + + + +Lets look at the problem of doing a binary search. Specifically we'll +look at a left biased binary search: Given a sorted list and some value, +we want to find the smallest index that we can insert that value at and +still have the result be sorted. + +So we've got the following properties: + +1. binary_search must always return a valid index to insert the value + at. +2. If we insert the value at that index the result must be sorted. +3. If we insert the value at any *smaller* index, the result must *not* + be sorted. + +Using Hypothesis we can write down tests for all these properties: + +```python +from hypothesis import given, strategies as st + + +@given(lists(integers()).map(sorted), integers()) +def test_binary_search_gives_valid_index(ls, v): + i = binary_search(ls, v) + assert 0 <= i <= len(ls) + + +@given(lists(integers()).map(sorted), integers()) +def test_inserting_at_binary_search_remains_sorted(ls, v): + i = binary_search(ls, v) + ls.insert(i, v) + assert sorted(ls) == ls + + +@given(lists(integers()).map(sorted), integers()) +def test_inserting_at_smaller_index_gives_unsorted(ls, v): + for i in range(binary_search(ls, v)): + ls2 = list(ls) + ls2.insert(i, v) + assert sorted(ls2) != ls +``` + +If these tests pass, our implementation must be perfectly correct, +right? They capture the specification of the binary_search function +exactly, so they should be enough. + +And they mostly are, but they suffer from one problem that will +sometimes crop up with property-based testing: They don't hit all bugs +with quite high enough probability. + +This is the difference between testing and mathematical proof: A proof +will guarantee that these properties *always* hold, while a test can +only guarantee that they hold in the areas that it's checked. A test +using Hypothesis will check a much wider area than most hand-written +tests, but it's still limited to a finite set of examples. + +Lets see how this can cause us problems. Consider the following +implementation of binary search: + +```python +def binary_search(list, value): + if not list: + return 0 + if value > list[-1]: + return len(list) + if value <= list[0]: + return 0 + lo = 0 + hi = len(list) - 1 + while lo + 1 < hi: + mid = (lo + hi) // 2 + pivot = list[mid] + if value < pivot: + hi = mid + elif value == pivot: + return mid + else: + lo = mid + return hi +``` + +This implements the common check that if our pivot index ever has +exactly the right value we return early there. Unfortunately in this +case that check is wrong: It violates the property that we should +always find the *smallest* property, so the third test should fail. + +And sure enough, if you run the test enough times it eventually *does* +fail: + +``` +Falsifying example: test_inserting_at_smaller_index_gives_unsorted( + ls=[0, 1, 1, 1, 1], v=1 + ) +``` + +(you may also get (ls=[-1, 0, 0, 0, 0], v=0)) + +However when I run it it usually *doesn't* fail the first time. It +usually takes somewhere between two and five runs before it fails. This +is because in order to trigger this behaviour being wrong you need +quite specific behaviour: value needs to appear in ls at least +twice, and it needs to do so in such a way that one of the indices where +it appears that is *not* the first one gets chosen as mid at some +point in the process. Hypothesis does some things that boost the +chances of this happening, but they don't boost it *that* much. + +Of course, once it starts failing Hypothesis's test database kicks in, +and the test keeps failing until the bug is fixed, but low probability +failures are still annoying because they move the point at which you +discover the problem further away from when you introduced it. This is +especially true when you're using [stateful testing +]({{site.url}}{% post_url 2016-04-19-rule-based-stateful-testing %}), +because the search space is so large that there are a lot of low +probability bugs. + +Fortunately there's an easy fix for this case: You can write additional +tests that are more likely to discover bugs because they are less +sensitively dependent on the example chosen by Hypothesis to exhibit +interesting behaviours. + +Consider the following test: + +```python +@given(lists(integers()).map(sorted), integers()) +def test_inserting_at_result_point_and_searching_again(ls, v): + i = binary_search(ls, v) + ls.insert(i, v) + assert binary_search(ls, v) == i +``` + +The idea here is that by doing a search, inserting the value at that +index, and searching again we cannot have moved the insert point: +Inserting there again would still result in a sorted list, and inserting +any earlier would still have resulted in an unsorted list, so this must +still be the same insert point (this should remind you a bit of +[the approach for testing optimizers we used before]( +{{site.url}}{% post_url 2016-05-29-testing-optimizers-with-hypothesis %}) +). + +This test fails pretty consistently because it doesn't rely nearly so +much on finding duplicates: Instead it deliberately creates them in a +place where they are likely to be problematic. + +So, in conclusion: + +1. When the problem is fully specified, this gives you a natural source + of tests that you can easily write using Hypothesis. +2. However this is where your tests should *start* rather than finish, + and you still need to think about other interesting ways to test your + software. diff --git a/HypothesisWorks.github.io/_posts/2016-07-04-calculating-the-mean.md b/HypothesisWorks.github.io/_posts/2016-07-04-calculating-the-mean.md new file mode 100644 index 0000000000..a97e34b02b --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-07-04-calculating-the-mean.md @@ -0,0 +1,192 @@ +--- +layout: post +tags: technical python properties intro +date: 2016-07-04 00:00 +title: Calculating the mean of a list of numbers +published: true +author: drmaciver +--- + +Consider the following problem: + +You have a list of floating point numbers. No nasty tricks - these +aren't NaN or Infinity, just normal "simple" floating point numbers. + +Now: Calculate the mean (average). Can you do it? + +It turns out this is a hard problem. It's hard to get it even *close* to +right. Lets see why. + + + +Consider the following test case using Hypothesis: + +```python +from hypothesis import given +from hypothesis.strategies import floats, lists + + +@given(lists(floats(allow_nan=False, allow_infinity=False), min_size=1)) +def test_mean_is_within_reasonable_bounds(ls): + assert min(ls) <= mean(ls) <= max(ls) +``` + +This isn't testing much about correctness, only that the value of the +mean is within reasonable bounds for the list: There are a lot of +functions that would satisfy this without being the mean. min and max +both satisfy this, as does the median, etc. + +However, almost nobody's implementation of the mean satisfies this. + +To see why, lets write our own mean: + +```python +def mean(ls): + return sum(ls) / len(ls) +``` + +This seems reasonable enough - it's just the definition of the mean - +but it's wrong: + +``` +assert inf <= 8.98846567431158e+307 + + where inf = mean([8.988465674311579e+307, 8.98846567431158e+307]) + + and 8.98846567431158e+307 = max([8.988465674311579e+307, 8.98846567431158e+307]) + +Falsifying example: test_mean_is_within_reasonable_bounds( + ls=[8.988465674311579e+307, 8.98846567431158e+307] +) +``` + +The problem is that finite floating point numbers may be large enough +that their sum overflows to infinity. When you then divide infinity by a +finite number you still get infinity, which is out of the range. + +So to prevent that overflow, lets try to bound the size of our numbers +by the length *first*: + +```python +def mean(ls): + return sum(l / len(ls) for l in ls) +``` + +``` +assert min(ls) <= mean(ls) <= max(ls) +assert 1.390671161567e-309 <= 1.390671161566996e-309 +where 1.390671161567e-309 = min([1.390671161567e-309, 1.390671161567e-309, 1.390671161567e-309]) +and 1.390671161566996e-309 = mean([1.390671161567e-309, 1.390671161567e-309, 1.390671161567e-309]) + +Falsifying example: test_mean_is_within_reasonable_bounds( + ls=[1.390671161567e-309, 1.390671161567e-309, 1.390671161567e-309] +) +``` + +In this case the problem you run into is not overflow, but the lack of +precision of floating point numbers: Floating point numbers are only +exact up to powers of two times an integer, so dividing by three will +cause rounding errors. In this case we have the problem that (x / 3) * 3 +may not be equal to x in general. + +So now we've got a sense of why this might be hard. Lets see how +existing implementations do at satisfying this test. + +First let's try numpy: + +```python +import numpy as np + + +def mean(ls): + return np.array(ls).mean() +``` + +This runs into the problem we had in our first implementation: + +``` +assert min(ls) <= mean(ls) <= max(ls) +assert inf <= 8.98846567431158e+307 + +where inf = mean([8.988465674311579e+307, 8.98846567431158e+307]) +and 8.98846567431158e+307 = max([8.988465674311579e+307, 8.98846567431158e+307]) + +Falsifying example: test_mean_is_within_reasonable_bounds( + ls=[8.988465674311579e+307, 8.98846567431158e+307] +) +``` + +There's also the new statistics module from Python 3.4. Unfortunately, +this is broken too +([this is fixed in 3.5.2](https://bugs.python.org/issue25177)): + +``` +OverflowError: integer division result too large for a float + +Falsifying example: test_mean_is_within_reasonable_bounds( + ls=[8.988465674311579e+307, 8.98846567431158e+307] +) +``` + +In the case where we previously overflowed to infinity this instead +raises an error. The reason for this is that internally the statistics +module is converting everything to the Fraction type, which is an +arbitrary precision rational type. Because of the details of where and +when they were converting back to floats, this produced a rational that +couldn't be readily converted back to a float. + +It's relatively easy to write an implementation which passes this test +by simply cheating and not actually calculating the mean: + +```python +def clamp(lo, v, hi): + return min(hi, max(lo, v)) + + +def mean(ls): + return clamp(min(ls), sum(ls) / len(ls), max(ls)) +``` + +i.e. just restricting the value to lie in the desired range. + +However getting an actually correct implementation of the mean (which +*would* pass this test) is quite hard: + +To see just how hard, here's a [30 page +paper on calculating the mean of two numbers](https://hal.archives-ouvertes.fr/file/index/docid/576641/filename/computing-midpoint.pdf). + +I wouldn't feel obliged to read that paper if I were you. I *have* read +it and I don't remember many of the details. + +This test is a nice instance of a general one: Once you've got the +[this code doesn't crash]({{site.url}}{% post_url 2016-04-15-getting-started-with-hypothesis %}), +tests working, you can start to layer on additional constraints on the +result value. As this example shows, even when the constraints you +impose are *very* lax it can often catch interesting bugs. + +It also demonstrates a problem: Floating point mathematics is *very* +hard, and this makes it somewhat unsuitable for testing with Hypothesis. + +This isn't because Hypothesis is *bad* at testing floating point code, +it's because it's good at showing you how hard programming actually is, +and floating point code is much harder than people like to admit. + +As a result, you probably don't care about the bugs it will find: +Generally speaking most peoples' attitude to floating point errors is +"Eh, those are weird numbers, we don't really care about that. It's +probably good enough". Very few people are actually prepared to do the +required work of a numerical sensitivity analysis that is needed if you +want your floating point code to be correct. + +I used to use this example a lot for demonstrating Hypothesis to people, +but because of these problems I tend not to any more: Telling people +about bugs they're not going to want to fix will get you neither bug +fixes nor friends. + +But it's worth knowing that this is a problem: Programming *is* really +hard, and ignoring the problems won't make it less hard. You can ignore +the correctness issues until they actually bite you, but it's best not +to be surprised when they do. + +And it's also worth remembering the general technique here, because this +isn't just useful for floating point numbers: Most code can benefit from +this, and most of the time the bugs it tells you won't be nearly this +unpleasant. diff --git a/HypothesisWorks.github.io/_posts/2016-07-09-hypothesis-3.4.1-release.md b/HypothesisWorks.github.io/_posts/2016-07-09-hypothesis-3.4.1-release.md new file mode 100644 index 0000000000..bdc24a9b72 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-07-09-hypothesis-3.4.1-release.md @@ -0,0 +1,27 @@ +--- +layout: post +tags: news python non-technical +date: 2016-07-09 00:00 +title: Hypothesis for Python 3.4.1 Release +published: true +author: drmaciver +--- + +This is a bug fix release for a single bug: + +* On Windows when running two Hypothesis processes in parallel (e.g. + using pytest-xdist) they could race with each other and one would + raise an exception due to the non-atomic nature of file renaming on + Windows and the fact that you can’t rename over an existing file. + This is now fixed. + +## Notes + +My tendency of doing immediate patch releases for bugs is unusual but +generally seems to be appreciated. In this case this was a bug that was +blocking +[a py.test merge](https://github.com/pytest-dev/pytest/pull/1705). + +I suspect this is not the last bug around atomic file creation on +Windows. Cross platform atomic file creation seems to be a harder +problem than I would have expected. diff --git a/HypothesisWorks.github.io/_posts/2016-07-13-hypothesis-3.4.2-release.md b/HypothesisWorks.github.io/_posts/2016-07-13-hypothesis-3.4.2-release.md new file mode 100644 index 0000000000..8dacd141cf --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-07-13-hypothesis-3.4.2-release.md @@ -0,0 +1,43 @@ +--- +layout: post +tags: news python non-technical +date: 2016-07-13 00:00 +title: 3.4.2 Release of Hypothesis for Python +published: true +author: drmaciver +--- + +This is a bug fix release, fixing a number of problems with the settings +system: + +* Test functions defined using @given can now be called from other + threads (Issue #337) +* Attempting to delete a settings property would previously have + silently done the wrong thing. Now it raises an AttributeError. +* Creating a settings object with a custom database_file parameter + was silently getting ignored and the default was being used instead. + Now it’s not. + +## Notes + +For historic reasons, _settings.py had been excluded from the +requirement to have 100% branch coverage. Issue #337 would have been +caught by a coverage requirement: the code in question simply couldn't +have worked, but it was not covered by any tests, so it slipped through. + +As part of the general principle that bugs shouldn't just be fixed +without addressing the reason why the bug slipped through in the first +place, I decided to impose the coverage requirements on _settings.py +as well, which is how the other two bugs were found. Both of these had +code that was never run during tests - in the case of the deletion bug +there was a \_\_delete\_\_ descriptor method that was never being run, +and in the case of the database\_file one there was a check later that +could never fire because the internal \_database field was always being +set in \_\_init\_\_. + +I feel like this experiment thoroughly validated that 100% coverage is a +useful thing to aim for. Unfortunately it also pointed out that the +settings system is *much* more complicated than it needs to be. I'm +unsure what to do about that - some of its functionality is a bit too +baked into the public API to lightly change, and I'm don't think it's +worth breaking that just to simplify the code. diff --git a/HypothesisWorks.github.io/_posts/2016-07-23-what-is-hypothesis.md b/HypothesisWorks.github.io/_posts/2016-07-23-what-is-hypothesis.md new file mode 100644 index 0000000000..6694f181b7 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-07-23-what-is-hypothesis.md @@ -0,0 +1,161 @@ +--- +layout: post +tags: python intro +date: 2016-07-24 00:00 +title: What is Hypothesis? +published: true +author: drmaciver +--- + +Hypothesis is a library designed to help you write what are called +*property-based tests*. + +The key idea of property based testing is that rather than writing a test +that tests just a single scenario, you write tests that describe a range +of scenarios and then let the computer explore the possibilities for you +rather than having to hand-write every one yourself. + +In order to contrast this with the sort of tests you might be used to, when +talking about property-based testing we tend to describe the normal sort of +testing as *example-based testing*. + +Property-based testing can be significantly more powerful than example based +testing, because it automates the most time consuming part of writing tests +- coming up with the specific examples - and will usually perform it better +than a human would. This allows you to focus on the parts that humans are +better at - understanding the system, its range of acceptable behaviours, +and how they might break. + +You don't *need* a library to do property-based testing. If you've ever +written a test which generates some random data and uses it for testing, +that's a property-based test. But having a library can help you a lot, +making your tests easier to write, more robust, and better at finding +bugs. In the rest of this article we'll see how. + + + +### How to use it + +The key object of Hypothesis is a *strategy*. A strategy is a recipe for +describing the sort of data you want to generate. The existence of a rich +and comprehensive strategy library is the first big advantage of Hypothesis +over a more manual process: Rather than having to hand-write generators +for the data you want, you can just compose the ones that Hypothesis +provides you with to get the data you want. e.g. if you want a lists of +floats, you just use the strategy lists(floats()). As well as being +easier to write, the resulting data will usually have a distribution +that is much better at finding edge cases than all but the most heavily +tuned manual implementations. + +As well as the basic out of the box strategy implementations, Hypothesis +has a number of tools for composing strategies with user defined functions +and constraints, making it fairly easy to generate the data you want. + +Note: For the remainder of this article I'll focus on the Hypothesis for +Python implementation. The Java implementation is similar, but has a number +of small differences that I'll discuss in a later article. + +Once you know how to generate your data, the main entry point to Hypothesis +is the @given decorator. This takes a function that accepts some arguments +and turns it into a normal test function. + +An important consequence of that is that Hypothesis is not itself a test +runner. It works inside your normal testing framework - it will work fine +with nose, py.test, unittest, etc. because all it does is expose a function +of the right name that the test runner can then pick up. + +Using it with a py.test or nose style test looks like this: + +```python +from mercurial.encoding import fromutf8b, toutf8b + +from hypothesis import given +from hypothesis.strategies import binary + + +@given(binary()) +def test_decode_inverts_encode(s): + assert fromutf8b(toutf8b(s)) == s +``` + +(This is an example from testing Mercurial which found two bugs: +[4927](https://bz.mercurial-scm.org/show_bug.cgi?id=4927) and +[5031](https://bz.mercurial-scm.org/show_bug.cgi?id=5031)). + +In this test we are asserting that for any binary string, converting +it to its utf8b representation and back again should result in the +same string we started with. The @given decorator then handles +executing this test over a range of different binary strings without +us having to explicitly specify any of the examples ourself. + +When this is first run, you will see an error that looks something +like this: + +``` +Falsifying example: test_decode_inverts_encode(s='\xc2\xc2\x80') + +Traceback (most recent call last): + File "/home/david/.pyenv/versions/2.7/lib/python2.7/site-packages/hypothesis/core.py", line 443, in evaluate_test_data + search_strategy, test, + File "/home/david/.pyenv/versions/2.7/lib/python2.7/site-packages/hypothesis/executors.py", line 58, in default_new_style_executor + return function(data) + File "/home/david/.pyenv/versions/2.7/lib/python2.7/site-packages/hypothesis/core.py", line 110, in run + return test(*args, **kwargs) + File "/home/david/hg/test_enc.py", line 8, in test_decode_inverts_encode + assert fromutf8b(toutf8b(s)) == s + File "/home/david/hg/mercurial/encoding.py", line 485, in fromutf8b + u = s.decode("utf-8") + File "/home/david/.pyenv/versions/2.7/lib/python2.7/encodings/utf_8.py", line 16, in decode + return codecs.utf_8_decode(input, errors, True) +UnicodeDecodeError: 'utf8' codec can't decode byte 0xc2 in position 1: invalid continuation byte +``` + +Note that the falsifying example is quite small. Hypothesis has a +"simplification" process which runs behind the scenes and generally +tries to give the impression that the test simply failed with one +example that happened to be a really nice one. + +Another important thing to note is that because of the random nature +of Hypothesis and because this bug is relatively hard to find, this +test may run successfully a couple of times before finding it. + +However, once that happens, when we rerun the test it will keep failing +with the same example. This is because Hypothesis has a local test +database that it saves failing examples in. When you rerun the test, +it will first try the previous failure. + +This is pretty important: It means that although Hypothesis is at its +heart random testing, it is *repeatable* random testing: A bug will +never go away by chance, because a test will only start passing if +the example that previously failed no longer failed. + +(This isn't entirely true because a bug could be caused by random +factors such as timing or hash randomization. However in these cases +it's true for example-based testing as well. If anything Hypothesis +is *more* robust here because it will tend to find these cases with +higher probability). + +Ultimately that's "all" Hypothesis does: It provides repeatability, +reporting and simplification for randomized tests, and it provides +a large library of generators to make it easier to write them. + +Because of these features, the workflow is a huge improvement on +writing your own property-based tests by hand, and thanks to the +library of generators it's often even easier than writing your +own example based tests by hand. + +### What now? + +If you want to read more on the subject, there are a couple places +you could go: + +* If you want to know more of the details of the process I described + when a test executes, you can check out the + [Anatomy of a test]({{site.url}}{% post_url 2016-04-16-anatomy-of-a-test %}) + article which will walk you through the steps in more detail. +* If you'd like more examples of how to use it, check out the rest of the + [intro section]({{site.url}}/articles/intro/). + +But really the best way to learn more is to try to use it! +As you've hopefully seen in this article, it's quite approachable to +get started with. Try writing some tests and see what happens. diff --git a/HypothesisWorks.github.io/_posts/2016-08-09-hypothesis-pytest-fixtures.md b/HypothesisWorks.github.io/_posts/2016-08-09-hypothesis-pytest-fixtures.md new file mode 100644 index 0000000000..fbd0263d3d --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-08-09-hypothesis-pytest-fixtures.md @@ -0,0 +1,151 @@ +--- +layout: post +tags: technical faq python +date: 2016-08-09 10:00 +title: How do I use pytest fixtures with Hypothesis? +published: true +author: drmaciver +--- + +[pytest](http://doc.pytest.org/en/latest/) is a great test runner, and is the one +Hypothesis itself uses for testing (though Hypothesis works fine with other test +runners too). + +It has a fairly elaborate [fixture system](http://doc.pytest.org/en/latest/fixture.html), +and people are often unsure how that interacts with Hypothesis. In this article we'll +go over the details of how to use the two together. + + + +Mostly, Hypothesis and py.test fixtures don't interact: Each just ignores the other's +presence. + +When using a @given decorator, any arguments that are not provided in the @given +will be left visible in the final function: + +```python +from inspect import signature + +from hypothesis import given, strategies as st + + +@given(a=st.none(), c=st.none()) +def test_stuff(a, b, c, d): + pass + + +print(signature(test_stuff)) +``` + +This then outputs the following: + +``` + +``` + +We've hidden the arguments 'a' and 'c', but the unspecified arguments 'b' and 'd' +are still left to be passed in. In particular, they can be provided as py.test +fixtures: + +```python +from pytest import fixture + +from hypothesis import given, strategies as st + + +@fixture +def stuff(): + return "kittens" + + +@given(a=st.none()) +def test_stuff(a, stuff): + assert a is None + assert stuff == "kittens" +``` + +This also works if we want to use @given with positional arguments: + +```python +from pytest import fixture + +from hypothesis import given, strategies as st + + +@fixture +def stuff(): + return "kittens" + + +@given(t.none()) +def test_stuff(stuff, a): + assert a is None + assert stuff == "kittens" +``` + +The positional argument fills in from the right, replacing the 'a' +argument and leaving us with 'stuff' to be provided by the fixture. + +Personally I don't usually do this because I find it gets a bit +confusing - if I'm going to use fixtures then I always use the named +variant of given. There's no reason you *can't* do it this way if +you prefer though. + +@given also works fine in combination with parametrized tests: + +```python +import pytest + +from hypothesis import given, strategies as st + + +@pytest.mark.parametrize("stuff", [1, 2, 3]) +@given(a=st.none()) +def test_stuff(a, stuff): + assert a is None + assert 1 <= stuff <= 3 +``` + +This will run 3 tests, one for each value for 'stuff'. + +There is one unfortunate feature of how this interaction works though: In pytest +you can declare fixtures which do set up and tear down per function. These will +"work" with Hypothesis, but they will run once for the entire test function +rather than once for each time given calls your test function. So the following +will fail: + +```python +from pytest import fixture + +from hypothesis import given, strategies as st + +counter = 0 + + +@fixture(scope="function") +def stuff(): + global counter + counter = 0 + + +@given(a=st.none()) +def test_stuff(a, stuff): + global counter + counter += 1 + assert counter == 1 +``` + +The counter will not get reset at the beginning of each call to the test function, +so it will be incremented each time and the test will start failing after the +first call. + +There currently aren't any great ways around this unfortunately. The best you can +really do is do manual setup and teardown yourself in your tests using +Hypothesis (e.g. by implementing a version of your fixture as a context manager). + +Long-term, I'd like to resolve this by providing a mechanism for allowing fixtures +to be run for each example (it's probably not correct to have *every* function scoped +fixture run for each example), but for now it's stalled because it [requires changes +on the py.test side as well as the Hypothesis side](https://github.com/pytest-dev/pytest/issues/916) +and we haven't quite managed to find the time and place to collaborate on figuring +out how to fix this yet. diff --git a/HypothesisWorks.github.io/_posts/2016-08-19-recursive-data.md b/HypothesisWorks.github.io/_posts/2016-08-19-recursive-data.md new file mode 100644 index 0000000000..452a095e82 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-08-19-recursive-data.md @@ -0,0 +1,164 @@ +--- +layout: post +tags: technical python intro +date: 2016-08-19 10:00 +title: Generating recursive data +published: true +author: drmaciver +--- + +Sometimes you want to generate data which is *recursive*. +That is, in order to draw some data you may need to draw +some more data from the same strategy. For example we might +want to generate a tree structure, or arbitrary JSON. + +Hypothesis has the *recursive* function in the hypothesis.strategies +module to make this easier to do. This is an article about how to +use it. + + + +Lets start with a simple example of drawing tree shaped data: +In our example a tree is either a single boolean value (a leaf +node), or a tuple of two child trees. So a tree might be e.g. True, +or (False, True), or ((True, True), False), etc. + +First off, it might not be obvious that you *need* the recursive +strategy. In principle you could just do this with composite: + + +```python +import hypothesis.strategies as st + + +@st.composite +def composite_tree(draw): + return draw( + st.one_of( + st.booleans(), + st.tuples(composite_tree(), composite_tree()), + ) + ) +``` + +If you try drawing examples from this you'll probably see one of +three scenarios: + +1. You'll get a single boolean value +2. You'll get a very large tree +3. You'll get a RecursionError from a stack overflow + +It's unlikely that you'll see any non-trivial small examples. + +The reason for this is that this sort of recursion tends to +explode in size: If this were implemneted as a naive random +generation process then the expected size of the tree would +be infinite. Hypothesis has some built in limiters to stop +it ever trying to actually generate infinitely large amounts +of data, but it will still tend to draw trees that are very +large if they're not trivial, and it can't do anything about +the recursion problem. + +So instead of using this sort of unstructured recursion, +Hypothesis exposes a way of doing recursion in a slightly more +structured way that lets it control the size of the +generated data much more effectively. This is the recursive +strategy. + +In order to use the recursive strategy you need two parts: + +1. A base strategy for generating "simple" instances of the + data that you want. +2. A function that takes a child strategy that generates data + of the type you want and returns a new strategy generating + "larger" instances. + +So for example for our trees of booleans and tuples we could +use booleans() for the first and something for returning tuples +of children for the second: + +```python +recursive_tree = st.recursive( + st.booleans(), lambda children: st.tuples(children, children) +) +``` + +The way to think about the recursive strategy is that you're +repeatedly building up a series of strategies as follows: + +```python +s1 = base +s2 = one_of(s1, extend(s1)) +s3 = one_of(s2, extend(s2)) +... +``` + +So at each level you augment the things from the previous +level with your extend function. Drawing from the resulting +recursive strategy then picks one of this infinite sequence +of strategies and draws from it (this isn't quite what happens +in practice, but it's pretty close). + +The resulting strategy does a much better job of drawing small +and medium sized trees than our original composite based one +does, and should never raise a RecursionError: + +``` +>>> recursive_tree.example() +((False, True), ((True, True), False)) + +>>> recursive_tree.example() +((((False, False), True), False), False) + +>>> recursive_tree.example() +(False, True) + +>>> recursive_tree.example() +True + +``` + +You can also control the size of the trees it draws with the +third parameter to recursive: + +``` +>>> st.recursive(st.booleans(), lambda children: st.tuples(children, children), max_leaves=2).example() +True + +>>> st.recursive(st.booleans(), lambda children: st.tuples(children, children), max_leaves=2).example() +(True, False) +``` + +The max_leaves parameter controls the number of values drawn from +the 'base' strategy. It defaults to 50, which will tend to give you +moderately sized values. This helps keep example sizes under control, +as otherwise it can be easy to create extend functions which cause the +size to grow very rapidly. + +In this particular example, Hypothesis will typically not hit the default, +but consider something like the following: + +``` +>>> st.recursive(st.booleans(), lambda children: st.lists(children, min_size=3)).example() +[[False, + True, + False, + False, + False, + True, + True, + True, + False, + False, + False, + True, + True, + False], + False, + [False, True, False, True, False], + [True, False, True, False, False, False]] +``` + +In this case the size of the example will tend to push up against the max_leaves value +because extend() grows the strategy in size quite rapidly, so if you want larger +examples you will need to turn up max_leaves. diff --git a/HypothesisWorks.github.io/_posts/2016-08-31-how-many-tests.md b/HypothesisWorks.github.io/_posts/2016-08-31-how-many-tests.md new file mode 100755 index 0000000000..44bc38378a --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-08-31-how-many-tests.md @@ -0,0 +1,145 @@ +--- +layout: post +tags: technical details faq python +date: 2016-08-31 00:00 +title: How many times will Hypothesis run my test? +published: true +author: drmaciver +--- + +This is one of the most common first questions about Hypothesis. + +People generally assume that the number of tests run will depend on +the specific strategies used, but that's generally not the case. +Instead Hypothesis has a fairly fixed set of heuristics to determine +how many times to run, which are mostly independent of the data +being generated. + +But how many runs is *that*? + +The short answer is 200. Assuming you have a default configuration +and everything is running smoothly, Hypothesis will run your test +200 times. + +The longer answer is "It's complicated". It will depend on the exact +behaviour of your tests and the value of some settings. In this article +I'll try to clear up some of the specifics of which +[settings](http://hypothesis.readthedocs.io/en/latest/settings.html) +affect the answer and how. + + + +Advance warning: This is a set of heuristics built up over time. It's +probably not the best choice of heuristics, but it mostly seems +to work well in practice. It will hopefully be replaced with a +simpler set of rules at some point. + +The first setting that affects how many times the test function will +be called is the timeout setting. This specifies a maximum amount of +time for Hypothesis to run your tests for. Once that has exceeded it +will stop and not run any more (note: This is a soft limit, so it +won't interrupt a test midway through). + +The result of this is that slow tests may get run fewer times. By +default the timeout is one minute, which is high enough that most +tests shouldn't hit it, if your tests take somewhere in the region +of 300-400ms on average they will start to hit the timeout. + +The timeout is respected regardless of whether the test passes or +fails, but other than that the behaviour for a passing test is +very different from a failing one. + +### Passing tests + +For the passing case there are two other settings that affect the +answer: max\_examples and max\_iterations. + +In the normal case, max\_examples is what you can think of as the +number of test runs. The difference comes when you start using +assume or filter (and a few other cases). + +Hypothesis distinguishes between a *valid* test run and an invalid one +- if assume has been called with a falsey value or at some point in +the generation process it got stuck (e.g. because filter couldn't +find any satisfying examples) it aborts the example and starts +again from the beginning. max\_examples counts only valid examples +while max\_iterations counts all examples, valid or otherwise. Some +duplicate tests will also be considered invalid (though Hypothesis +can't distinguish all duplicates. e.g. if you did +integers().map(lambda x: 1) it would think you had many distinct values +when you only had one). The default value for max_iterations +is currently 1000. + +To see why it's important to have the max\_iterations limit, +consider something like: + +```python +from hypothesis import assume, given, strategies as st + + +@given(st.integers()) +def test_stuff(i): + assume(False) +``` + +Then without a limit on invalid examples this would run forever. + +Conversely however, treating valid examples specially is useful because +otherwise even casual use of assume would reduce the number of tests +you run, reducing the quality of your testing. + +Another thing to note here is that the test with assume(False) will +actually fail, raising: + +``` +hypothesis.errors.Unsatisfiable: Unable to satisfy assumptions of hypothesis test_stuff. Only 0 examples considered satisfied assumptions +``` + +This is because of the min\_satisfying\_examples setting: If +Hypothesis couldn't find enough valid test cases then it will +fail the test rather than silently doing the wrong thing. + +min\_satisfying\_examples will never increase the number of tests +run, only fail the test if that number of valid examples haven't +been run. If you're hitting this failure you can either turn it +down or turn the timeout or max\_iterations up. Better, you can +figure out *why* you're hitting that and fix it, because it's +probably a sign you're not getting much benefit out of Hypothesis. + + +### Failing tests + +If in the course of normal execution Hypothesis finds an example +which causes your test to fail, it switches into shrinking mode. + +Shrinking mode tries to take your example and produce a smaller +one. It ignores max\_examples and max\_iterations but respects +timeout. It also respects one additional setting: max\_shrinks. + +max\_shrinks is the maximum number of *failing* tests that Hypothesis +will see before it stops. It may try any number of valid or invalid +examples in the course of shrinking. This is because failing +examples tend to be a lot rarer than passing or invalid examples, +so it makes more sense to limit based on that if we want to get +good examples out at the end. + +Once Hypothesis has finished shrinking it will run your test once +more to replay it for display: In the final run only it will print +out the example and any notes, and will let the exception bubble +up to the test runner to be handled as normal. + + +### In Conclusion + +"It's complicated". + +These heuristics are probably not the best. They've evolved +over time, and are definitely not the ones that you or I would +obviously come up with if you sat down and designed the system +from scratch. + +Fortunately, you're not expected to know these heuristics by heart +and mostly shouldn't have to. I'm working on a new feature that +will help show how many examples Hypothesis has tried and help +debug why it's stopped at that point. Hopefully it will be coming +in a release in the near future. diff --git a/HypothesisWorks.github.io/_posts/2016-09-04-hypothesis-vs-eris.md b/HypothesisWorks.github.io/_posts/2016-09-04-hypothesis-vs-eris.md new file mode 100644 index 0000000000..6c9d547908 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-09-04-hypothesis-vs-eris.md @@ -0,0 +1,224 @@ +--- +layout: post +tags: technical python intro +date: 2016-09-04 15:00 +title: Hypothesis vs. Eris +published: true +author: giorgiosironi +--- + +[Eris](https://github.com/giorgiosironi/eris/) is a library for property-based testing of PHP code, inspired by the mature frameworks that other languages provide like [QuickCheck](https://hackage.haskell.org/package/QuickCheck), Clojure's [test.check](https://github.com/clojure/test.check) and of course Hypothesis. + +Here is a side-by-side comparison of some basic and advanced features that have been implemented in both Hypothesis and Eris, which may help developers coming from either Python or PHP and looking at the other side. + + + +## Hello, world + +The first example can be considered an `Hello, world` of randomized testing: given two random numbers, their sum should be constant no matter the order in which they are summed. The test case has two input parameters, so that it can be run dozens or hundreds of times, each run with different and increasingly complex input values. + +Hypothesis provides an idiomatic Python solution, the `@given` decorator, that can be used to compose the strategies objects instantiated from `hypothesis.strategies`. The test case is very similar to an example-based one, except for the addition of arguments: + +```python +import unittest + +from hypothesis import given, strategies as st + + +class TestEris(unittest.TestCase): + "Comparing the syntax and implementation of features with Eris" + + @given(st.integers(), st.integers()) + def test_sum_is_commutative(self, first, second): + x = first + second + y = second + first + self.assertEqual( + x, y, "Sum between %d and %d should be commutative" % (first, second) + ) +``` + +In this example, I am using the unittest syntax for maximum test portability, but this does not affect Hypothesis, which works across testing frameworks. + +Eris provides functionality with a trait instead, that can be composed into the test cases that need access to randomized input. The test method is not augmented with additional parameters, but its code is moved inside an anonymous function for the `then()` primitive. Input distributions are defined in a mathematically-named `forAll()` primitive: + +```php +forAll( + Generator\int(), + Generator\int() + ) + ->then(function ($first, $second) { + $x = $first + $second; + $y = $second + $first; + $this->assertEquals( + $x, + $y, + "Sum between {$first} and {$second} should be commutative" + ); + }); + } +``` + +Both these tests will be run hundreds of times, each computing two sums and comparing them for equality with the assertion library of the underlying testing framework (unittest and PHPUnit). + +## Composing strategies + +A very simple composition problem consists of generating a collection data structured whose elements are drawn from a known distribution, for example a list of integers. Hypothesis provides *strategies* that can compose other strategies, in this case to build a list of random integers of arbitrary length: + +```python +@given(st.lists(st.integers())) +def test_sorting_a_list_twice_is_the_same_as_sorting_it_once(self, xs): + xs.sort() + ys = list(xs) + xs.sort() + self.assertEqual(xs, ys) +``` + +Eris calls *generators* the objects representing statistical distributions, but uses the same compositional pattern for higher-order values like lists and all collections: + +```php + public function testArraySortingIsIdempotent() + { + $this + ->forAll( + Generator\seq(Generator\nat()) + ) + ->then(function ($array) { + sort($array); + $expected = $array; + sort($array); + $this->assertEquals($expected, $array); + }); + } +``` + +In both these test cases, we generate a random list and check that the sorting operation is idempotent: operating it twice on the input data gives the same result as operating it once. + +## Filtering generated values + +Not all problems are abstract enough to accept all values in input, so it may be necessary to exclude part of the generated values when they do not fit our needs. + +Hypothesis provides a `filter()` method to apply a lambda to values, expressing a condition for them to be included in the test: + +```python +@given(st.integers().filter(lambda x: x > 42)) +def test_filtering(self, x): + self.assertGreater(x, 42) +``` + +Eris allows to filter values with a predicate in the same way, but prefers to allocate the filter to the generic `ForAll` object rather than decorate it on each Generator: + +```php + public function testWhenWithAnAnonymousFunctionWithGherkinSyntax() + { + $this + ->forAll( + Generator\choose(0, 1000) + ) + ->when(function ($n) { + return $n > 42; + }) + ->then(function ($number) { + $this->assertTrue( + $number > 42, + "\$number was filtered to be more than 42, but it's $number" + ); + }); + } +``` + +Filtering and the constructs that follow are integrated with shrinking in both Hypothesis and Eris: once a test fails and shrinking starts, constraints and transformations will continue to be applied to the generated values so that the simpler inputs are still going to belong to the same distribution; in this scenario, shrinking would only be able to propose numbers greater than 42. + +## Transforming generated values + +Another common need consists of transforming the generated value to a different space, for example the set of all even numbers rather than the (larger) set of integers. Hypothesis allows to do this by passing a lambda to the `map()` method of a strategy: + +```python +@given(st.integers().map(lambda x: x * 2)) +def test_mapping(self, x): + self.assertEqual(x % 2, 0) +``` + +Eris instead provides a `Map` higher-order generator, which applies the lambda during generation: + +```php + public function testApplyingAFunctionToGeneratedValues() + { + $this + ->forAll( + Generator\map( + function ($n) { return $n * 2; }, + Generator\nat() + ) + ) + ->then(function ($number) { + $this->assertTrue( + $number % 2 == 0, + "$number is not even" + ); + }); + } +``` + +In both cases, the advantage of using the `map` support from the library (rather than writing our own multiplying code in the tests) is that the resulting object can be further composed to build larger data structures like a list or set of even numbers. + +## Generators with random parameters + +It's possible to build even stricter values, that have internal constraints that must be satisfied but can't easily be generated by applying a pure function to a previously generated value. + +Hypothesis provides the `flatmap()` method to pass the output of an inner strategy to a lambda that creates an outer strategy to use in the test. Here a list of 4 integers is passed to the lambda, to generate a tuple consisting of the list and a random element chosen from it: + +```python +@given( + st.lists(st.integers(), min_size=4, max_size=4).flatmap( + lambda xs: st.tuples(st.just(xs), st.sampled_from(xs)) + ) +) +def test_list_and_element_from_it(self, pair): + (generated_list, element) = pair + self.assertIn(element, generated_list) +``` + +Eris does the same with a slightly different naming, calling this primitive `bind`: + +```php + public function testCreatingABrandNewGeneratorFromAGeneratedValue() + { + $this + ->forAll( + Generator\bind( + Generator\vector(4, Generator\nat()), + function ($vector) { + return Generator\tuple( + Generator\elements($vector), + Generator\constant($vector) + ); + } + ) + ) + ->then(function ($tuple) { + list($element, $vector) = $tuple; + $this->assertContains($element, $vector); + }); + } +``` + +## What the future brings + +Hypothesis is a much more mature project than Eris, especially when it comes to keeping state between test runs or acting as a generic random data provider rather than as an extension to a testing framework. It will be interesting to continue porting Hypothesis features to the PHP world, given the original features and patterns that Hypothesis shows with respect to the rest of the `*Check` world. + +## References + +The Hypothesis examples shown in this post can be found in [this example repository](https://github.com/giorgiosironi/hypothesis-exploration/blob/master/test_eris.py). + +The Eris examples can instead be found in the example/ folder of [the](https://github.com/giorgiosironi/eris/blob/master/examples/IntegerTest.php#L9) [Eris](https://github.com/giorgiosironi/eris/blob/master/examples/SequenceTest.php#L31) [repository](https://github.com/giorgiosironi/eris/blob/master/examples/WhenTest.php#L8) [on](https://github.com/giorgiosironi/eris/blob/master/examples/MapTest.php#L8) [GitHub](https://github.com/giorgiosironi/eris/blob/master/examples/BindTest.php#L8). diff --git a/HypothesisWorks.github.io/_posts/2016-09-23-hypothesis-3.5.0-release.md b/HypothesisWorks.github.io/_posts/2016-09-23-hypothesis-3.5.0-release.md new file mode 100644 index 0000000000..25240b557a --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-09-23-hypothesis-3.5.0-release.md @@ -0,0 +1,66 @@ +--- +layout: post +tags: news python non-technical +date: 2016-09-23 00:00 +title: 3.5.0 and 3.5.1 Releases of Hypothesis for Python +published: true +author: drmaciver +--- + +This is a combined release announcement for two releases. 3.5.0 +was released yesterday, and 3.5.1 has been released today after +some early bug reports in 3.5.0 + +## Changes + +### 3.5.0 - 2016-09-22 + +This is a feature release. + +* fractions() and decimals() strategies now support min_value and max_value + parameters. Thanks go to Anne Mulhern for the development of this feature. +* The Hypothesis pytest plugin now supports a --hypothesis-show-statistics parameter + that gives detailed statistics about the tests that were run. Huge thanks to + [Jean-Louis Fuchs](https://github.com/ganwell) and [Adfinis-SyGroup](https://www.adfinis-sygroup.ch/) + for funding the development of this feature. +* There is a new event() function that can be used to add custom statistics. + +Additionally there have been some minor bug fixes: + +* In some cases Hypothesis should produce fewer duplicate examples (this will mostly + only affect cases with a single parameter). +* py.test command line parameters are now under an option group for Hypothesis (thanks + to David Keijser for fixing this) +* Hypothesis would previously error if you used function annotations on your tests under + Python 3.4. +* The repr of many strategies using lambdas has been improved to include the lambda body + (this was previously supported in many but not all cases). + + +### 3.5.1 - 2016-09-23 + +This is a bug fix release. + +* Hypothesis now runs cleanly in -B and -BB modes, avoiding mixing bytes and unicode. +* unittest.TestCase tests would not have shown up in the new statistics mode. Now they + do. +* Similarly, stateful tests would not have shown up in statistics and now they do. +* Statistics now print with pytest node IDs (the names you'd get in pytest verbose mode). + + +## Notes + +Aside from the above changes, there are a couple big things behind the scenes of this +release that make it a big deal. + +The first is that the flagship chunk of work, statistics, is a long-standing want to +have that has never quite been prioritised. By funding it, Jean-Louis and Adfinis-SyGroup +successfully bumped it up to the top of the priority list, making it the first funded +feature in Hypothesis for Python! + +Another less significant but still important is that this release marks the first real +break with an unofficial Hypothesis for Python policy of not having any dependencies +other than the standard library and backports. This release adds a dependency on the +uncompyle6 package. This may seem like an odd choice, but it was invaluable for fixing +the repr behaviour, which in turn was really needed for providing good statistics +for filter and recursive strategies. diff --git a/HypothesisWorks.github.io/_posts/2016-10-01-pytest-integration-sponsorship.md b/HypothesisWorks.github.io/_posts/2016-10-01-pytest-integration-sponsorship.md new file mode 100644 index 0000000000..9c717f4608 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-10-01-pytest-integration-sponsorship.md @@ -0,0 +1,65 @@ +--- +layout: post +tags: news python non-technical +date: 2016-10-01 00:00 +title: Seeking funding for deeper integration between Hypothesis and pytest +published: true +author: drmaciver +--- + +Probably the number one complaint I hear from Hypothesis users is that it +"doesn't work" with py.test fixtures. [This isn't true](http://hypothesis.works/articles/hypothesis-pytest-fixtures/), +but it does have one very specific limitation in how it works that annoys people: +It only runs function scoped fixtures once for the entire test, not once per +example. Because of the way people use function scoped fixtures for handling +stateful things like databases, this often causes people problems. + +I've been [maintaining for a while](https://github.com/pytest-dev/pytest/issues/916) that +this is impossible to fix without some changes on the pytest end. + +The good news is that this turns out not to be the case. After some conversations with +pytest developers, some examining of other pytest plugins, and a bunch of prototyping, +I'm pretty sure it's possible. It's just really annoying and a lot of work. + +So that's the good news. The bad news is that this isn't going to happen without +someone funding the work. + +I've now spent about a week of fairly solid work on this, and what I've got is +quite promising: The core objective of running pytest fixtures for every examples +works fairly seamlessly. + +But it's now in the long tail of problems that will need to be squashed before +this counts as an actual production ready releasable piece of work. A number of +things *don't* work. For example, currently it's running some module scoped +fixtures once per example too, which it clearly shouldn't be doing. It also +currently has some pretty major performance problems that are bad enough that +I would consider them release blocking. + +As a result I'd estimate there's easily another 2-3 weeks of work needed to +get this out the door. + +Which brings us to the crux of the matter: 2-3 additional weeks of free work +on top of the one I've already done is 3-4 weeks more free work than I +particularly want to do on this feature, so without sponsorship it's not +getting finished. + +I typically charge £400/day for work on Hypothesis (this is heavily discounted +off my normal rates), so 2-3 weeks comes to £4000 to £6000 (roughly $5000 +to $8000) that has to come from somewhere. + +I know there are a number of companies out there using pytest and Hypothesis +together. I know from the amount of complaining about this integration that +this is a real problem you're experiencing. So, I think this money should +come from those companies. Besides helping to support a tool you've already +got a lot of value out of, this will expand the scope of what you can easily +test with Hypothesis a lot, and will be hugely beneficial to your bug finding +efforts. + +This is a model that has worked well before with the funding of the recent +statistics work by [Jean-Louis Fuchs](https://github.com/ganwell) and +[Adfinis-SyGroup](https://www.adfinis-sygroup.ch/), and I'm confident it can +work well again. + +If you work at such a company and would like to talk about funding some or +part of this development, please email me at +[drmaciver@hypothesis.works](mailto:drmaciver@hypothesis.works). diff --git a/HypothesisWorks.github.io/_posts/2016-10-17-canonical-serialization.md b/HypothesisWorks.github.io/_posts/2016-10-17-canonical-serialization.md new file mode 100644 index 0000000000..773c16ec11 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-10-17-canonical-serialization.md @@ -0,0 +1,130 @@ +--- +layout: post +tags: python intro technical properties +date: 2016-10-17 06:00 +title: Another invariant to test for encoders +published: true +author: drmaciver +--- + +[The encode/decode invariant]({{site.url}}{% post_url 2016-04-16-encode-decode-invariant %}) +is one of the most important properties to know about for testing your code with Hypothesis +or other property-based testing systems, because it captures a very common pattern and is +very good at finding bugs. + +But how do you go beyond it? If encoders are that common, surely there must be other things +to test with them? + + + +The first thing that people tend to try is to simply reverse the order of operations. Continuing +the same example from the encode/decode post, this would look something like this: + +```python +from hypothesis import given +from hypothesis.strategies import characters, integers, lists, tuples + + +@given(lists(tuples(characters(), integers(1, 10)))) +def test_encode_inverts_decode(s): + assert encode(decode(s)) == s +``` + +But unlike the other way around, this test can fail for reasons that are not obviously +errors in the system under test. In particular this will fail with the example [('0', 1), ('0', 1)], +because this will be decoded into '00', which will then be encoded into [('0', 2)]. + +In general, it is quite common to have multiple non-canonical representations of data +when encoding: e.g. JSON has non-significant whitespace, an IP address has a wide range +of human readable representations, most compressors will evolve to improve their compression +over time but will still be able to handle compressed files made with old versions of +the code, etc. + +So this test *shouldn't* be expected to pass. + +To rescue this, we might imagine we had some sort of function make\_canonical which +took an encoded data representation and replaced it with a canonical version of that (e.g. +by normalizing whitespace). The test could then look like this: + + +```python +@given(lists(tuples(characters(), integers(1, 10)))) +def test_encode_inverts_decode(s): + assert make_canonical(encode(decode(s))) == make_canonical(s) +``` + +But where would we get that make\_canonical function from? It's not something we really +want to write ourselves. + +Fortunately we can put together the pieces we already have to define such a function +fairly easily. To see this, lets think about what properties make\_canonical should have. + +The following seem reasonable: + +1. encode should always produce canonical data. i.e. encode(t) == make_canonical(encode(t)) +2. Canonical data should represent the same value. i.e. decode(s) == decode(make_canonical(s)) + +But we already know that decode(encode(t)) == t from our original property, so we have a +natural source of data that is the output of encode: We just decode the data and then +encode it again. + +This gives us the following natural definition of make_canonical: + +```python +def make_canonical(data): + return encode(decode(data)) +``` + +But this is nearly the same as the thing we were testing, so we can rewrite our test as: + +```python +@given(lists(tuples(characters(), integers(1, 10)))) +def test_encode_inverts_decode(s): + assert make_canonical(make_canonical(s)) == make_canonical(s) +``` + +This property is called +being idempotent (annoyingly "idempotent" gets used to mean something subtly different +in most API design, but this is the original mathematical meaning). + +It's less obviously necessary than the original one, and you can certainly +write encode/decode pairs that are arguably correct but don't have it (e.g. because +they change the order of keys in a dictionary, or include a timestamp or sequence +number in the output), but I think it's +still worth having and testing. Enforcing consistency like this both helps with debugging +when things go wrong and also tends to flush out other bugs along the way. + +Even if you don't want to enforce this property, it highlights an important issue: You +do need *some* sort of testing of the decoder that doesn't just operate on output from +the encoder, because the encoder will potentially only output a relatively small subset +of the valid range of the format. + +Often however you'll get the property for free. If the encode and decode functions +have the property that whenever x == y then f(x) == f(y), then this property automatically +holds, because make_canonical(x) is encode(decode(encode(decode(x)))), and we know from the +first property that decode(encode(t)) == t, so with t = decode(x) this expression is +encode(decode(x)), which is make_canonical(x) as required. + +Most encode/decode pairs will have this property, but not all. + +The easiest ways to fail to have it are to have side-effects (the aforementioned sequence +number or randomization), but even without side effects it's possible for it to fail +if equality doesn't capture every detail about the type. For example in +Python, if 1.0 was serialized as 1, then the two would compare equal and the property +would pass, but when re-encoding it might exhibit very different properties (although +you'd hope that it wouldn't). Another example is that in Python an OrderedDict and a +dict compare equal regardless of iteration order, which means that two apparently +equal types might encode to different things if they have different iteration orders +defined. + +Ultimately these issues are probably quite niche. It's likely still worth testing for +this property, both because of these problems and also because often [mathematically +equivalent properties can still catch different issues]({{site.url}}{% post_url 2016-06-30-tests-as-complete-specifications %}), +but it's significantly less important than the more general property we started +with. + +-------------------------------------- + +Thanks to [Georges Dubus](https://twitter.com/georgesdubus) who pointed out +the key insight behind the last section on this property following from the original +one. diff --git a/HypothesisWorks.github.io/_posts/2016-10-31-hypothesis-3.6.0-release.md b/HypothesisWorks.github.io/_posts/2016-10-31-hypothesis-3.6.0-release.md new file mode 100644 index 0000000000..618d314eb5 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-10-31-hypothesis-3.6.0-release.md @@ -0,0 +1,57 @@ +--- +layout: post +tags: news python non-technical +date: 2016-10-31 00:00 +title: 3.6.0 Release of Hypothesis for Python +published: true +author: drmaciver +--- + +This is a release announcement for the 3.6.0 release of Hypothesis for +Python. It's a bit of an emergency release. + +Hypothesis 3.5.0 inadvertently added a dependency on GPLed code (see below +for how this happened) which this release removes. This means that if you +are running Hypothesis 3.5.x then there is a good chance you are in violation +of the GPL and you should update immediately. + +Apologies for any inconvenience this may have caused. + + + +### From the Changelog + +This release reverts Hypothesis to its old pretty printing of lambda functions +based on attempting to extract the source code rather than decompile the bytecode. +This is unfortunately slightly inferior in some cases and may result in you +occasionally seeing things like `lambda x: ` in statistics reports and +strategy reprs. + +This removes the dependencies on uncompyle6, xdis and spark-parser. + +The reason for this is that the new functionality was based on uncompyle6, which +turns out to introduce a hidden GPLed dependency - it in turn depended on xdis, +and although the library was licensed under the MIT license, it contained some +GPL licensed source code and thus should have been released under the GPL. + +My interpretation is that Hypothesis itself was never in violation of the GPL +(because the license it is under, the Mozilla Public License v2, is fully +compatible with being included in a GPL licensed work), but I have not consulted +a lawyer on the subject. Regardless of the answer to this question, adding a +GPLed dependency will likely cause a lot of users of Hypothesis to inadvertently +be in violation of the GPL. + +As a result, if you are running Hypothesis 3.5.x you really should upgrade to +this release immediately. + +### Notes + +This Halloween release brought to you by the specter of inadvertent GPL +violations (but sadly this is entirely real and neither trick nor treat). + +This dependency also caused a number of other problems, so in many ways its +not entirely a bad thing that it's gone, but it's still sad to remove +functionality. At some point in the future I will try to restore the +lost functionality, but doing it without access to uncompyle6 will be a +moderate amount of work, so it's not going to be high priority in the +near future. diff --git a/HypothesisWorks.github.io/_posts/2016-12-05-integrated-shrinking.md b/HypothesisWorks.github.io/_posts/2016-12-05-integrated-shrinking.md new file mode 100644 index 0000000000..6f3852bb5b --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-12-05-integrated-shrinking.md @@ -0,0 +1,167 @@ +--- +layout: post +tags: technical details python alternatives +date: 2016-12-05 10:00 +title: Integrated vs type based shrinking +published: true +author: drmaciver +--- + +One of the big differences between Hypothesis and Haskell QuickCheck is +how shrinking is handled. + +Specifically, the way shrinking is handled in Haskell QuickCheck is bad +and the way it works in Hypothesis (and also in test.check and EQC) is +good. If you're implementing a property based testing system, you should +use the good way. If you're using a property based testing system and it +doesn't use the good way, you need to know about this failure mode. + +Unfortunately many (and possibly most) implementations of property based +testing are based on Haskell's QuickCheck and so make the same mistake. + + + +The big difference is whether shrinking is integrated into generation. + +In Haskell's QuickCheck, shrinking is defined based on *types*: Any +value of a given type shrinks the same way, regardless of how it is +generated. In Hypothesis, test.check, etc. instead shrinking is part +of the generation, and the generator controls how the values it produces +shrinks (this works differently in Hypothesis and test.check, and probably +differently again in EQC, but the user visible result is largely the +same) + +This is not a trivial distinction. Integrating shrinking into generation +has two large benefits: + +* Shrinking composes nicely, and you can shrink anything you can generate + regardless of whether there is a defined shrinker for the type produced. +* You can guarantee that shrinking satisfies the same invariants as generation. + +The first is mostly important from a convenience point of view: Although +there are some things it let you do that you can't do in the type based +approach, they're mostly of secondary importance. It largely just saves +you from the effort of having to write your own shrinkers. + +But the second is *really* important, because the lack of it makes your +test failures potentially extremely confusing. + +To see this, lets consider the following example in Hypothesis: + +```python +from hypothesis import given +from hypothesis.strategies import integers + +even_numbers = integers().map(lambda x: x * 2) + + +@given(even_numbers) +def test_even_numbers_are_even(n): + assert n % 2 == 0 +``` + +This test always passes: We generate an even number by multiplying +an integer we've generated by two. No problem. + +Now suppose we made the test fail: + + +```python +from hypothesis import given +from hypothesis.strategies import integers + +even_numbers = integers().map(lambda x: x * 2) + + +@given(even_numbers) +def test_even_numbers_are_even(n): + assert n % 2 == 0 + assert n <= 4 +``` + +This test will of course fail: Any value of n which is at least 5 will +fail the test. + +But which assertion will fail, and what value will cause it to fail? + +In Hypothesis it's the second one, and it will fail with n=6: The numbers +passed in will still always be even integers, and the smallest even +integer which fails the test is 6. + +But suppose Hypothesis implemented type based shrinking (very early +versions of it *did* implement type based shrinking, but this stopped +being the case somewhere well before 1.0 when the API looked very +different). + +In that case Hypothesis would just know that these things that were +failing the tests were integers, so it would say "How about 1? 1 is a +perfectly valid integer. Lets try 1". + +It would pass in n=1, the first assertion would trigger, and the test +would promptly fail. A successful shrink! + +But it's completely not what we wanted. We were just trying to test on +even integers and the shrinking messed this up. + +This is in some sense the classic +[shrinkers are fuzzers](http://blog.regehr.org/archives/1284) problem +where an error is reduced to a different error, but it's a particularly +annoying version of that because an error we care about is being reduced +to an error we don't care about. + +So we have to duplicate +the constraint logic in our test to make this work: + + +```python +from hypothesis import assume, given +from hypothesis.strategies import integers + +even_numbers = integers().map(lambda x: x * 2) + + +@given(even_numbers) +def test_even_numbers_are_even(n): + assume(n % 2 == 0) + assert n % 2 == 0 + assert n <= 4 +``` + +(Having both the assume and the first assert there is of course +redundant) + +In this example the problem was relatively obvious and so easy to +work around, but as your invariants get more implicit and subtle +it becomes really problematic: In Hypothesis it's easy and +convenient to +[generate quite complex data]({{site.url}}{% post_url 2016-05-11-generating-the-right-data %}), +and trying to recreate the invariants that are automatically +satisfied with that in your tests and/or your custom shrinkers would +quickly become a nightmare. + +I don't think it's an accident that the main systems to get this right are +in dynamic languages. It's certainly not *essential* - [the original proposal that +lead to the implementation for test.check was for +Haskell](https://mail.haskell.org/pipermail/libraries/2013-November/021674.html), +and [Jack](https://github.com/ambiata/disorder.hs/tree/master/disorder-jack) is +an alternative property based system for Haskell that does this - but you +feel the pain much more quickly in dynamic languages because the typical +workaround for this problem in Haskell is to define a newtype, which lets you +turn off the default shrinking for your types and possibly define your own. + +But that's a workaround for a problem that shouldn't be there in the first place, +and using it will still result in your having to encode the invariants into your +your shrinkers, which is more work and more brittle than just having it work +automatically. + +So although (as far as I know) none of the currently popular property based +testing systems for statically typed languages implement this behaviour +correctly, they absolutely can and they absolutely should. It will improve +users' lives significantly. + +This of course goes doubly for dynamic languages, where even working around +this problem is hard. + +So, in conclusion, just say no to type based shrinking. It's a bad idea, +and failures your property based tests will become significantly harder +to understand in its presence. diff --git a/HypothesisWorks.github.io/_posts/2016-12-08-compositional-shrinking.md b/HypothesisWorks.github.io/_posts/2016-12-08-compositional-shrinking.md new file mode 100644 index 0000000000..da1d1d3226 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-12-08-compositional-shrinking.md @@ -0,0 +1,145 @@ +--- +layout: post +tags: technical details python alternatives +date: 2016-12-08 9:00 +title: Compositional shrinking +published: true +author: drmaciver +--- + +In [my last article about shrinking]({{site.url}}{% post_url 2016-12-05-integrated-shrinking %}), +I discussed the problems with basing shrinking on the type of the values +to be shrunk. + +In writing it though I forgot that there was a halfway house which is +also somewhat bad (but significantly less so) that you see in a couple +of implementations. + +This is when the shrinking is not type based, but still follows the +classic shrinking API that takes a value and returns a lazy list of +shrinks of that value. Examples of libraries that do this are +[theft](https://github.com/silentbicycle/theft) and +[QuickTheories](https://github.com/NCR-CoDE/QuickTheories). + +This works reasonably well and solves the major problems with type +directed shrinking, but it's still somewhat fragile and importantly +does not compose nearly as well as the approaches that Hypothesis +or test.check take. + +Ideally, as well as not being based on the types of the values being +generated, shrinking should not be based on the actual values generated +at all. + +This may seem counter-intuitive, but it actually works pretty well. + + + +Rather than going into implementation details just yet, lets start +with why this is important. + +Consider the example from the last post: + + +```python +from hypothesis import given +from hypothesis.strategies import integers + +even_numbers = integers().map(lambda x: x * 2) + + +@given(even_numbers) +def test_even_numbers_are_even(n): + assert n % 2 == 0 +``` + +We took a strategy and composed it with a function mapping over +the values that that strategy produced to get a new strategy. + +Suppose the Hypothesis strategy implementation looked something +like the following: + +```python +class SearchStrategy: + def generate(self, random): + raise NotImplementedErro() + + def shrink(self, value): + return () +``` + +i.e. we can generate a value and we can shrink a value that we've +previously generated. By default we don't know how to generate values +(subclasses have to implement that) and we can't shrink anything, +which subclasses are able to fix if they want or leave as is if +they're fine with that. + +(This is in fact how a very early implementation of it looked) + +This is essentially the approach taken by theft or QuickTheories, +and the problem with it is that under this implementation the +'map' function we used above is impossible to define in a way +that preserves shrinking: In order to shrink a generated value, +you need some way to invert the function you're composing with +(which is in general impossible even if your language somehow +exposed the facilities to do it, which it almost certainly +doesn't) so you could take the generated value, map it back +to the value that produced it, shrink that and then compose +with the mapping function. + +Hypothesis and test.check both support even more complicated +composition of strategies (Hypothesis somewhat better than +test.check - both support the same operations, but Hypothesis's +underlying implementation works somewhat better for more +complicated compositions), but even the simplest of compositions +fails if you need to be able to shrink arbitrary values. + +The key idea for fixing this is as follows: In order to shrink +*outputs* it almost always suffices to shrink *inputs*. Although +in theory you can get functions where simpler input leads to more +complicated output, in practice this seems to be rare enough +that it's OK to just shrug and accept more complicated test +output in those cases. + +Given that, the way to shrink the output of a mapped strategy +is to just shrink the value generated from the first strategy +and feed it to the mapping function. + +Which means that you need an API that can support that sort +of shrinking. + +The way this works in test.check is that instead of generating +a single value it generates an entire (lazy) tree of values +with shrinks for them. See [Reid Draper's article on the +subject](http://reiddraper.com/writing-simple-check/) for slightly +more detail. + +This supports mapping fairly easily: We just apply the mapping +function to the rose tree - both the initial generated value, +and all the shrunk child values. + +Hypothesis's implementation is more complicated so will have to +wait for another article, but the key idea behind it is that +Hypothesis takes the "Shrinking outputs can be done by shrinking +inputs" idea to its logical conclusion and has a single unified +intermediate representation that *all* generation is based off. +Strategies can provide hints about possibly useful shrinks to +perform on that representation, but otherwise have very little +control over the shrinking process at all. This supports mapping +even more easily, because a strategy is just a function which +takes an IR object and returns a value, so the mapped strategy +just does the same thing and applies the mapping function. + +Obviously I think Hypothesis's implementation is better, but +test.check's implementation is entirely respectable too and +is probably easier to copy right now if you're implementing +a property based testing system from scratch. + +But I do think that whichever one you start from it's important +to take away the key idea: You can shrink outputs by shrinking +inputs, and strategies should compose in a way that preserves +shrinking. + +The result is significantly more convenient to use because it +means that users will rarely or never have to write their own +shrinking functions, and there are fewer possible places for +shrinking and generation to get out of sync. diff --git a/HypothesisWorks.github.io/_posts/2016-12-10-how-hypothesis-works.md b/HypothesisWorks.github.io/_posts/2016-12-10-how-hypothesis-works.md new file mode 100644 index 0000000000..3e1c7da75b --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2016-12-10-how-hypothesis-works.md @@ -0,0 +1,442 @@ +--- +layout: post +tags: python details technical +date: 2016-12-10 11:00 +title: How Hypothesis Works +published: true +author: drmaciver +--- + +Hypothesis has a very different underlying implementation to any other +property-based testing system. As far as I know, it's an entirely novel +design that I invented. + +Central to this design is the following feature set which *every* +Hypothesis strategy supports automatically (the only way to break +this is by having the data generated depend somehow on external +global state): + +1. All generated examples can be safely mutated +2. All generated examples can be saved to disk (this is important because + Hypothesis remembers and replays previous failures). +3. All generated examples can be shrunk +4. All invariants that hold in generation must hold during shrinking ( + though the probability distribution can of course change, so things + which are only supported with high probability may not be). + +(Essentially no other property based systems manage one of these claims, +let alone all) + +The initial mechanisms for supporting this were fairly complicated, but +after passing through a number of iterations I hit on a very powerful +underlying design that unifies all of these features. + +It's still fairly complicated in implementation, but most of that is +optimisations and things needed to make the core idea work. More +importantly, the complexity is quite contained: A fairly small kernel +handles all of the complexity, and there is little to no additional +complexity (at least, compared to how it normally looks) in defining +new strategies, etc. + +This article will give a high level overview of that model and how +it works. + + + +Hypothesis consists of essentially three parts, each built on top +of the previous: + +1. A low level interactive byte stream fuzzer called *Conjecture* +2. A strategy library for turning Conjecture's byte streams into + high level structured data. +3. A testing interface for driving test with data from Hypothesis's + strategy library. + +I'll focus purely on the first two here, as the latter is complex +but mostly a matter of plumbing. + +The basic object exposed by Conjecture is a class called TestData, +which essentially looks like an open file handle you can read +bytes from: + +```python +class TestData: + def draw_bytes(self, n): + ... +``` + +(note: The Python code in this article isn't an exact copy of what's +found in Hypothesis, but has been simplified for pedagogical reasons). + +A strategy is then just an object which implements a single abstract +method from the strategy class: + +```python +class SearchStrategy: + def do_draw(self, data): + raise NotImplementedError() +``` + +The testing interface then turns test functions plus the strategies +they need into something that takes a TestData object and returns +True if the test fails and False if it passes. + +For a simple example, we can implement a strategy for unsigned +64-bit integers as follows: + + +```python +class Int64Strategy: + def do_draw(self, data): + return int.from_bytes(data.draw_bytes(8), byteorder="big", signed=False) +``` + +As well as returning bytes, draw_bytes can raise an exception +that stops the test. This is useful as a way to stop examples +from getting too big (and will also be necessary for shrinking, +as we'll see in a moment). + +From this it should be fairly clear how we support saving and +mutation: Saving every example is possible because we can just +write the bytes that produced it to disk, and mutation is possible +because strategies are just returning values that we don't +in any way hang on to. + +But how does shrinking work? + +Well the key idea is the one I mentioned in +[my last article about shrinking +]({{site.url}}{% post_url 2016-12-08-compositional-shrinking %}) - +shrinking inputs suffices to shrink outputs. In this case the +input is the byte stream. + +Once Hypothesis has found a failure it begins shrinking the +byte stream using a TestData object that looks like the following: + +```python +class ShrinkingTestData: + def __init__(self, data): + self.data = data + self.index = 0 + + def draw_bytes(self, n): + if self.index + n > len(self.data): + raise StopTest() + result = self.data[self.index : self.index + n] + self.index += n + return result +``` + +Shrinking now reduces to shrinking the byte array that gets +passed in as data, subject to the condition that our transformed +test function still returns True. + +Shrinking of the byte array is designed to try to minimize it +according to the following rules: + +1. Shorter is always simpler. +2. Given two byte arrays of the same length, the one which is + lexicographically earlier (considering bytes as unsigned 8 + bit integers) is simpler. + +You can imagine that some variant of [Delta Debugging](https://en.wikipedia.org/wiki/Delta_Debugging) +is used for the purpose of shrinking the byte array, +repeatedly deleting data and lowering bytes until no +byte may be deleted or lowered. It's a lot more complicated +than that, but I'm mostly going to gloss over that part +for now. + +As long as the strategy is well written (and to some extent +even when it's not - it requires a certain amount of active +sabotage to create strategies that produce more complex data +given fewer bytes) this results in shrinks to the byte array +giving good shrinks to the generated data. e.g. our 64-bit +unsigned integers are chosen to be big endian so that +shrinking the byte data lexicographically shrinks the integer +towards zero. + +In order to get really good deleting behaviour in our strategies +we need to be a little careful about how we arrange things, so +that deleting in the underlying bytestream corresponds to +deleting in generated data. + +For example, suppose we tried to implement lists as follows: + + +```python +class ListStrategy(SearchStrategy): + def __init__(self, elements): + self.elements = elements + + def do_draw(self, data): + n_elements = integers(0, 10).do_draw(self.elements) + return [self.elements.do_draw(data) for _ in range(n_elements)] +``` + +The problem with this is that deleting data doesn't actually +result in deleting elements - all that will happen is that +drawing will run off the end of the buffer. You can +potentially shrink n_elmements, but that only lets you +delete things from the end of the list and will leave a +bunch of left over data at the end if you do - if this is +the last data drawn that's not a problem, and it might be +OK anyway if the data usefully runs into the next strategy, +but it works fairly unreliably. + +I am in fact working on an improvement to how shrinking works +for strategies that are defined like this - they're quite +common in user code, so they're worth supporting - but it's +better to just have deletion of elements correspond to +deletion of data in the underlying bytestream. We can do +this as follows: + + +```python +class ListStrategy(SearchStrategy): + def __init__(self, elements): + self.elements = elements + + def do_draw(self, data): + result = [] + while booleans().do_draw(data): + result.append(self.elements.do_draw(data)) + return result +``` + +We now draw lists as a series True, element, +True, element, ..., False, etc. So if you delete the +interval in the byte stream that starts with a True +and finishes at the end of an element, that just deletes +that element from the list and shifts everything afterwards +left one space. + +Given some careful strategy design this ends up working +pretty well. It does however run into problems in two +minor cases: + +1. It doesn't generate very good data +2. It doesn't shrink very well + +Fortunately both of these are fixable. + +The reason for the lack of good data is that Conjecture +doesn't know enough to produce a good distribution of bytes +for the specific special values for your strategy. e.g. in +our unsigned 64 bit integer examples above it can probably +guess that 0 is a special value, but it's not necessarily +obvious that e.g. focusing on small values is quite useful. + +This gets worse as you move further away from things that +look like unsigned integers. e.g. if you're turning bytes +into floats, how is Conjecture supposed to know that +Infinity is an interesting value? + +The simple solution is to allow the user to provide a +distribution hint: + + +```python +class TestData: + def draw_bytes(self, n, distribution=None): + ... +``` + +Where a distribution function takes a Random object +and a number of bytes. + +This lets users specify the distribution of bytes. It +won't necessarily be respected - e.g. it certainly isn't +in shrinking, but the fuzzer can and does mutate the +values during generation too - but it provides a good +starting point which allows you to highlight special +values, etc. + +So for example we could redefine our integer strategy +as: + + +```python +class Int64Strategy: + def do_draw(self, data): + def biased_distribution(random, n): + if random.randint(0, 1): + return random.randint(0, 100).to_bytes(n, byteorder="big", signed=False) + else: + return uniform(random, n) + + return int.from_bytes( + data.draw_bytes(8, biased_distribution), byteorder="big", signed=False + ) +``` + +Now we have a biased integer distribution which will +produce integers between 0 and 100 half the time. + +We then use the strategies to generate our initial +buffers. For example we could pass in a TestData +implementation that looked like this: + +```python +class GeneratingTestData(TestData): + def __init__(self, random, max_bytes): + self.max_bytes = max_bytes + self.random = random + self.record = bytearray() + + def draw_bytes(self, n, distribution): + if n + len(self.record) > self.max_bytes: + raise StopTest() + result = distribution(self.random, n) + self.record.extend(result) + return result +``` + +This draws data from the provided distribution +and records it, so at the end we have a record of +all the bytes we've drawn so that we can replay +the test afterwards. + +This turns out to be mostly enough. I've got some +pending research to replace this API with something +a bit more structured (the ideal would be that instead +of opaque distribution objects you draw from an explicit +mixture of grammars), but for the moment research on +big changes like that is side lined because nobody is +funding Hypothesis development, so I've not got very far +with it. + +Initial designs tried to avoid this approach by using +data from the byte stream to define the distribution, +but this ended up producing quite opaque structures in +the byte stream that didn't shrink very well, and this +turned out to be simpler. + +The second problem of it not shrinking well is also +fairly easily resolved: The problem is not that we +*can't* shrink it well, but that shrinking ends up +being slow because we can't tell what we need to +do: In our lists example above, the only way we +currently have to delete elements is to delete the +corresponding intervals, and the only way we have +to find the right intervals is to try *all* of them. +This potentially requires O(n^2) deletions to get the +right one. + +The solution is just to do a bit more book keeping as we +generate data to mark useful intervals. TestData now +looks like this: + +```python +class TestData: + def start_interval(self): + ... + + def stop_interval(self): + ... + + def draw_bytes(self, n): + ... + + def draw(self, strategy): + self.start_interval() + result = strategy.do_draw(self) + self.stop_interval() + return result +``` + + +We then pass everything through data.draw instead +of strategy.do_draw to maintain this bookkeeping. + +These mark useful boundaries in the bytestram that +we can try deleting: Intervals which don't cross a +value boundary are much more likely to be useful to +delete. + +There are a large number of other details that are +required to make Hypothesis work: The shrinker and +the strategy library are both carefully developed +to work together, and this requires a fairly large +number of heuristics and special cases to make things +work, as well as a bunch of book keeping beyond the +intervals that I've glossed over. + +It's not a perfect system, but it works and works well: +This has been the underlying implementation of Hypothesis +since the 3.0 release in early 2016, and the switch over +was nearly transparent to end users: the previous +implementation was much closer to a classic QuickCheck +model (with a great deal of extra complexity to support +the full Hypothesis feature set). + +In a lot of cases it even works better than heavily +customized solutions: For example, a benefit of the byte +based approach is that all parts of the data are +fully comprehensible to it. Often more structured +shrinkers get stuck in local minima because shrinking +one part of the data requires simultaneously shrinking +another part of the data, whileas Hypothesis can just +spot patterns in the data and speculatively shrink +them together to see if it works. + +The support for chaining data generation together +is another thing that benefits here. In Hypothesis +you can chain strategies together like this: + + +```python +class SearchStrategy: + def do_draw(self, data): + raise NotImplementedError() + + def flatmap(self, bind): + return FlatmappedStrategy(self, bind) + + +class FlatmappedStrategy(SearchStrategy): + def __init__(self, base, bind): + self.base = base + self.bind = bind + + def do_draw(self, data): + value = data.draw(self.base) + return data.draw(self.bind(value)) +``` + +The idea is that flatmap lets you chain strategy definitions together by +drawing data that is dependent on a value from other strategies. + +This works fairly well in modern Hypothesis, but has historically (e.g. in +test.check or pre 3.0 Hypothesis) been a problem for integrated testing and +generation. + +The reason this is normally a problem is that if you shrink the first value +you've drawn then you essentially *have* to invalidate the value drawn from +bind(value): There's no real way to retain it because it came from a completely +different generator. This potentially results in throwing away a lot of +previous work if a shrink elsewhere suddenly makes it to shrink the +initial value. + +With the Hypothesis byte stream approach this is mostly a non-issue: As long as +the new strategy has roughly the same shape as the old strategy it will just +pick up where the old shrinks left off because they operate on the same +underlying byte stream. + +This sort of structure *does* cause problems for Hypothesis if shrinking the +first value would change the structure of the bound strategy too much, but +in practice it usually seems to work out pretty well because there's enough +flexibility in how the shrinks happen that the shrinker can usually work past +it. + +This model has proven pretty powerful even in its current form, but there's +also a lot of scope to expand it. + +But hopefully not by too much. One of the advantages of the model in its +current form though is its simplicity. The [Hypothesis for Java +prototype](https://github.com/HypothesisWorks/hypothesis-java) was written in +an afternoon and is pretty powerful. The whole of the Conjecture implementation +in Python is a bit under a thousand significant lines of fairly portable code. +Although the strategy library and testing interface are still a fair bit of +work, I'm still hopeful that the Hypothesis/Conjecture approach is the tool +needed to bring an end to the dark era of property based testing libraries +that don't implement shrinking at all. diff --git a/HypothesisWorks.github.io/_posts/2017-03-09-hypothesis-for-researchers.md b/HypothesisWorks.github.io/_posts/2017-03-09-hypothesis-for-researchers.md new file mode 100644 index 0000000000..4284a05390 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2017-03-09-hypothesis-for-researchers.md @@ -0,0 +1,336 @@ +--- +layout: post +tags: python details technical +date: 2017-03-09 11:00 +title: Hypothesis for Computer Science Researchers +published: true +author: drmaciver +--- + +I'm in the process of [trying to turn my work on Hypothesis into a PhD](http://www.drmaciver.com/2017/03/looking-into-starting-a-phd/) +and I realised that I don't have a good self-contained summary as to why researchers should care about it. + +So this is that piece. I'll try to give a from scratch introduction to the why and what of Hypothesis. It's primarily +intended for potential PhD supervisors, but should be of general interest as well (especially if you +work in this field). + +### Why should I care about Hypothesis from a research point of view? + +The short version: + +Hypothesis takes an existing effective style of testing (property-based testing) which has proven highly effective in practice +and makes it accessible to a much larger audience. It does so by taking several previously unconnected ideas from the existing +research literature on testing and verification, and combining them to produce a novel implementation that has proven very effective +in practice. + +The long version is the rest of this article. + + + +The remainder is divided into several sections: + +* [What is Hypothesis?](#what-is-hypothesis) is a from-scratch introduction to Hypothesis. If you are already familiar with + property-based testing (e.g. from QuickCheck) you can probably skip this. +* [How is Hypothesis innovative?](#how-is-hypothesis-innovative) is about the current state of the art of Hypothesis and why + it's interesting. If you've already read [How Hypothesis Works](http://hypothesis.works/articles/how-hypothesis-works/) this + is unlikely to teach you anything new and you can skip it. +* [What prior art is it based on?](#what-prior-art-is-it-based-on) is a short set of references for some of the inspirations + for Hypothesis. You probably shouldn't skip this, because it's short and the linked material is all interesting. +* [What are some interesting research directions?](#what-are-some-interesting-research-directions) explores possible directions + I'm looking into for the future of Hypothesis, some of which I would hope to include in any PhD related to it that I worked + on. You probably shouldn't skip this if you care about this document at all. +* [What should you do with this information?](#what-should-you-do-with-this-information) simply closes off the article and + winds things down. + +So, without further ado, the actual content. + +### What is Hypothesis? + +Hypothesis is an implementation of *property-based testing*, an idea that originated with a Haskell library called QuickCheck. + +Property-based testing is a way to augment your unit tests with a source of structured random data that allows a tool to explore +the edge cases of your tests and attempt to find errors automatically. I've made a [longer and more formal discussion](http://hypothesis.works/articles/what-is-property-based-testing/) +of this definition in the past. + +An example of a property-based test using Hypothesis: + +```python +from hypothesis import given, strategies as st + + +@given(st.lists(st.integers())) +def test_sort_is_idempotent(ls): + sort1 = sorted(ls) + assert sorted(sort1) == sort1 +``` + +This exposes a normal function which can be picked up by a standard runner such as py.test. You can also just call it +directly: + +```python +if __name__ == "__main__": + test_sort_is_idempotent() +``` + +When the test is run, Hypothesis will generate random lists of integers +and pass them to the test. The test sorts the integers, then sorts them again, and asserts that the two results are the same. + +As long as the test passes for every input Hypothesis feeds it this will appear to be a normal test. If it fails however, +Hypothesis will then repeatedly rerun it with progressively simpler examples to try and find a minimal input that causes +the failure. + +To see this, suppose we implemented the following rather broken implementation of sorted: + +```python +def sorted(ls): + return list(reversed(ls)) +``` + +Then on running we would see the following output: + +``` + @given(st.lists(st.integers())) + def test_sort_is_idempotent(ls): + sort1 = sorted(ls) + > assert sorted(sort1) == sort1 + E assert [0, 1] == [1, 0] + E At index 0 diff: 0 != 1 + E Use -v to get the full diff + + sorting.py:12: AssertionError + + ---- Hypothesis ---- + + Falsifying example: test_sort_is_idempotent(ls=[0, 1]) +``` + +Hypothesis probably started with a much more complicated example (the test fails for essentially any list with more +than one element) and then successfully reduced it to the simplest possible example: A list with two distinct elements. + +Importantly, when the test is rerun, Hypothesis will start from the falsifying example it found last time rather than +trying to generate and shrink a new one from scratch. In this particular case that doesn't matter very much - the +example is found very quickly and it always finds the same one - but for more complex and slower tests this is an +vital part of the development work flow: It means that tests run much faster and don't stop failing until +the bug is actually fixed. + +Tests can also draw more data as they execute: + +```python +@given(st.lists(st.integers(), min_size=1), st.data()) +def test_sort_is_idempotent(ls, data): + ls.sort() + i = data.draw(st.integers(0, len(ls) - 1)) + assert ls[i - 1] <= ls[i] +``` + +This fails because we've forgotten than `i` may be zero, and also about Python's negative indexing of lists: + +``` + @given(st.lists(st.integers(), min_size=1), st.data()) + def test_sort_is_idempotent(ls, data): + ls.sort() + i = data.draw(st.integers(0, len(ls) - 1)) + > assert ls[i - 1] <= ls[i] + E assert 1 <= 0 + + sorting.py:15: AssertionError + + ---- Hypothesis ---- + + Falsifying example: test_sort_is_idempotent(ls=[0, 1], data=data(...)) + Draw 1: 0 +``` + +Simplification and example saving work as normal for data drawn in this way. + +Hypothesis also has a form of [model based testing](http://hypothesis.works/articles/rule-based-stateful-testing/), +in which you specify a set of valid operations on your API and it attempts to generate whole programs using those +operations and find a simple one that breaks. + +### How is Hypothesis innovative? + +From an end user point of view, Hypothesis adds several important things: + +* It exists at all and people use it. Historically this sort of testing has been found mostly within the functional + programming community, and attempts to make it work in other languages have not seen much success or widespread + adoption. Some of this is due to novel implementation details in Hypothesis, and some is due to design decisions + making it "feel" like normal testing instead of formal methods. +* Specifying data generators is much easier than in traditional QuickCheck methods, and you get a great deal more + functionality "for free" when you do. This is similar to [test.check](https://github.com/clojure/test.check) for + Clojure, or indeed to the [Erlang version of QuickCheck](http://www.quviq.com/products/erlang-quickcheck/), but + some of the design decisions of Hypothesis make it significantly more flexible here. +* The fact that arbitrary examples can be saved and replayed significantly improves the development work-flow. Other + implementations of property-based testing either don't do this at all, only save the seed, or rely on being able to + serialize the generated objects (which can break invariants when reading them back in). +* The fact that you can generate additional data within the test is often extremely useful, and seems to be unique + to Hypothesis in this category of testing tool. + +These have worked together well to fairly effectively bring property based testing "to the masses", and Hypothesis +has started to see increasingly widespread use within the Python community, and is being actively used in the +development of tools and libraries, as well as in the development of both CPython and pypy, the two major implementations +of Python. + +Much of this was made possible by Hypothesis's novel implementation. + +From an implementation point of view, the novel feature of Hypothesis is this: Unlike other implementations of +property-based testing, it does not need to understand the structure of the data it is generating at all (it sometimes +has to make guesses about it, but its correctness is not dependent on the accuracy of those guesses). + +Hypothesis is divided into three logically distinct parts: + +1. A core engine called *Conjecture*, which can be thought of as an interactive fuzzer for lightly structured byte streams. +2. A strategy library, which is designed to take Conjecture's output and turn it into arbitrary values representable + in the programming language. +3. An interface to external test runners that takes tests built on top of the strategy library and runs them using Conjecture + (in Python this mostly just consists of exposing a function that the test runners can pick up, but in the + [Java Prototype](http://github.com/HypothesisWorks/hypothesis-java) this is more involved and ends up having + to interact with some interesting JUnit specific features. + +Conjecture is essentially the interesting part of Hypothesis's implementation and is what supports most of its functionality: +Generation, shrinking, and serialization are all built into the core engine, so implementations of strategies do not require +any awareness of these features to be correct. They simply repeatedly ask the Conjecture engine for blocks of bytes, which +it duly provides, and they return the desired result. + +If you want to know more about this, I have previously written +[How Hypothesis Works](http://hypothesis.works/articles/how-hypothesis-works/), which provides a bit more detail +about Conjecture and how Hypothesis is built on top of it. + +### What prior art is it based on? + +I've done a fair bit of general reading of the literature in the course of working on Hypothesis. + +The two main papers on which Hypothesis is based are: + +* [QuickCheck: a lightweight tool for random testing of Haskell programs](https://dl.acm.org/citation.cfm?id=351266) essentially + started the entire field of property-based testing. Hypothesis began life as a QuickCheck implementation, and its user facing + API continues to be heavily based on QuickCheck, even though the implementation has diverged very heavily from it. +* [EXPLODE: a lightweight, general system for finding serious storage system errors](https://dl.acm.org/citation.cfm?id=1298469) + provided the key idea on which the Conjecture engine is based - instead of doing static data generation separate from the tests, + provide tests with an interactive primitive from which they can draw data. + +Additionally, the following are major design inspirations in the Conjecture engine, although their designs are not currently +used directly: + +* [American Fuzzy Lop](http://lcamtuf.coredump.cx/afl/) is an excellent security-oriented fuzzer, although one without much + academic connections. I've learned a fair bit about the design of fuzzers from it. For a variety of pragmatic reasons I don't + currently use its most important innovation (branch coverage metrics as a tool for corpus discovery), but I've successfully + prototyped implementations of that on top of Hypothesis which work pretty well. +* [Swarm Testing](https://dl.acm.org/citation.cfm?id=2336763) drove a lot of the early designs of Hypothesis's data generation. + It is currently not explicitly present in the Conjecture implementation, but some of what Conjecture does to induce deliberate + correlations in data is inspired by it. + +### What are some interesting research directions? + +I have a large number of possible directions that my work on Hypothesis could be taken. + +None of these are *necessarily* a thing that would be the focus of a PhD - in doing a PhD I would almost certainly +focus on a more specific research question that might include some or all of them. These are just areas that I am interested +in exploring which I think might form an interesting starting point, and whatever focus I actually end up with will likely +be more carefully tailored in discussion with my potential supervisors. + +One thing that's also worth considering: Most of these research directions are ones that would result in improvements +to Hypothesis without changing its public interface. This results in a great practical advantage to performing the +research because of the relatively large (and ever-growing) corpus of open source projects which are already using +Hypothesis - many of these changes could at least partly be validated by just running peoples' existing tests and seeing +if any new and interesting bugs are found! + +Without further ado, here are some of what I think are the most interesting directions to go next. + +#### More structured byte streams + +My current immediate research focus on Hypothesis is to replace the core Conjecture primitive with a more structured one +that bears a stronger resemblance to its origins in EXPLODE. This is designed to address a number of practical problems +that Hypothesis users currently experience (mostly performance related), but it also opens up a number of other novel +abstractions that can be built on top of the core engine. + +The idea is to pare down the interface so that when calling in to Conjecture you simply draw a single byte, specifying +a range of possible valid bytes. This gives Conjecture much more fine-grained information to work with, which opens up +a number of additional features and abstractions that can be built on top of it. + +From this primitive you can then rebuild arbitrary weighted samplers that shrink correctly (using a variation of the +Alias Method), and arbitrary grammars (probably using [Boltzmann Samplers](https://dl.acm.org/citation.cfm?id=1024669) + or similar). + +This will provide a much more thorough basis for high quality data generation than the current rather ad hoc method +of specifying byte streams. + +This is perhaps more engineering than research, but I think it would at the bare minimum make any paper I wrote about +the core approach of Hypothesis significantly more compelling, and it contains a number of interesting applications of +the theory. + +#### Glass box testing + +Currently Conjecture treats the tests it calls as a black box and does not get much information about what the tests +it executes are actually doing. + +One obvious thing to do which brings in some more ideas from e.g. American Fuzzy Lop is to use more coverage information, +but so far I haven't had much success with making my prototypes of this idea suitable for real world use. The primary +reason for this so far has been that all of the techniques I've found have worked well when tests are allowed +to run for minutes or hours, but the current design focus of Hypothesis assumes tests have seconds to run at most, +which limits the utility of these methods and means they haven't been a priority so far. + +But in principle this should be an extremely profitable line of attack, even with that limitation, and I would like +to explore it further. + +The main idea would be to add a notion of "tags" to the core Conjecture engine which could be used to guide the +search. Coverage would be one source of tags, but others are possible. For example, my previous work on +[Schroedinteger](https://github.com/DRMacIver/schroedinteger) implements what is essentially a form of lightweight +[Concolic testing](https://en.wikipedia.org/wiki/Concolic_testing) that would be another possibly interesting source +of information to use. + +Exactly how much of this is original research and how much is just applications of existing research is yet to be +determined, but I think it very likely that at the very least figuring out how to make use of this sort of information +in sharply bounded time is likely to bear interesting fruit. The opportunity to see how Concolic testing behaves in the +wild is also likely to result in a number of additional questions. + +#### Making the Conjecture engine smarter + +A thing I've looked into in the past is the possible use of grammar inference to improve shrinking and data generation. + +At the time the obstacle I ran into was that the algorithm I was using - +[an optimized variation of L\* search](https://dl.acm.org/citation.cfm?id=73047) - did not get good performance in +practice on the problems I tried it on. + +[Synthesizing Program Input Grammars](https://arxiv.org/abs/1608.01723) promises to lift this restriction by providing +much better grammar inference in practical scenarios that are quite closely related to this problem domain, so I would +like to revisit this and see if it can prove useful. + +There are likely a number of other ways that the Conjecture engine can probe the state of the system under test +to determine interesting potential behaviours, especially in combination with glass box testing features. + +I think there are a lot of potentially interesting research directions in here - especially if this is combined with +the glass box testing. Given that I haven't even been able to make this perform acceptably in the past, the first +one would be to see if I can! + +This will also require a fair bit of practical experimentation to see what works well at actually finding bugs and what +doesn't. This is one area in particular where a corpus of open source projects tested with Hypothesis will be extremely +helpful. + +#### Other testing abstractions + +Despite Hypothesis primarily being a library for property based testing, the core Conjecture engine actually has +very little to do with property-based testing and is a more powerful low-level testing abstraction. It would be interesting +to see how far that could be taken - the existing stateful/model-based testing is one partial step in that direction, +but it could also potentially be used more directly for other things. e.g. in tandem with some of the above features +it could be used for low-level fuzzing of binaries, or using it to drive thread scheduling. + +The nice thing about the Conjecture separation is that because it is so self-contained, it can be used as the core +building block on which other tools can be rebuilt and gain a lot of its major features for free. + +I don't currently have any concrete plans in this direction, but it seems likely there are some interesting possibilities +here that will emerge after more review of the testing literature. + +This is probably just engineering unless some particularly interesting application emerges, but I think the basic +potential of the technology would probably give pretty good odds of such an application. + +### What should you do with this information? + +It depends who you are. + +* If I'm already talking to you because you're a potential PhD supervisor, tell me what about this interests you and + ask me lots of questions. +* If you're a potential PhD supervisor who I'm *not* already talking to but you'd like me to, please let me know! +* If you're somebody else, it's rather up to you. Feel free to send me papers, questions, etc. + +Whoever you are, if you found this document interesting I'd love to hear from you. Drop me an email at +[david@drmaciver.com](mailto:david@drmaciver.com). diff --git a/HypothesisWorks.github.io/_posts/2017-04-05-how-not-to-die-hard-with-hypothesis.md b/HypothesisWorks.github.io/_posts/2017-04-05-how-not-to-die-hard-with-hypothesis.md new file mode 100644 index 0000000000..387dc0b5c7 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2017-04-05-how-not-to-die-hard-with-hypothesis.md @@ -0,0 +1,215 @@ +--- +layout: post +tags: python technical intro +date: 2017-04-05 10:00 +title: Solving the Water Jug Problem from Die Hard 3 with TLA+ and Hypothesis +published: true +author: nchammas +--- + +_This post was originally published on [the author's personal site](http://nchammas.com/writing/how-not-to-die-hard-with-hypothesis). +It is reproduced here with his permission._ + +In the movie [Die Hard with a Vengeance](https://en.wikipedia.org/wiki/Die_Hard_with_a_Vengeance) +(aka Die Hard 3), there is +[this famous scene](https://www.youtube.com/watch?v=6cAbgAaEOVE) where +John McClane (Bruce Willis) and Zeus Carver (Samuel L. Jackson) +are forced to solve a problem or be blown up: Given a 3 gallon jug and +5 gallon jug, how do you measure out exactly 4 gallons of water? + +
+ +

+(The video title is wrong. It's Die Hard 3.) +

+
+ + + +Apparently, you can solve this problem using a formal specification +language like [TLA+](https://en.wikipedia.org/wiki/TLA%2B). I don't know +much about this topic, but it appears that a [formal specification language](https://en.wikipedia.org/wiki/Formal_specification) +is much like a programming language in that it lets you describe the +behavior of a system. However, it's much more rigorous and builds +on mathematical techniques that enable you to reason more effectively +about the behavior of the system you're describing than you can with +a typical programming language. + +In a recent discussion on Hacker News about TLA+, +I came across [this comment](https://news.ycombinator.com/item?id=13919251) +which linked to a fun and simple example +showing [how to solve the Die Hard 3 problem with TLA+](https://github.com/tlaplus/Examples/blob/master/specifications/DieHard/DieHard.tla). +I had to watch the first two lectures from [Leslie Lamport's video course on TLA+](http://lamport.azurewebsites.net/video/videos.html) +to understand the example well, but once I did I was reminded of the +idea of property-based testing and, specifically, [Hypothesis](http://hypothesis.works/). + +So what's property-based testing? It's a powerful way of testing your +logic by giving your machine a high-level description of how your code +should behave and letting it generate test cases automatically to see +if that description holds. Compare that to traditional unit testing, +for example, where you manually code up specific inputs and outputs +and make sure they match. + +## How not to Die Hard with Hypothesis + +Hypothesis has an excellent implementation of property-based testing +[for Python](https://github.com/HypothesisWorks/hypothesis-python). +I thought to myself: I wonder if you can write that +Die Hard specification using Hypothesis? As it turns out, Hypothesis +supports [stateful testing](https://hypothesis.readthedocs.io/en/latest/stateful.html), +and I was able to port the [TLA+ example](https://github.com/tlaplus/Examples/blob/master/specifications/DieHard/DieHard.tla) +to Python pretty easily: + +```python +from hypothesis import note, settings +from hypothesis.stateful import RuleBasedStateMachine, invariant, rule + + +class DieHardProblem(RuleBasedStateMachine): + small = 0 + big = 0 + + @rule() + def fill_small(self): + self.small = 3 + + @rule() + def fill_big(self): + self.big = 5 + + @rule() + def empty_small(self): + self.small = 0 + + @rule() + def empty_big(self): + self.big = 0 + + @rule() + def pour_small_into_big(self): + old_big = self.big + self.big = min(5, self.big + self.small) + self.small = self.small - (self.big - old_big) + + @rule() + def pour_big_into_small(self): + old_small = self.small + self.small = min(3, self.small + self.big) + self.big = self.big - (self.small - old_small) + + @invariant() + def physics_of_jugs(self): + assert 0 <= self.small <= 3 + assert 0 <= self.big <= 5 + + @invariant() + def die_hard_problem_not_solved(self): + note(f"> small: {self.small} big: {self.big}") + assert self.big != 4 + + +# The default of 200 is sometimes not enough for Hypothesis to find +# a falsifying example. +with settings(max_examples=2000): + DieHardTest = DieHardProblem.TestCase +``` + +Calling `pytest` on this file quickly digs up a solution: + +``` +self = DieHardProblem({}) + + @invariant() + def die_hard_problem_not_solved(self): + note("> small: {s} big: {b}".format(s=self.small, b=self.big)) +> assert self.big != 4 +E AssertionError: assert 4 != 4 +E + where 4 = DieHardProblem({}).big + +how-not-to-die-hard-with-hypothesis.py:17: AssertionError +----------------------------- Hypothesis ----------------------------- +> small: 0 big: 0 +Step #1: fill_big() +> small: 0 big: 5 +Step #2: pour_big_into_small() +> small: 3 big: 2 +Step #3: empty_small() +> small: 0 big: 2 +Step #4: pour_big_into_small() +> small: 2 big: 0 +Step #5: fill_big() +> small: 2 big: 5 +Step #6: pour_big_into_small() +> small: 3 big: 4 +====================== 1 failed in 0.22 seconds ====================== +``` + +## What's Going on Here + +The code and test output are pretty self-explanatory, but here's a +recap of what's going on: + +We're defining a state machine. That state machine has an initial +state (two empty jugs) along with some possible transitions. Those +transitions are captured with the `rule()` decorator. The initial +state and possible transitions together define how our system works. + +Next we define invariants, which are properties that must always hold +true in our system. Our first invariant, `physics_of_jugs`, says that +the jugs must hold an amount of water that makes sense. For example, +the big jug can never hold more than 5 gallons of water. + +Our next invariant, `die_hard_problem_not_solved`, is where it gets +interesting. Here we're declaring that the problem of getting exactly +4 gallons in the big jug _cannot_ be solved. Since Hypothesis's job +is to test our logic for bugs, it will give our state machine a +thorough shake down and see if we ever violate our invariants. +In other words, we're basically goading Hypothesis into solving the +Die Hard problem for us. + +I'm not entirely clear on how Hypothesis does its work, but I know +the basic summary is this: It takes the program properties we've +specified -- including things like rules, invariants, data types, and +function signatures -- and generates data or actions to probe the +behavior of our program. If Hypothesis finds a piece of data or +sequence of actions that get our program to violate its stated properties, it +tries to whittle that down to a _minimum falsifying example_---i.e. +something that exposes the same problem but with a minimum number of +steps. This makes it much easier for you to understand how Hypothesis +broke your code. + +Hypothesis's output above tells us that it was able to violate the +`die_hard_problem_not_solved` invariant and provides us with a +minimal reproduction showing exactly how it did so. That reproduction +is our solution to the problem. It's also how McClane and Carver did +it in the movie! + +## Final Thoughts + +All in all, I was pretty impressed with how straightforward it was to +translate the TLA+ example into Python using Hypothesis. And when +Hypothesis spit out the solution, I couldn't help but smile. It's +pretty cool to see your computer essentially generate a program that +solves a problem for you. And the Python version of the Die Hard +"spec" is not much more verbose than the +original in TLA+, though TLA+'s notation for current vs. next value +(e.g. `small` vs. `small'`) is elegant and cuts out the need to have +variables like `old_small` and `old_big`. + +I don't know how Hypothesis compares to TLA+ in a general sense. I've +only just started to learn about property-based testing and TLA+, and +I wonder if they have a place in the work that I do these days, which +is mostly Data Engineering-type stuff. Still, I found this little +exercise fun, and I hope you learned something interesting from it. + +_Thanks to [Julia], [Dan], Laura, Anjana, and Cip for reading drafts +of this post._ + +[Julia]: http://jvns.ca/ +[Dan]: https://danluu.com/ diff --git a/HypothesisWorks.github.io/_posts/2017-07-16-types-and-properties.md b/HypothesisWorks.github.io/_posts/2017-07-16-types-and-properties.md new file mode 100644 index 0000000000..93f83402d3 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2017-07-16-types-and-properties.md @@ -0,0 +1,190 @@ +--- +layout: post +tags: technical details python alternatives +date: 2017-07-16 10:00 +title: Moving Beyond Types +published: true +author: drmaciver +--- + +If you look at the original property-based testing library, the Haskell version of QuickCheck, +tests are very closely tied to types: The way you typically specify a property is by inferring +the data that needs to be generated from the types the test function expects for its arguments. + +This is a bad idea. + + + +The first part of why +[I've talked about already]({{site.url}}{% post_url 2016-12-05-integrated-shrinking %}) - +you don't want to tie the shrinking of the data to its type, because it makes testing +significantly more brittle. + +But you could solve that and still have data generation be tied to type, and it would still +be a bad idea. + +The reason for this is very simple: Often you want to generate something much more specific +than any value of a given type. + +If you look at the [Hypothesis strategies module](https://hypothesis.readthedocs.io/en/latest/data.html) +and what you can generate, many of these look like types. But if you look closer, nearly +every single one of them has options to configure them to be more specific. + +Consider something like the following: + +```python +from statistics import mean + +from hypothesis import given, strategies as st + + +@given(st.lists(st.floats(allow_nan=False, allow_infinity=False), min_size=1)) +def test_mean_is_in_bounds(ls): + assert min(ls) <= mean(ls) <= max(ls) +``` + +In this test we've restricted the domain we're testing on so we can focus on a property +that doesn't hold in generality: The mean of an empty list is an +error, so that needs a special case test for it, and calculations with `NaN` or infinity +are special cases that don't always have a well-defined result and even when they do +don't necessarily satisfy this property. + +This happens a lot: Frequently there are properties that only hold in some restricted +domain, and so you want more specific tests for that domain to complement your other +tests for the larger range of data. + +When this happens you need tools to generate something more specific, and those requirements +don't map naturally to types. + +Consider writing this code based on types instead: + +```python +from statistics import mean +from typing import List + +from hypothesis import given, strategies as st + + +@given(ls=...) +def test_mean_is_in_bounds(ls: List[float]): + assert min(ls) <= mean(ls) <= max(ls) +``` + +(this code doesn't work at the time of this writing, but it will soon - +[the pull request to implement it](https://github.com/HypothesisWorks/hypothesis-python/pull/643) +is in fairly late-stage review). + +But this doesn't do the right thing: We've dropped the conditions from the +previous test that our floats are all finite and our lists are all non-empty. So now +we have to add a precondition to make the test valid: + +```python +import math +from statistics import mean +from typing import List + +from hypothesis import assume, given, strategies as st + + +@given(ls=...) +def test_mean_is_in_bounds(ls: List[float]): + assume(len(ls) > 1) + assume(all(math.isfinite(x) for x in ls)) + assert min(ls) <= mean(ls) <= max(ls) +``` + +But this is now substantially longer and less readable than the original approach! + +In Haskell, traditionally we would fix this with a newtype declaration which wraps the type. +We could find a newtype NonEmptyList and a newtype FiniteFloat and then say that we actually +wanted a NonEmptyList[FiniteFloat] there. + +In Python we could probably do more or less the same thing, either by creating new wrapper +types or by subclassing list and float (which you shouldn't do. Subclassing builtins in Python +leads to really weird behaviour) if we wanted to save a few lines, but it's much more noisy. + +But why should we bother? Especially if we're only using these in one test, we're not actually +interested in these types at all, and it just adds a whole bunch of syntactic noise when you +could just pass the data generators directly. Defining new types for the data you want to generate +is purely a workaround for a limitation of the API. + +If you were working in a dependently typed language where you could already naturally express +this in the type system it *might* be OK (I don't have any direct experience of working in +type systems that strong), but I'm sceptical of being able to make it work well - you're unlikely +to be able to automatically derive data generators in the general case, because the needs of +data generation "go in the opposite direction" from types (a type is effectively a predicate which +consumes a value, where a data generator is a function that produces a value, so in order to produce +a generator for a type automatically you need to basically invert the predicate). I suspect most +approaches here will leave you with a bunch of sharp edges, but I would be interested to see +experiments in this direction. + +That being said, I don't think it's that useful to do. Where stronger types are naturally +present in the program, taking advantage of them to get better testing is certainly worth +doing, but creating a specific type to represent the exact range of valid inputs to a +specific test isn't because for tests the impact of a run time error that you could have caught at +compile time is substantially reduced (all it causes is a spurious test failure - where your +tests are long running or non-deterministic as in property-based testing this *does* matter, +but it still doesn't cause a problem in production). + +But this is creating hard problems where none need exist, and beside which it's not the situation +with most of the current generation of languages and property based libraries for them. +Here, using explicit data generators is a clear improvement, and just as well-typed +(in a statically typed language each data generator has a return type after all). + +You *can* use data generators directly in Haskell QuickCheck too, with an explicit +[forAll](https://hackage.haskell.org/package/QuickCheck-2.10.0.1/docs/Test-QuickCheck-Property.html#v:forAll) +but it's almost as awkward as the newtype approach, particularly if you want more than one +data generator (it's even more awkward if you want shrinking - you have to use forAllWithShrink and +explicitly pass a shrink function). + +This is more or less intrinsic to the type based approach. If you want tinkering with the +data generation to be non-awkward, starting from data generators needs to become the default. + +And experience suggests that when you make customising the data generation easy, people do +it. It's nice to be able to be more specific in your testing, but if you make it too high +effort people either don't do it, or do it using less effective methods like adding +preconditions to tests (assume in Hypothesis, or `==>` in QuickCheck), either of which reduces +the quality of your testing and causes more bugs to slip through as a result. + +Fortunately, it already *is* the default in most of the newer implementations of +property-based testing. The only holdouts are ones that directly copied Haskell QuickCheck. + +Originally this was making a virtue of a necessity - most of the implementations +which started off the trend of data generator first tests are either for dynamic languages +(e.g. Erlang, Clojure, Python) or languages with very weak type systems (e.g. C) where +type first is more or less impossible, but it's proven to be a much more usable design. + +And it's perfectly compatible with static typing too. [Hedgehog](https://hackage.haskell.org/package/hedgehog) +is a relatively recent property-based testing library for Haskell that takes this approach, +and it works just as well in Haskell as it does in any language. + +It's also perfectly compatible with being able to derive a data generator from a type +for the cases where you really want to. We saw a hint at that with the upcoming +Hypothesis implementation above. You could easily do the same by having something +like the following in Haskell (mimicking the type class of QuickCheck): + +```haskell +class Arbitrary a where + arbitrary :: Gen a +``` + +You can then simply use `arbitrary` like you would any other data generator. As far as I know +Hedgehog doesn't do this anywhere (but you can use QuickCheck's Arbitrary with +the hedgehog-quickcheck package), but in principle there's nothing stopping it. + +Having this also makes it much easier to define new data generators. I'm unlikely to use the +support for `@given` much, but I'm much more excited that it will also +work with `builds`, which will allow for a fairly seamless transition between +inferring the default strategy for a type and writing a custom generator. You +will, for example, be able to do `builds(MyType)` and have every constructor +argument automatically filled in (if it's suitably annotated), but you can +also do e.g. `builds(MyType, some_field=some_generator)` to override a particular +default while leaving the others alone. + +(This API is somewhere where the dynamic nature of Python helps a fair bit, but you +could almost certainly do something equivalent in Haskell with a bit more noise +or a bit more template Haskell) + +So this approach doesn't have to be data generator-only, even if it's data generator first, +but if you're going to pick one the flexibility of the data generator based test specification +is hard to beat, regardless of how good your type system is. diff --git a/HypothesisWorks.github.io/_posts/2017-09-14-multi-bug-discovery.md b/HypothesisWorks.github.io/_posts/2017-09-14-multi-bug-discovery.md new file mode 100644 index 0000000000..c2d101e279 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2017-09-14-multi-bug-discovery.md @@ -0,0 +1,197 @@ +--- +layout: post +tags: technical details python +date: 2017-09-26 12:00 +title: When multiple bugs attack +published: true +author: drmaciver +--- + +When Hypothesis finds an example triggering a bug, it tries to shrink the example +down to something simpler that triggers it. This is a pretty common feature, and +most property-based testing libraries implement something similar (though there +are a number of differences between them). Stand-alone test case reducers are +also fairly common, as it's a useful thing to be able to do when reporting bugs +in external projects - rather than submitting a giant file triggering the bug, +a good test case reducer can often shrink it down to a couple of lines. + +But there's a problem with doing this: How do you know that the bug you started +with is the same as the bug you ended up with? + +This isn't just an academic question. [It's very common for the bug you started +with to slip to another one](https://blog.regehr.org/archives/1284). + +Consider for example, the following test: + +```python +from hypothesis import given, strategies as st + + +def mean(ls): + return sum(ls) / len(ls) + + +@given(st.lists(st.floats())) +def test(ls): + assert min(ls) <= mean(ls) <= max(ls) +``` + +This has a number of interesting ways to fail: We could pass `NaN`, we could +pass `[-float('inf'), +float('inf')]`, we could pass numbers which trigger a +precision error, etc. + +But after test case reduction, we'll pass the empty list and it will fail +because we tried to take the min of an empty sequence. + +This isn't necessarily a huge problem - we're still finding a bug after all +(though in this case as much in the test as in the code under test) - +and sometimes it's even desirable - you find more bugs this way, and sometimes +they're ones that Hypothesis would have missed - but often it's not, and an +interesting and rare bug slips to a boring and common one. + +Historically Hypothesis has had a better answer to this than most - because +of the Hypothesis example database, all intermediate bugs are saved and a +selection of them will be replayed when you rerun the test. So if you fix +one bug then rerun the test, you'll find the other bugs that were previously +being hidden from you by that simpler bug. + +But that's still not a great user experience - it means that you're not getting +nearly as much information as you could be, and you're fixing bugs in +Hypothesis's priority order rather than yours. Wouldn't it be better if Hypothesis +just told you about all of the bugs it found and you could prioritise them yourself? + +Well, as of Hypothesis 3.29.0, released a few weeks ago, now it does! + +If you run the above test now, you'll get the following: + +``` +Falsifying example: test(ls=[nan]) +Traceback (most recent call last): + File "/home/david/hypothesis-python/src/hypothesis/core.py", line 671, in run + print_example=True, is_final=True + File "/home/david/hypothesis-python/src/hypothesis/executors.py", line 58, in default_new_style_executor + return function(data) + File "/home/david/hypothesis-python/src/hypothesis/core.py", line 120, in run + return test(*args, **kwargs) + File "broken.py", line 8, in test + def test(ls): + File "/home/david/hypothesis-python/src/hypothesis/core.py", line 531, in timed_test + result = test(*args, **kwargs) + File "broken.py", line 9, in test + assert min(ls) <= mean(ls) <= max(ls) +AssertionError + +Falsifying example: test(ls=[]) +Traceback (most recent call last): + File "/home/david/hypothesis-python/src/hypothesis/core.py", line 671, in run + print_example=True, is_final=True + File "/home/david/hypothesis-python/src/hypothesis/executors.py", line 58, in default_new_style_executor + return function(data) + File "/home/david/hypothesis-python/src/hypothesis/core.py", line 120, in run + return test(*args, **kwargs) + File "broken.py", line 8, in test + def test(ls): + File "/home/david/hypothesis-python/src/hypothesis/core.py", line 531, in timed_test + result = test(*args, **kwargs) + File "broken.py", line 9, in test + assert min(ls) <= mean(ls) <= max(ls) +ValueError: min() arg is an empty sequence + +You can add @seed(67388524433957857561882369659879357765) to this test to reproduce this failure. +Traceback (most recent call last): + File "broken.py", line 12, in + test() + File "broken.py", line 8, in test + def test(ls): + File "/home/david/hypothesis-python/src/hypothesis/core.py", line 815, in wrapped_test + state.run() + File "/home/david/hypothesis-python/src/hypothesis/core.py", line 732, in run + len(self.falsifying_examples,))) +hypothesis.errors.MultipleFailures: Hypothesis found 2 distinct failures. +``` + +(The stack traces are a bit noisy, I know. +[We have an issue open about cleaning them up](https://github.com/HypothesisWorks/hypothesis-python/issues/848)). + +All of the different bugs are minimized simultaneously and take full advantage of Hypothesis's +example shrinking, so each bug is as easy (or hard) to read as if it were the only bug we'd found. + +This isn't perfect: The heuristic we use for determining if two bugs are the same is whether they +have the same exception type and the exception is thrown from the same line. This will necessarily +conflate some bugs that are actually different - for example, `[float('nan')]`, +`[-float('inf'), float('inf')]` and `[3002399751580415.0, 3002399751580415.0, 3002399751580415.0]` +each trigger the assertion in the test, but they are arguably "different" bugs. + +But that's OK. The heuristic is deliberately conservative - the point is not that it can +distinguish whether any two examples are the same bug, just that any two examples it distinguishes +are different enough that it's interesting to show both, and this heuristic definitely manages that. + +As far as I know this is a first in property-based testing libraries (though something like it is +common in fuzzing tools, and [theft is hot on our tail with something similar]( +https://github.com/silentbicycle/theft/compare/develop-failure_tagging)) and there's been +[some interesting related but mostly orthogonal research]( +http://www.cse.chalmers.se/~nicsma/papers/more-bugs.pdf) in Erlang QuickCheck. + +It was also surprisingly easy. + +A lot of things went right in writing this feature, some of them technical, some of them social, +somewhere in between. + +The technical ones are fairly straightforward: Hypothesis's core model turned out to be very +well suited to this feature. Because Hypothesis has a single unified intermediate representation +which defines a total ordering for simplicity, adapting Hypothesis to shrink multiple things at +once was quite easy - whenever we attempt a shrink and it produces a different bug than the one +we were looking for, we compare it to our existing best example for that bug and replace it if +the current one is better (or we've discovered a new bug). We then just repeatedly run the shrinking +process for each bug we know about until they've all been fully shrunk. + +This is in a sense not surprising - I've been thinking about the problem of multiple-shrinking for +a long time and, while this is the first time it's actually appeared in Hypothesis, the current +choice of model was very much informed by it. + +The social ones are perhaps more interesting. Certainly I'm very pleased with how they turned +out here. + +The first is that this work emerged tangentially from +[the recent Stripe funded work](https://stripe.com/blog/hypothesis) - Stripe paid me +to develop some initial support for testing Pandas code with Hypothesis, and I observed +a bunch of bug slippage happening in the wild while I was testing that (it turns out there +are quite a lot of ways to trigger exceptions from Pandas - they weren't really Pandas +bugs so much as bugs in the Pandas integration, but they still slipped between several +different exception types), so that was what got me thinking about this problem again. + +Not by accident, this feature also greatly simplified the implementation +of [the new deadline feature](https://hypothesis.readthedocs.io/en/latest/settings.html#hypothesis.settings.deadline) +that [Smarkets](https://smarkets.com/) funded, which was going to have to have a lot of +logic about how deadlines and bugs interacted, but all that went away as soon as we were +able to handle multiple bugs sensibly. + +This has been a relatively consistent theme in Hypothesis development - practical problems +tend to spark related interesting theoretical developments. It's not a huge exaggeration +to say that the fundamental Hypothesis model exists because I wanted to support testing +Django nicely. So the recent funded development from Stripe and Smarkets has been a +great way to spark a lot of seemingly unrelated development and improve Hypothesis +for everyone, even outside the scope of the funded work. + +Another thing that really helped here is our review process, and [the review from Zac +in particular](https://github.com/HypothesisWorks/hypothesis-python/pull/836). + +This wasn't the feature I originally set out to develop. It started out life as a +much simpler feature that used much of the same machinery, and just had a goal of +avoiding slipping to new errors all together. Zac pushed back with some good questions +around whether this was really the correct thing to do, and after some experimentation +and feedback I eventually hit on the design that lead to displaying all of the errors. + +Our [review handbook](https://github.com/HypothesisWorks/hypothesis-python/blob/master/guides/review.rst) +emphasises that code review is a collaborative design process, and I feel this was +a particularly good example of that. We've created a great culture of code review, +and we're reaping the benefits (and if you want to get in on it, we could always +use more people able and willing to do review...). + +All told, I'm really pleased with how this turned out. I think it's a nice example +of getting a lot of things right up front and this resulting in a really cool new +feature. + +I'm looking forward to seeing how it behaves in the wild. If you notice any +particularly fun examples, do [let me know](mailto:david@drmaciver.com), or write +up a post about them yourself! diff --git a/HypothesisWorks.github.io/_posts/2017-09-28-threshold-problem.md b/HypothesisWorks.github.io/_posts/2017-09-28-threshold-problem.md new file mode 100644 index 0000000000..dace0058e8 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2017-09-28-threshold-problem.md @@ -0,0 +1,106 @@ +--- +layout: post +tags: technical details python +date: 2017-09-28 11:00 +title: The Threshold Problem +published: true +author: drmaciver +--- + +In [my last post]({{site.url}}{% post_url 2017-09-14-multi-bug-discovery %}) I mentioned +the problem of bug slippage: When you start with one bug, reduce the test case, and end +up with another bug. + +I've run into another related problem twice now, and it's not one I've seen talked about +previously. + +The problem is this: Sometimes shrinking makes a bug seem much less interesting than it +actually is. + + + +I first noticed this problem when [Ned Batchelder](https://nedbatchelder.com/) asked me +about some confusing behaviour he was seeing: He was testing some floating point code and +had an assertion that the error was not greater than some threshold. Let's say 0.5 (I could +dig up the IRC logs, but the exact number doesn't matter). + +Hypothesis said "Ah ha! Here is an example where the error is 0.500001. A bug!". + +Ned sighed and thought "Oh great, floating point precision issues", but on further +investigation it turned out that that wasn't it at all. The error could be arbitrarily large, +it's just that Hypothesis reliably gave an example where it was almost as small as it could +possibly be and still fail. + +This wasn't a bug, either. This is how Hypothesis, QuickCheck, and all of the other tools +in this family are designed to work. + +The problem is that test case reduction is designed to produce the simplest example +possible to demonstrate the bug. If the bug can be expressed as happening when some +score exceeds some threshold, and the score is one that tends to increase with example +size, then the failing example that a property-based testing library gives you will tend +to be one where the score is barely above that threshold, making the problem look much +less bad than it actually is. + +This isn't even a bug in Hypothesis - QuickCheck or any other property-based testing would +do the same. It's literally working as intended. + +Arguably it's not even really a problem: Hypothesis has demonstrated the bug, and it's +done so with a simple example which should thus be easy to understand. + +But I can't help but feel that we could do better. It definitely produces misleading +examples even if they technically demonstrate the right problem, and misleading examples +are a great way to waste the user's time. + +I also ran into this problem again recently, where it was more of a problem because it +was resulting in flaky tests. + +I recently introduced [a deadline feature](https://hypothesis.readthedocs.io/en/latest/settings.html?highlight=deadline#hypothesis.settings.deadline) +as part of the work on Hypothesis performance legibility that [Smarkets](https://smarkets.com/) +are funding. This causes slow examples to be treated as failures: If an example passes +but took longer than your deadline to run, it raises `DeadlineExceeded`. This is treated +as a normal error and Hypothesis shrinks it like anything else (including allowing it to +participate in the multi-shrinking process). + +The problem is that it's exactly this sort of threshold problem: You literally have a +score (the run time) and a threshold (the deadline) such that when the score exceeds +the threshold the test fails. Large examples are certainly likely to be slower, so you +will consistently get examples which are right on the boundary of being too slow. + +Which is fine, except that Hypothesis relies on repeatability to display test errors - +once it has a minimized example, it replays the test so it can show you the example, +print the exception, etc. And test run times are not actually repeatable - a test that +takes 201ms on first running might take 199ms on the next run. This then results in +Hypothesis thinking the test is flaky - it previously raised `DeadlineExceeded`, and now it +doesn't. This lead to [Issue 892](http://github.com/HypothesisWorks/hypothesis-python/issues/892), +where Florian Bruhin ran into precisely this problem when testing [Qutebrowser](https://www.qutebrowser.org/). + +The [solution I've ended up opting for there](https://github.com/HypothesisWorks/hypothesis-python/pull/899) +is to temporarily raise the deadline during shrinking +to something halfway between the actual deadline and the largest runtime we've seen. This +ensures that we shrink to a larger threshold than the deadline, and then when we replay +we should comfortably exceed the real deadline unless the test performance actually *is* +really flaky (in which case I've also improved the error message). + +This solution is currently very specific to the problem of the deadlines, and that's +fine - there's no need to rush to a fully general solution, and deadlines have slightly +different constraints than other variants of this due to the unreliability of timing - +but it is something I'd like to see solved more generally. + +One thing I have thought about for a while is adding some notion of scoring to +Hypothesis - e.g. letting people record some score that recorded your progress +in testing (testing games where the score could be e.g. the level you've reached, or +your literal score in the game, was one use case I had in mind). This would seem to be +another good example for that - if you could make your score available to Hypothesis +in some way (or if Hypothesis could figure it out automatically!), then a similar +solution to the above could be used: If Hypothesis notices that the score of the +shrunk example is drastically different from the score of the starting example, it +could try rerunning the shrinking process with the additional constraint that the +score should stay closer to that of the original example, and display the newly shrunk +example with the larger (or smaller) score alongside it. This would work as part +of the new multiple failures reporting, so you would see both examples side by side. + +This needs more thought before I jump in and implement something, but I think this is +an important problem to solve to improve the usability of Hypothesis in particular and +property-based testing in general. Shrinking is a great start to making the problems +testing exposes legible to users, but it's only a start, and we need to do more to +try to improve developers' productivity when debugging the problems we show them. diff --git a/HypothesisWorks.github.io/_posts/2018-01-08-smarkets.md b/HypothesisWorks.github.io/_posts/2018-01-08-smarkets.md new file mode 100644 index 0000000000..f67f02f515 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2018-01-08-smarkets.md @@ -0,0 +1,192 @@ +--- +layout: post +tags: python +date: 2018-01-08 06:00 +title: Smarkets's funding of Hypothesis +published: true +author: drmaciver +--- + +Happy new year everybody! + +In this post I'd like to tell you about one of the nice things that happened in 2017: +The Hypothesis work that was funded by [Smarkets](https://smarkets.com/careers) +Smarkets are an exchange for peer-to-peer trading of bets but, more importantly for us, +they are fairly heavy users of Hypothesis for the Python part of their stack. + + + +Smarkets approached me a while back to talk about possibly funding some Hypothesis development. +We talked a bit about what their biggest pain points with Hypothesis were, +and it emerged that they were having a lot of trouble with Hypothesis performance. + +We talked a bit about how to speed Hypothesis up, and I suggested some diagnostic tests +they could do to see where the problem was, and it fairly quickly emerged that in fact +they mostly *weren't* having trouble with Hypothesis performance - Hypothesis indeed was +much slower than it should have been in their use case, but more than an order of magnitude +more time was spent in their test code rather than in Hypothesis. + +So on further discussion we decided that actually their big problem was not the performance +of Hypothesis per se, but instead the *legibility* of Hypothesis performance problems - when +tests using Hypothesis were slow, it was non-obvious why that might be the case, and it +might be extremely difficult to reproduce the problem. + +This is the sort of problem where it's really useful to have user feedback and funding, +because it's more or less a non-problem for me and - to a lesser extent - anyone who already +works on Hypothesis. Because we've got much deeper knowledge of the internals and the failure +modes, we're largely just used to working around these issues. More feedback from Hypothesis +would be *helpful*, but it's not *essential*. + +So, given that in the normal course of things Hypothesis development is mostly driven by what +we feel like working on, this is really the sort of work that will only happen with a source +of external funding for it. Thus it's really great that Smarkets were willing to step up and +fund it! + +After some discussion we ended up settling on four features that would significantly improve +the situation for them: + +### Identification of examples causing slow tests + +This was the introduction of the [deadline](https://hypothesis.readthedocs.io/en/latest/settings.html#hypothesis.settings.deadline) +feature, which causes Hypothesis to treat slow tests as failures - when set, a test that takes +longer than its set deadline (not counting data generation) raises a `DeadlineExceeded` error. + +This is a bit of a blunt instrument, but it is *very* effective at getting test runtime under control! + +There turned out to be some amusing complications in developing it, so this feature was spread over a +number of releases as we found and worked out its various kinks (3.27.0, 3.31.1, 3.38.2). + +One of the interesting problems we found was that deadlines have a +[threshold problem]({{site.url}}{% post_url 2017-09-28-threshold-problem %}) - because the shrinking +process tends to find examples which are just on the cusp of failure, often when you rerun a fully shrunk example it doesn't fail! + +I went back and forth on the best solution for this for a while, but in the end the best solution turned out to be a simple one - +raise the deadline during example generation and shrinking, then replay with the actual set deadline. + +This does mean that tests that are *right* on the cusp of being too slow may pass artificially, but that's substantially better +than introducing flaky failures. + +On top of this we also needed to make this feature play well with [inline data generation](https://hypothesis.readthedocs.io/en/latest/data.html#drawing-interactively-in-tests) - +the complicating factor was that data generation is much faster when replaying examples or shrinking than it is during generation +(the generation logic is rather too complicated. I have some long-term plans to make it simpler, which would make this difference largely go away). +Fortunately, the work done for the next feature made this easy to do. + +### Breakdown of generation time in statistics + +This was a fairly simple change, prompted by the initial confusion we had in diagnosing +Smarket's test problems: If you can't tell the difference between slow data generation and +slow tests, your initial guesses about performance may be very misleading! So this change +updated [the statistics reporting system](https://hypothesis.readthedocs.io/en/latest/details.html#statistics) +to report what fraction of time is spent in data generation. If you run tests with statistics +you'll now see a line like the following: + +``` +- Fraction of time spent in data generation: ~ 12% +``` + +A small change, but a very helpful one! This came in in Hypothesis 3.38.4. + +### Health check overhaul + +Hypothesis has had a health check system for a while. Its general goal is to suggest that +you might not want to do that thing you're doing - roughly analogous to compiler warnings. +It's useful for guiding you into correct use of Hypothesis and helping you avoid things that +might degrade the quality of your testing. + +It's historically had some problems: In particular the main source of health checks did not +actually run your tests! They just run the data generation with the test stubbed out. +This meant that you could very easily accidentally bypass the health checks by e.g. +using inline data generation, or doing your filtering with assume. + +It also had the problem that it wasn't running the real data generation algorithm but instead +an approximation to it. +This meant that things that would work fine in practice would sometimes fail health checks. + +This piece of work was an overhaul of the health check system to solve these problems and to expand the scope of the problems it could find. + +It ended up working very well. So well in fact that it found some problems in Hypothesis's built in library of strategies! + +It was split across a number of releases: + +* In 3.37.0 we deprecated a number of existing health checks that no longer did anything useful. +* In 3.38.0 I overhauled the health check system to be based on actual test execution, solving the existing limitations of it. +* In 3.39.0 I added a new health check that tests whether the smallest example of the test was too large to allow reasonable testing - + accidentally generating very large examples being a common source of performance bugs. + +The health check in 3.39.0 turned out to catch a major problem in Hypothesis's handling of blacklist\_characters and some +regular expression constructs, so prior to that we had to release 3.38.8 to fix those! + +Over all I'm much happier with the new health check system and think it does a much better job of shaping user behaviour to get better results out of Hypothesis. + +### Printing reproduction steps + +Historically output from Hypothesis has looked something like this: + +``` +Falsifying example: test_is_minimal(ls=[0], v=1) +``` + +Or, if you had a failed health check, like the following: + +``` +hypothesis.errors.FailedHealthCheck: It looks like your strategy is filtering out a lot of data. Health check found 50 filtered examples but only +0 good ones. This will make your tests much slower, and also will probably distort the data generation quite a lot. You should adapt your strategy + to filter less. This can also be caused by a low max_leaves parameter in recursive() calls. + +See https://hypothesis.readthedocs.io/en/latest/healthchecks.html for more information about this. If you want to disable just this health check, +add HealthCheck.filter_too_much to the suppress_health_check settings for this test. +``` + +This is fine if you're running the tests locally, but if your failure is on CI this can be difficult to reproduce. +If you got a falsifying example, you're only able to reproduce it if all of your arguments have sensible reprs (which may not be the case even if you restrict yourself to Hypothesis's built in strategies - e.g. using inline data generation prevents it!). +If you got a health check failure, there's nothing that helps you reproduce it at all! + +So the proposed feature for this was to print out the random seed that produced this: + +``` +You can add @seed(302934307671667531413257853548643485645) to this test or run pytest with --hypothesis-seed=302934307671667531413257853548643485645 to reproduce this failure. +``` + +This was a great idea, and seemed to work out pretty well when we introduced it in 3.30.0, but on heavier use in the wild turned out to have some fairly major problems! + +The big issue is that in order to reproduce Hypothesis's behaviour on a given run you need to know not +just the random seed that got you there, but also the state of Hypothesis's example database! +Hypothesis [maintains a cache of many of the previously run examples](https://hypothesis.readthedocs.io/en/latest/database.html), +and uses it to inform the testing by replaying test cases that e.g. failed the last time they were run, +or covered some hard to reach line in the code. +Even for examples that don't come from the database, Hypothesis under the hood is a mutation based +fuzzer, so all the examples it finds will depend on the examples it loaded. + +The initial solution to this (3.40.0) was just to turn off seed printing when its output would be misleading. +This worked, but was fairly non-ideal even just for Smarkets - they *do* use the database in their CI, so this would result in a lot of failures to print. + +After some discussion, I decided that given that the feature wasn't nearly as useful as intended, +so I threw in an extra freebie feature to make up the gap in functionality: +[@reproduce\_failure](https://hypothesis.readthedocs.io/en/latest/reproducing.html#reproducing-an-example-with-with-reproduce-failure). +This uses Hypothesis's internal format to replicate the functionality of the database in a way that is easy to copy and paste into your code. +It took some careful designing to make it usable - my big concern was that people would leave this in their code, blocking future upgrades to Hypothesis - but in the end I'm *reasonably* happy with the result. + +As a bonus, the work here allowed me to sort out one big concern about seed printing: We still needed a way to reproduce health check failures when the database was being used. +The solution was in the end easy: We just don't use the examples from the database in the part where the main health checks are running. +This still leaves a few health checks which could theoretically be hard to reproduce (the main one is the hung test health check, but that one tends to reproduce fairly reliably on any seed if you have deadlines on). + +So this leaves us with a state where health check failures will suggest `@seed` and example failures will suggest `@reproduce_failure` (where necessary. The linked documentation spells this out in more detail). + +### Ten releases later + +In the end the Smarkets work came to a total of exactly 10 releases, some larger than other. + +The end result has been very beneficial, and not just to Smarkets! +I've had several users report back improvements to their tests as a result of the new health checks, +and I've personally found the `@reproduce_failure` feature remarkably useful. + +I'm very happy to have done this work, and am grateful to Smarkets for funding it. + +I think this sort of thing where commercial users fund the "boring" features that are very useful for people using the tool at scale but maintainers are unlikely to work on under their own initiative is a +very good one, and I hope we'll do more of it in future. + +As a bonus, Smarkets kindly agreed to put online the talk I gave them about Hypothesis (largely intended +to raise awareness of it and property-based testing among their teams who aren't using it yet). +If you want to learn more about some of the philosophy and practice behind using this sort of +testing, or want something to send to people who aren't convinced yet, you can watch it +[here](https://smarketshq.com/a-talk-on-hypothesis-e7182b95ced1). diff --git a/HypothesisWorks.github.io/_posts/2018-02-27-continuous-releases.md b/HypothesisWorks.github.io/_posts/2018-02-27-continuous-releases.md new file mode 100644 index 0000000000..96564d7ed2 --- /dev/null +++ b/HypothesisWorks.github.io/_posts/2018-02-27-continuous-releases.md @@ -0,0 +1,121 @@ +--- +layout: post +tags: development-process +date: 2018-02-27 07:00 +title: The Hypothesis continuous release process +published: true +author: alexwlchan +--- + +If you watch [the Hypothesis changelog][changelog], you'll notice the rate of releases sped up dramatically in 2017. +We released over a hundred different versions, sometimes multiple times a day. + +This is all thanks to our continuous release process. +We've completely automated the process of releasing, so every pull request that changes code gets a new release, without any human input. +In this post, I'll explain how our continuous releases work, and why we find it so useful. + +[changelog]: https://hypothesis.readthedocs.io/en/latest/changes.html + + + +## How it works + +In the past, Hypothesis was released manually. +Somebody had to write a changelog, tag a new release on GitHub, and run some manual pip commands to publish a new version to PyPI -- and only David had the credentials for the latter. + +This meant that releases were infrequent, and features spent a long time in master before they were available to `pip install`. +The pace of development picked up in 2017 -- partly as new maintainers arrived, and partly groundwork for [David's upcoming (now started) PhD][phd] -- and we wanted to be able to release more frequently. +We decided to automate the entire release process. + +Now, when you create a pull request that changes the Hypothesis code -- anything that gets installed by pip -- you have to include a `RELEASE.rst` file which describes your change. +Here's an example from [a recent pull request][recent]: + + RELEASE_TYPE: patch + + This release changes the way in which Hypothesis tries to shrink the size of + examples. It probably won't have much impact, but might make shrinking faster + in some cases. It is unlikely but not impossible that it will change the + resulting examples. + +The first line says whether this is a major, minor, or patch release (using [semantic versioning][semver]). +The rest is a description of the changes in your patch. + +We have a test in CI that checks for this file -- any change to the core code needs a release file, even [fixing a typo][typo]. +If you need a release file but haven't written one, the tests fail and your pull request won't be merged. + +Sometimes we write a release file even if there aren't changes to the core code, but we think it's worth a release anyway. +For example, changes to the installation code in `setup.py`, or larger changes to our test code for the benefit of downstream packagers. + +Once you've written a release file and the pull request is merged into master, and after all the other tests have passed, our CI uses this file to create a new release. + +First, it works out the new version number, and updates it in [version.py][version.py]. +Then it copies the release description into the changelog, including the new version number and the current date. +For example: + + -------------------- + 3.44.25 - 2018-02-05 + -------------------- + + This release changes the way in which Hypothesis tries to shrink the size of + examples. It probably won't have much impact, but might make shrinking faster + in some cases. It is unlikely but not impossible that it will change the + resulting examples. + +These two changes are saved as a new commit, and that commit gets tagged as the new release. +The tag and the commit are pushed to GitHub, and then CI builds a new package and publishes it to PyPI. + +So with no very little extra work, every code change triggers a new release, and it's usually available within half an hour of merging the pull request. + +This exact system might not scale to larger teams. +In particular, you can't merge new features until the code in master has been released -- you get conflicts around `RELEASE.rst` -- so you can only merge one pull request at a time. +And in Hypothesis, we never backport bugfixes to old major or minor releases -- you'd need some changes to support that. + +But Hypothesis only has one full-time contributor, and everybody else works on it in their free time, we don't create patches fast enough for this to be a problem. +For us, it works exceptionally well. + +[phd]: http://www.drmaciver.com/2017/04/life-changes-announcement-academia-edition/ +[recent]: https://github.com/HypothesisWorks/hypothesis-python/pull/1101 +[semver]: https://semver.org/ +[typo]: https://github.com/HypothesisWorks/hypothesis-python/pull/1069 +[version.py]: https://github.com/HypothesisWorks/hypothesis-python/blob/master/src/hypothesis/version.py + +## Why bother? + +Moving to continuous releases has been amazing. + +The big benefit is that nobody has to do manual releases any more. +Before we had this system, changelogs had to be assembled and written by hand, which meant reading the commit log since the last release. +This is both boring and prone to error -- in the past, a release might contain multiple changes, and it was easy to overlook or forget something in the changelog. +No more! + +Another benefit is that our releases happen much more quickly. +Every patch is available as soon as our tests confirm it's okay, not when somebody remembers to do a release. +If something's been merged, it's either available for download, or it will be very shortly. + +Releasing more often means each individual release is much smaller, which makes it much easier to find the source of bugs or regressions. +If somebody finds a bug, we can trace it to a specific release (and corresponding pull request), and there's a relatively small amount of code to inspect. + +Automation also makes our release process more reliable. +Manual steps have scope for error, and we've had a few dodgy releases in the past. +This process has cut over 100 releases near flawlessly. + +Finally, every contributor gets to make a release. +If you submit a patch that gets accepted, your change is available immediately, and it's entirely your work. +This may less of tangible benefit, but it gives off nice fuzzy feelings, especially if it's your first patch. +(Speaking of which, we're always looking [for new contributors][contributors]!) + +[contributors]: https://github.com/HypothesisWorks/hypothesis-python/blob/master/CONTRIBUTING.rst + +## I'm ruined for everything else + +I've become used to code being available almost immediately after it's merged into master -- which isn't true for the vast majority of projects. +When I go to a repo with a bug report, see that a bugfix was merged two weeks ago, but there's yet to be a new release, it's hard not to feel a little impatient. + +I've started using this in my other repos -- both these scripts exactly, and derivatives of the same idea. + +If you'd like to try this yourself (and I'd really encourage you to do so!), all the scripts for this process are under the same MPL license as Hypothesis itself. +Look in the [scripts directory][scripts] of the main repo. +In particular, `check-release-file.py` looks for a release note on pull requests, and `deploy.py` is what actually cuts the release. +The code will probably need tweaking for your repo (it's closely based on the Hypothesis repo), but hopefully it provides a useful starting point. + +[scripts]: https://github.com/HypothesisWorks/hypothesis-python/tree/master/scripts diff --git a/HypothesisWorks.github.io/articles/alternatives/feed/index.xml b/HypothesisWorks.github.io/articles/alternatives/feed/index.xml new file mode 100644 index 0000000000..7772bcc5ff --- /dev/null +++ b/HypothesisWorks.github.io/articles/alternatives/feed/index.xml @@ -0,0 +1,4 @@ +--- +layout: blog_feed +tag: alternatives +--- diff --git a/HypothesisWorks.github.io/articles/alternatives/index.md b/HypothesisWorks.github.io/articles/alternatives/index.md new file mode 100644 index 0000000000..9f21c1c719 --- /dev/null +++ b/HypothesisWorks.github.io/articles/alternatives/index.md @@ -0,0 +1,14 @@ +--- +layout: blog_listing +date: 2015-04-15 15:00 +tag: alternatives +--- + +There are plenty of other implementations of property based testing! + +We think Hypothesis is the future, and by far the best in the languages that we've +done serious implementations for, but some of the others are definitely also worth +paying attention to. + +So we have. These are articles we've written taking a look at other implementations +of property based testing and related concepts. diff --git a/HypothesisWorks.github.io/articles/details/feed/index.xml b/HypothesisWorks.github.io/articles/details/feed/index.xml new file mode 100644 index 0000000000..7438ea30f5 --- /dev/null +++ b/HypothesisWorks.github.io/articles/details/feed/index.xml @@ -0,0 +1,4 @@ +--- +layout: blog_feed +tag: details +--- diff --git a/HypothesisWorks.github.io/articles/details/index.md b/HypothesisWorks.github.io/articles/details/index.md new file mode 100644 index 0000000000..9fbd2333d5 --- /dev/null +++ b/HypothesisWorks.github.io/articles/details/index.md @@ -0,0 +1,12 @@ +--- +layout: blog_listing +date: 2015-04-15 15:00 +tag: details +--- + +These are articles centered on detailed understanding of a particular aspect +of how to use Hypothesis. They're not in depth looks at the internals, but +are focused on practical questions of how to use it. + +If you're new to Hypothesis we recommend skipping this section for now and +checking out [the intro section](/articles/intro) instead. diff --git a/HypothesisWorks.github.io/articles/development-process/feed/index.xml b/HypothesisWorks.github.io/articles/development-process/feed/index.xml new file mode 100644 index 0000000000..5338caa279 --- /dev/null +++ b/HypothesisWorks.github.io/articles/development-process/feed/index.xml @@ -0,0 +1,4 @@ +--- +layout: blog_feed +tag: development-process +--- diff --git a/HypothesisWorks.github.io/articles/development-process/index.md b/HypothesisWorks.github.io/articles/development-process/index.md new file mode 100644 index 0000000000..99be1a4290 --- /dev/null +++ b/HypothesisWorks.github.io/articles/development-process/index.md @@ -0,0 +1,5 @@ +--- +layout: blog_listing +date: 2015-04-15 15:00 +tag: development-process +--- diff --git a/HypothesisWorks.github.io/articles/example/feed/index.xml b/HypothesisWorks.github.io/articles/example/feed/index.xml new file mode 100644 index 0000000000..4c632c4ee1 --- /dev/null +++ b/HypothesisWorks.github.io/articles/example/feed/index.xml @@ -0,0 +1,4 @@ +--- +layout: blog_feed +tag: example +--- diff --git a/HypothesisWorks.github.io/articles/example/index.md b/HypothesisWorks.github.io/articles/example/index.md new file mode 100644 index 0000000000..ea572839c4 --- /dev/null +++ b/HypothesisWorks.github.io/articles/example/index.md @@ -0,0 +1,7 @@ +--- +layout: blog_listing +date: 2015-04-15 15:00 +tag: example +------------ + +This is a collection of worked examples of using Hypothesis. \ No newline at end of file diff --git a/HypothesisWorks.github.io/articles/faq/feed/index.xml b/HypothesisWorks.github.io/articles/faq/feed/index.xml new file mode 100644 index 0000000000..043e50f745 --- /dev/null +++ b/HypothesisWorks.github.io/articles/faq/feed/index.xml @@ -0,0 +1,4 @@ +--- +layout: blog_feed +tag: faq +--- diff --git a/HypothesisWorks.github.io/articles/faq/index.md b/HypothesisWorks.github.io/articles/faq/index.md new file mode 100644 index 0000000000..374927d34d --- /dev/null +++ b/HypothesisWorks.github.io/articles/faq/index.md @@ -0,0 +1,7 @@ +--- +layout: blog_listing +date: 2015-04-15 15:00 +tag: faq +--- + +These are short articles that answer common questions about Hypothesis. diff --git a/HypothesisWorks.github.io/articles/feed/index.xml b/HypothesisWorks.github.io/articles/feed/index.xml new file mode 100644 index 0000000000..6c5366bfe0 --- /dev/null +++ b/HypothesisWorks.github.io/articles/feed/index.xml @@ -0,0 +1,3 @@ +--- +layout: blog_feed +--- diff --git a/HypothesisWorks.github.io/articles/index.md b/HypothesisWorks.github.io/articles/index.md new file mode 100644 index 0000000000..a9cae17016 --- /dev/null +++ b/HypothesisWorks.github.io/articles/index.md @@ -0,0 +1,21 @@ +--- +layout: blog_listing +date: 2015-04-15 15:00 +--- + +We write a lot, about Hypothesis in particular but also about software and testing in general. +Here you can find all of that. + +If you wanted something specific, you can also browse broken down by tag. + +On [the technical front](/articles/technical) we have: + +* [General introductory articles about Hypothesis](/articles/intro/) +* [In depth dives into specific questions about how to use Hypothesis](/articles/details/) +* [Articles using the Python version of Hypothesis](/articles/python/) +* [Articles about alternatives to Hypothesis](/articles/alternatives/) + +And [more generally](/articles/non-technical) we have: + +* [Articles about how we think software should be written](/articles/writing-good-software/) +* [Articles about our philosophical and ethical principles](/articles/principles/) diff --git a/HypothesisWorks.github.io/articles/intro/feed/index.xml b/HypothesisWorks.github.io/articles/intro/feed/index.xml new file mode 100644 index 0000000000..2e0456c5da --- /dev/null +++ b/HypothesisWorks.github.io/articles/intro/feed/index.xml @@ -0,0 +1,4 @@ +--- +layout: blog_feed +tag: intro +--- diff --git a/HypothesisWorks.github.io/articles/intro/index.md b/HypothesisWorks.github.io/articles/intro/index.md new file mode 100644 index 0000000000..59c1ce062b --- /dev/null +++ b/HypothesisWorks.github.io/articles/intro/index.md @@ -0,0 +1,22 @@ +--- +layout: blog_listing +date: 2015-04-15 15:00 +tag: intro +--- + +These are a collection of introductory articles that can help you to learn how to use Hypothesis +effectively. You don't have to read all of these to get started! Feel free to dip in and out as +the mood takes you. + +We recommend the following reading order: + +1. [What is Hypothesis?](/articles/what-is-hypothesis/) +2. [Getting started with Hypothesis](/articles/getting-started-with-hypothesis/) +3. [Evolving toward property-based testing with Hypothesis](/articles/incremental-property-based-testing/) +4. [Generating the right data](/articles/generating-the-right-data/) +5. [Testing performance optimizations](/articles/testing-performance-optimizations/) +6. [The Encode/Decode Invariant](/articles/encode-decode-invariant) +7. [Rule Based Stateful Testing](/articles/rule-based-stateful-testing) + +After that, either just experiment with Hypothesis on your own, with [help from the documentation](https://hypothesis.readthedocs.io), +or check out some of [the other articles here](/articles/) for more inspiration. diff --git a/HypothesisWorks.github.io/articles/news/feed/index.xml b/HypothesisWorks.github.io/articles/news/feed/index.xml new file mode 100644 index 0000000000..c0370d6849 --- /dev/null +++ b/HypothesisWorks.github.io/articles/news/feed/index.xml @@ -0,0 +1,4 @@ +--- +layout: blog_feed +tag: news +--- diff --git a/HypothesisWorks.github.io/articles/news/index.md b/HypothesisWorks.github.io/articles/news/index.md new file mode 100644 index 0000000000..ee9cdaa919 --- /dev/null +++ b/HypothesisWorks.github.io/articles/news/index.md @@ -0,0 +1,8 @@ +--- +layout: blog_listing +date: 2015-04-15 15:00 +tag: news +--- + +This tag simply covers news and announcements - new products, new developments, +upcoming events, etc. diff --git a/HypothesisWorks.github.io/articles/non-technical/feed/index.xml b/HypothesisWorks.github.io/articles/non-technical/feed/index.xml new file mode 100644 index 0000000000..d20482794e --- /dev/null +++ b/HypothesisWorks.github.io/articles/non-technical/feed/index.xml @@ -0,0 +1,4 @@ +--- +layout: blog_feed +tag: non-technical +--- diff --git a/HypothesisWorks.github.io/articles/non-technical/index.md b/HypothesisWorks.github.io/articles/non-technical/index.md new file mode 100644 index 0000000000..0677cec2a4 --- /dev/null +++ b/HypothesisWorks.github.io/articles/non-technical/index.md @@ -0,0 +1,9 @@ +--- +layout: blog_listing +date: 2015-04-15 15:00 +tag: non-technical +--- + +These are articles we think are of general interest to anyone who is involved at any +point in the process of making software. Or maybe even more broadly! They don't assume +any technical background to speak of. diff --git a/HypothesisWorks.github.io/articles/principles/feed/index.xml b/HypothesisWorks.github.io/articles/principles/feed/index.xml new file mode 100644 index 0000000000..96161fb514 --- /dev/null +++ b/HypothesisWorks.github.io/articles/principles/feed/index.xml @@ -0,0 +1,4 @@ +--- +layout: blog_feed +tag: principles +--- diff --git a/HypothesisWorks.github.io/articles/principles/index.md b/HypothesisWorks.github.io/articles/principles/index.md new file mode 100644 index 0000000000..8b95e02afa --- /dev/null +++ b/HypothesisWorks.github.io/articles/principles/index.md @@ -0,0 +1,10 @@ +--- +layout: blog_listing +date: 2015-04-15 15:00 +tag: principles +--- + +You don't go into open source testing tools if you're just in it for the money. + +We have principles, philosophical and moral, and we will talk about them from +time to time. Under this tag. diff --git a/HypothesisWorks.github.io/articles/properties/feed/index.xml b/HypothesisWorks.github.io/articles/properties/feed/index.xml new file mode 100644 index 0000000000..c9191729ce --- /dev/null +++ b/HypothesisWorks.github.io/articles/properties/feed/index.xml @@ -0,0 +1,4 @@ +--- +layout: blog_feed +tag: properties +--- diff --git a/HypothesisWorks.github.io/articles/properties/index.md b/HypothesisWorks.github.io/articles/properties/index.md new file mode 100644 index 0000000000..0bb325cf85 --- /dev/null +++ b/HypothesisWorks.github.io/articles/properties/index.md @@ -0,0 +1,10 @@ +--- +layout: blog_listing +date: 2015-04-15 15:00 +tag: properties +--- + +A collection of articles about interesting properties to look for in the software you're testing. + +If you're casting around trying to figure out how to use Hypothesis better, or you have a particular +piece of software, this is a good place to look for inspiration. diff --git a/HypothesisWorks.github.io/articles/python/feed/index.xml b/HypothesisWorks.github.io/articles/python/feed/index.xml new file mode 100644 index 0000000000..b303146c5f --- /dev/null +++ b/HypothesisWorks.github.io/articles/python/feed/index.xml @@ -0,0 +1,4 @@ +--- +layout: blog_feed +tag: python +--- diff --git a/HypothesisWorks.github.io/articles/python/index.md b/HypothesisWorks.github.io/articles/python/index.md new file mode 100644 index 0000000000..f47b473213 --- /dev/null +++ b/HypothesisWorks.github.io/articles/python/index.md @@ -0,0 +1,8 @@ +--- +layout: blog_listing +date: 2015-04-15 15:00 +tag: python +--- + +Articles that *use* the Python version of Hypothesis. Many of these will illustrate +more widely applicable principles. diff --git a/HypothesisWorks.github.io/articles/technical/feed/index.xml b/HypothesisWorks.github.io/articles/technical/feed/index.xml new file mode 100644 index 0000000000..2c243b1bf8 --- /dev/null +++ b/HypothesisWorks.github.io/articles/technical/feed/index.xml @@ -0,0 +1,4 @@ +--- +layout: blog_feed +tag: technical +--- diff --git a/HypothesisWorks.github.io/articles/technical/index.md b/HypothesisWorks.github.io/articles/technical/index.md new file mode 100644 index 0000000000..544863ebb7 --- /dev/null +++ b/HypothesisWorks.github.io/articles/technical/index.md @@ -0,0 +1,9 @@ +--- +layout: blog_listing +date: 2015-04-15 15:00 +tag: technical +--- + +These are articles that are primarily of interest to people who are actually going +to want to write code using Hypothesis. You're welcome to read it anyway if you're +not, but it might not be your thing. diff --git a/HypothesisWorks.github.io/articles/writing-good-software/feed/index.xml b/HypothesisWorks.github.io/articles/writing-good-software/feed/index.xml new file mode 100644 index 0000000000..12bd4b83f2 --- /dev/null +++ b/HypothesisWorks.github.io/articles/writing-good-software/feed/index.xml @@ -0,0 +1,4 @@ +--- +layout: blog_feed +tag: writing-good-software +--- diff --git a/HypothesisWorks.github.io/articles/writing-good-software/index.md b/HypothesisWorks.github.io/articles/writing-good-software/index.md new file mode 100644 index 0000000000..677b631897 --- /dev/null +++ b/HypothesisWorks.github.io/articles/writing-good-software/index.md @@ -0,0 +1,8 @@ +--- +layout: blog_listing +date: 2015-04-15 15:00 +tag: writing-good-software +--- + +We are opinionated about how software should be written. +Here we've collected some of our thoughts on the subject. diff --git a/HypothesisWorks.github.io/assets/css/jekyll-github.css b/HypothesisWorks.github.io/assets/css/jekyll-github.css new file mode 100644 index 0000000000..92fe8d1419 --- /dev/null +++ b/HypothesisWorks.github.io/assets/css/jekyll-github.css @@ -0,0 +1,68 @@ +/* + * GitHub style for Pygments syntax highlighter, for use with Jekyll + * Courtesy of GitHub.com + */ + +.highlight pre, pre, .highlight .hll { background-color: #f8f8f8; border: 1px solid #ccc; padding: 6px 10px; border-radius: 3px; } +.highlight .c { color: #999988; font-style: italic; } +.highlight .err { color: #a61717; background-color: #e3d2d2; } +.highlight .k { font-weight: bold; } +.highlight .o { font-weight: bold; } +.highlight .cm { color: #999988; font-style: italic; } +.highlight .cp { color: #999999; font-weight: bold; } +.highlight .c1 { color: #999988; font-style: italic; } +.highlight .cs { color: #999999; font-weight: bold; font-style: italic; } +.highlight .gd { color: #000000; background-color: #ffdddd; } +.highlight .gd .x { color: #000000; background-color: #ffaaaa; } +.highlight .ge { font-style: italic; } +.highlight .gr { color: #aa0000; } +.highlight .gh { color: #999999; } +.highlight .gi { color: #000000; background-color: #ddffdd; } +.highlight .gi .x { color: #000000; background-color: #aaffaa; } +.highlight .go { color: #888888; } +.highlight .gp { color: #555555; } +.highlight .gs { font-weight: bold; } +.highlight .gu { color: #800080; font-weight: bold; } +.highlight .gt { color: #aa0000; } +.highlight .kc { font-weight: bold; } +.highlight .kd { font-weight: bold; } +.highlight .kn { font-weight: bold; } +.highlight .kp { font-weight: bold; } +.highlight .kr { font-weight: bold; } +.highlight .kt { color: #445588; font-weight: bold; } +.highlight .m { color: #009999; } +.highlight .s { color: #dd1144; } +.highlight .n { color: #333333; } +.highlight .na { color: teal; } +.highlight .nb { color: #0086b3; } +.highlight .nc { color: #445588; font-weight: bold; } +.highlight .no { color: teal; } +.highlight .ni { color: purple; } +.highlight .ne { color: #990000; font-weight: bold; } +.highlight .nf { color: #990000; font-weight: bold; } +.highlight .nn { color: #555555; } +.highlight .nt { color: navy; } +.highlight .nv { color: teal; } +.highlight .ow { font-weight: bold; } +.highlight .w { color: #bbbbbb; } +.highlight .mf { color: #009999; } +.highlight .mh { color: #009999; } +.highlight .mi { color: #009999; } +.highlight .mo { color: #009999; } +.highlight .sb { color: #dd1144; } +.highlight .sc { color: #dd1144; } +.highlight .sd { color: #dd1144; } +.highlight .s2 { color: #dd1144; } +.highlight .se { color: #dd1144; } +.highlight .sh { color: #dd1144; } +.highlight .si { color: #dd1144; } +.highlight .sx { color: #dd1144; } +.highlight .sr { color: #009926; } +.highlight .s1 { color: #dd1144; } +.highlight .ss { color: #990073; } +.highlight .bp { color: #999999; } +.highlight .vc { color: teal; } +.highlight .vg { color: teal; } +.highlight .vi { color: teal; } +.highlight .il { color: #009999; } +.highlight .gc { color: #999; background-color: #EAF2F5; } diff --git a/HypothesisWorks.github.io/assets/css/site.css b/HypothesisWorks.github.io/assets/css/site.css new file mode 100644 index 0000000000..00de1da66f --- /dev/null +++ b/HypothesisWorks.github.io/assets/css/site.css @@ -0,0 +1,272 @@ +@import url(https://fonts.googleapis.com/css?family=Lato:400,700); + +/* global settings */ +body { + font-family: 'Lato', 'sans-serif'; + padding-bottom:60px; + padding-top:50px; +} + +.page-header{ + margin: 10px 0; + text-align:center; + padding-bottom:25px; +} +.page-header h1{ + margin: 0; +} + + +/*navbar stuff*/ +.name-holder{ + margin-left:10px; +} + + +/*Primary navigation bar*/ + +.navbar .navbar-brand { + padding: 0px; +} +.navbar .navbar-brand img { + height: 51px;/* because the navbar has a height of 50px and 1px border bottom*/ +} + + +body{ + font-size: 18px; +} + +p { + margin: 0.5em auto 0.75em auto; +} + +h4, h3{ + margin: 1em auto 0.5em auto; +} + +h4 { + font-size: 20px; +} + +h3 { + font-size: 25px; +} + + +h2 { + font-size: 30px; +} + +.navbar-right li{ + font-size: 20px; + text-align: center; +} +@media (max-width: 992px) { + #main-menu li{ + font-size: 15px; + } + #main-menu li a { + padding-left: 10px; + padding-right: 10px; + } +} + + +/*makes an anchor inactive(not clickable)*/ + +.inactive-link { + pointer-events: none; + cursor: default; +} + +/*blog stuff */ +.blog-short{ + border-bottom:1px solid #ddd; + padding-bottom:30px; + display: table; + +} +.blog-short .excerpt{ + min-height:100px; +} +.margin10{margin-bottom:10px; margin-right:10px;} + + +/*footer*/ +.footer { + background-color: #f5f5f5; + bottom: 0; + min-height: 10px; + position: fixed; + width: 100%; + font-weight: bold; +} + +/* social buttons */ +/*social buttons */ +.social-btn-holder{ + padding:10px; + margin-top:5px; + margin-bottom:5px; +} +.social-btn-holder .btn-social{ + font-size:12px; + font-weight:bold; +} + +.btn-social{ + color: white; + opacity:0.9; +} +.btn-social:hover { + color: white; + opacity:1; +} +.btn-facebook { +background-color: #3b5998; +} +.btn-twitter { +background-color: #00aced; +} +.btn-linkedin { +background-color:#0e76a8; +} +.btn-github{ + background-color:#000000; +} +.btn-google { + background-color: #c32f10; +} +.btn-stackoverflow{ + background-color: #D38B28; +} + +.btn-hackerrank{ + background-color: #27AB5B; +} +.btn-soundcloud{ + background-color: #FF7202; +} + + +/* utility classes */ +.no-margin:{ + margin:0; +} +.border-bottom{ + border-bottom: 1px solid #eee; +} +.centered-text{ + text-align: center; +} + + +/* front page stuff */ +.imgcontainer img{ + width: 100%; +} + +div.page{ +} + +#site-title{ + line-height: 50px; + color: #002E62; + font-weight: bold; +} + +.container h1,h2,h3,h4,h5,h6{ + color: #002e62; +} +a{ + color: #0069df; +} +#main-menu a{ + color:#002e62; +} + +.label-tag{ + background-color: #002e62; +} +.label-tag a{ + color: white; +} +.blog-header{ + background-color: #f8f8f8 +} + +blockquote.testimonial { + width: 100%; + display: block; + background: #f9f9f9; + border: none; + margin: 1em 0 1em 0; + font-size: 20px; + padding: 1em; + text-align: left; +} + +blockquote.testimonial footer{ + margin-top: 10px; +} + +blockquote.testimonial p { + margin-top: 0; +} + + +blockquote.testimonial footer{ + text-align: right; +} + +h2 { + font-size: 40px; + text-align: center; + margin: 40px auto 15px auto; +} + +h1 { + font-size: 50px; + text-align: center; + margin: 0px auto 25px auto; +} + +form#tinyletter { + display: block; + width: 100%; + margin: 1.5em 0 0; + padding-right: 1em; + text-align: right; +} + +input#tlemail { + width: 200px; +} + +div.article-listing div.panel { + width: 90%; + margin: 0 auto 10px auto ; +} + +div.article-listing div.panel-heading { + text-align: center; +} + +div.panel div.rss-plug{ + text-align: right; +} + +div.container { + max-width: 50em; +} + +pre { + margin: 1em 2em 0.5em 2em; +} + + +.post-metadata div.author { + font-size: 20px; +} + diff --git a/HypothesisWorks.github.io/assets/js/site.js b/HypothesisWorks.github.io/assets/js/site.js new file mode 100644 index 0000000000..776933d21f --- /dev/null +++ b/HypothesisWorks.github.io/assets/js/site.js @@ -0,0 +1,18 @@ +/* + Contains the site specific js + +*/ + +$(document).ready(function() { + (function() { + if (document.location.hash) { + setTimeout(function() { + window.scrollTo(window.scrollX, window.scrollY - 100); + }, 10); + } + })(); + $(window).on("hashchange", function () { + window.scrollTo(window.scrollX, window.scrollY - 100); + }); + +}); diff --git a/HypothesisWorks.github.io/index.md b/HypothesisWorks.github.io/index.md new file mode 100644 index 0000000000..185e799238 --- /dev/null +++ b/HypothesisWorks.github.io/index.md @@ -0,0 +1,70 @@ +--- +title: Most testing is ineffective +layout: page +date: 2015-04-23 22:03 +--- + +Normal "automated" software testing is surprisingly manual. Every scenario the computer runs, someone had to +write by hand. Hypothesis can fix this. + +Hypothesis is a new generation of tools for automating your testing process. It combines human understanding of +your problem domain with machine intelligence to improve the quality of your testing process while spending +*less* time writing tests. + +Don't believe us? Here's what some of our users have to say: + +
+

+At Lyst we've used it in a wide variety of situations, from testing APIs to machine learning algorithms and in all +cases it's given us a great deal more confidence in that code. +

+ +
+ + +
+When it comes to validating the correctness of your tools, nothing comes close to the thoroughness and power of Hypothesis. + + +
+ + +
+Hypothesis has been brilliant for expanding the coverage of our test cases, and also for making them much easier to read and understand, so we’re sure we’re testing the things we want in the way we want. + +
+ +
+Hypothesis has located real defects in our code which went undetected by traditional test cases, simply because Hypothesis is more relentlessly devious about test case generation than us mere humans! + +
+ +See more at our [testimonials page](/testimonials/). + +## What is Hypothesis? + +Hypothesis is a modern implementation of [property based testing](https://en.wikipedia.org/wiki/QuickCheck), designed from the ground up for mainstream languages. + +Hypothesis runs your tests against a much wider range of scenarios than a human tester could, finding edge cases +in your code that you would otherwise have missed. It then turns them into simple and easy to understand failures +that save you time and money compared to fixing them if they slipped through the cracks and a user had run into +them instead. + +Hypothesis currently has [a fully featured open source Python implementation](https://github.com/HypothesisWorks/hypothesis-python/) and [a proof of concept Java implementation](https://github.com/HypothesisWorks/hypothesis-java) that we are looking for customers to partner with to turn into a finished project. +Plans for C and C++ support are also in the works. + +## How do I use it? + +Hypothesis integrates into your normal testing workflow. Getting started is as simple as installing a library and +writing some code using it - no new services to run, no new test runners to learn. + +Right now only the Python version of Hypothesis is production ready. To get started with it, check out +[the documentation](https://hypothesis.readthedocs.io/en/latest/) or read some of the +[introductory articles here on this site](/articles/intro/). + +Once you've got started, or if you have a large number of people who want to get started all at once, +you may wish to engage [our training services](/training). + +If you still want to know more, sign up to our newsletter to get an email every 1-2 weeks about the latest and greatest Hypothesis developments and how to test your software better. + +
diff --git a/HypothesisWorks.github.io/products/index.md b/HypothesisWorks.github.io/products/index.md new file mode 100644 index 0000000000..94b2edcb41 --- /dev/null +++ b/HypothesisWorks.github.io/products/index.md @@ -0,0 +1,51 @@ +--- +title: Products +layout: page +date: 2015-05-19 15:00 +--- + +## Hypothesis for Python + +This is our current primary focus and the only currently production ready +implementation of the Hypothesis design. + +It features: + +* A full implementation of [property based testing]( + {{site.url}}{% post_url 2016-05-13-what-is-property-based-testing %}) for + Python, including [stateful testing]({{site.url}}{% post_url 2016-04-19-rule-based-stateful-testing %}). +* An extensive library of data generators and tools for writing your own. +* Compatible with py.test, unittest, nose and Django testing, and probably many + others besides. +* Supports CPython and PyPy 3.6 and later (older versions are EoL upstream). +* Open source under the [Mozilla Public License 2.0](https://www.mozilla.org/en-US/MPL/2.0/) + +To use Hypothesis for Python, simply add the *hypothesis* package to your requirements, +or *pip install hypothesis* directly. + +The code is [available on GitHub](https://github.com/HypothesisWorks/hypothesis-python) +and documentation is available [on readthedocs](https://hypothesis.readthedocs.io/). + +## Hypothesis for Java + +Hypothesis for Java is currently a *feasibility prototype* only and is not ready +for production use. We are looking for initial customers to help fund getting it +off the ground. + +As a prototype it currently features: + +* Enough of the core Hypothesis model to be useful. +* Good JUnit integration. +* A small library of data generators. + +The end goal is for Hypothesis for Java to have feature parity with Hypothesis +for Python, and to take advantage of the JVM's excellent concurrency support +to provide parallel testing of your code, but it's not there yet. + +The current prototype is released under the AGPL3 (this is not the intended +license for the full version, which will most likely be Apache licensed) and +is [available on GitHub](https://github.com/HypothesisWorks/hypothesis-java). + +Email us at [hello@hypothesis.works](mailto:hello@hypothesis.works) if you want +to know more about Hypothesis for Java or want to discuss being an early customer +of it. diff --git a/HypothesisWorks.github.io/provision.sh b/HypothesisWorks.github.io/provision.sh new file mode 100644 index 0000000000..a19016ac28 --- /dev/null +++ b/HypothesisWorks.github.io/provision.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + + +set -e -x + +sudo ln -fs /usr/share/zoneinfo/UTC /etc/localtime + +if [ ! "$(command -v node)" ] ; then + # Ugh + curl -sL https://deb.nodesource.com/setup_4.x | sudo -E bash - +fi + +sudo apt-get install -y git libreadline-dev libssl-dev zlib1g-dev build-essential nodejs psmisc + +if [ ! -d ~/.rbenv ]; then + git clone https://github.com/rbenv/rbenv.git ~/.rbenv +fi + +if [ ! -d ~/.rbenv/plugins/ruby-build ]; then + git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build +fi + +cd /vagrant + + +export PATH="$HOME/.rbenv/bin:$PATH" + +eval "$(rbenv init -)" + +rbenv install -s 2.3.0 + +rbenv local 2.3.0 + +if [ ! "$(command -v bundle)" ] ; then + gem install bundler +fi +bundle install + +if [ ! "$(killall bundle 2>/dev/null)" ]; then + sleep 1 + rm -f jekyll.log +fi + +nohup bundle exec jekyll serve -H 0.0.0.0 --force_polling > jekyll.log 2>&1 & + +sleep 1 + +cat jekyll.log diff --git a/HypothesisWorks.github.io/scratch/README.md b/HypothesisWorks.github.io/scratch/README.md new file mode 100644 index 0000000000..bdbdb5600c --- /dev/null +++ b/HypothesisWorks.github.io/scratch/README.md @@ -0,0 +1,5 @@ +# Scratch + +This is a directory for useful example files I wrote in the course of +writing articles. Nothing in here is likely to be particularly useful +to you. \ No newline at end of file diff --git a/HypothesisWorks.github.io/scratch/voting.py b/HypothesisWorks.github.io/scratch/voting.py new file mode 100644 index 0000000000..fbabb40382 --- /dev/null +++ b/HypothesisWorks.github.io/scratch/voting.py @@ -0,0 +1,78 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +from collections import Counter + +from hypothesis import find, strategies as st + + +@st.composite +def election(draw, max_candidates=10): + candidates = list(range(draw(st.integers(2, max_candidates)))) + return draw(st.lists(st.permutations(candidates), min_size=1)) + + +def candidates_for_election(election): + return sorted({c for cs in election for c in cs}) + + +def plurality_winner(election): + counts = Counter(vote[0] for vote in election) + winning_score = max(counts.values()) + winners = [c for c, v in counts.items() if v == winning_score] + if len(winners) > 1: + return None + else: + return winners[0] + + +def irv_winner(election): + candidates = candidates_for_election(election) + while len(candidates) > 1: + scores = Counter() + for vote in election: + for c in vote: + if c in candidates: + scores[c] += 1 + break + losing_score = min(scores[c] for c in candidates) + candidates = [c for c in candidates if scores[c] > losing_score] + if not candidates: + return None + else: + return candidates[0] + + +def differing_without_ties(election): + irv = irv_winner(election) + if irv is None: + return False + plurality = plurality_winner(election) + if plurality is None: + return False + return irv != plurality + + +def is_majority_dominated(election, c): + scores = Counter() + for vote in election: + for d in vote: + if d == c: + break + scores[d] += 1 + return any(score > len(election) / 2 for score in scores.values()) + + +def find_majority_dominated_winner(method): + def test(election): + winner = method(election) + return winner is not None and is_majority_dominated(election, winner) + + return find(election(), test) diff --git a/HypothesisWorks.github.io/services/index.md b/HypothesisWorks.github.io/services/index.md new file mode 100644 index 0000000000..30bf5de3b3 --- /dev/null +++ b/HypothesisWorks.github.io/services/index.md @@ -0,0 +1,47 @@ +--- +title: Services and Support +layout: page +date: 2015-04-23 21:08 +--- + +We provide a range of contracting services in the areas of both development and training to help you make the most out of Hypothesis. There are possibilities for both remote work and on-site engagements. + +For remote engagements, we can accommodate distributed teams by offering training sessions over video conferencing solutions. We can also provide a second set of eyes for pair and team programming efforts, or participate in selective code reviews to ensure you and your team are using Hypothesis optimally. + +## Ports to new languages + +Hypothesis is designed to be easy to port to new languages, but we will rarely start on new ports unless someone pays for the development. So if there's a language you want to use Hypothesis in and you currently can't, hire us to fix that! + +As well as making Hypothesis available in a new language, we usually learn new things about the design space when doing this which tends to produce improvements that get rolled back into other languages and makes the next port that much easier. + +We currently have [a prototype Java port](https://github.com/HypothesisWorks/hypothesis-java) and are actively considering a port to C (which would in turn simplify the port to many other languages), but most languages should be feasible so if you'd like a port to a different one, just ask us. + +This one comes with the added bonus that it will make you very popular! We get a lot of questions about Hypothesis ports from people who can't fund them, and anyone who funds the development of a version of Hypothesis will get mentioned prominently in the README and documentation. + +## Performance tweaks + +Hypothesis performance is currently pretty good, and for most use cases the bottleneck will be your test code rather than Hypothesis. There are, however, pathological cases where it tends to slow down. This is particularly true if you are generating very complex data. + +If you find yourself in this situation, we can help! We'll first try to help you analyze exactly *why* your tests are slow and see if we can help you modify them to be faster. If the problem does turn out to be something we should fix on the Hypothesis end, we can do that too. + +If you're really pushing Hypothesis performance hard you may wish to consider hiring us to complete the C port and then rebuild the version for Python (or your language of choice) on top of it. + +## Feature requests + +As well as performance improvements, we're available for hire for any other specific development on Hypothesis. + +There are a large number of directions that Hypothesis can go in and only so much time in the day. If there's a particular feature you need us to bump something up our priority list, you can hire us to implement it. + +## Custom testing projects + +Although we think anyone can use Hypothesis (even *without* the benefit of [our training](/training/)), sometimes you really just want an expert to do the work, either to help you get started or to give you confidence in the results. If you have a particular piece of software that you really need to be well tested, you can hire us to do that for you. + +## Support contracts + +We can provide support contracts guaranteeing priority to your bug reports and answering your questions when you get stuck. Availability of these is somewhat limited due to capacity constraints, but we are still able to take on new customers. + +## Get in touch! + +If any of the above sound just like what you need, or if there's another Hypothesis related project that doesn't +quite fit, drop us a line at [hello@hypothesis.works](mailto:hello@hypothesis.works) and lets talk +details! diff --git a/HypothesisWorks.github.io/sitemap.xml b/HypothesisWorks.github.io/sitemap.xml new file mode 100644 index 0000000000..493ea5215e --- /dev/null +++ b/HypothesisWorks.github.io/sitemap.xml @@ -0,0 +1,41 @@ +--- +layout: null +sitemap: + exclude: "yes" +--- + + + + {% for post in site.posts %} + {% unless post.published == false %} + + {{ post.url | prepend: site.baseurl}} + {% if post.sitemap.lastmod %} + {{ post.lastmodified | date_to_xmlschema }} + {% elsif post.date %} + {{ post.date | date_to_xmlschema }} + {% else %} + {{ site.time | date_to_xmlschema }} + {% endif %} + weekly + 0.5 + + {% endunless %} + {% endfor %} + {% for page in site.pages %} + {% unless page.sitemap.exclude == "yes" %} + + {{ page.url | prepend: site.baseurl }} + {% if page.lastmodified %} + {{ page.lastmodified | date_to_xmlschema}} + {% elsif page.date %} + {{ page.date | date_to_xmlschema }} + {% else %} + {{ site.time | date_to_xmlschema }} + {% endif %} + weekly + 0.5 + + {% endunless %} + {% endfor %} + diff --git a/HypothesisWorks.github.io/testimonials/index.html b/HypothesisWorks.github.io/testimonials/index.html new file mode 100644 index 0000000000..673c8f2d54 --- /dev/null +++ b/HypothesisWorks.github.io/testimonials/index.html @@ -0,0 +1,159 @@ +--- +title: Testimonials +layout: page +date: 2015-04-25 22:03 +--- + +
+

+Hypothesis is easy to learn and powerful implementation of property based testing, +and generally an invaluable tool. At Lyst we've used it in a wide variety of +situations, from testing APIs to machine learning algorithms and in all +cases it's given us a great deal more confidence in that code. +

+
Alex Stapleton, Lead Backend Engineer at Lyst
+
+ +
+

+Hypothesis is the single most powerful tool in my toolbox for working with +algorithmic code, or any software that produces predictable output from a wide +range of sources. When using it with Priority, Hypothesis consistently found +errors in my assumptions and extremely subtle bugs that would have taken months +of real-world use to locate. In some cases, Hypothesis found subtle deviations +from the correct output of the algorithm that may never have been noticed at +all. +

+

+When it comes to validating the correctness of your tools, nothing comes close +to the thoroughness and power of Hypothesis. +

+ +
+ +
+

Hypothesis has been brilliant for expanding the coverage of our test cases, +and also for making them much easier to read and understand, +so we're sure we're testing the things we want in the way we want.

+ +
+ +
+

+At Sixty North we use Hypothesis for testing +Segpy, an open source Python library for +shifting data between Python data structures and SEG Y files which contain +geophysical data from the seismic reflection surveys used in oil and gas +exploration.

+ +

This is our first experience of property-based testing – as opposed to example-based +testing. Not only are our tests more powerful, they are also much better +explanations of what we expect of the production code. In fact, the tests are much +closer to being specifications. Hypothesis has located real defects in our code +which went undetected by traditional test cases, simply because Hypothesis is more +relentlessly devious about test case generation than us mere humans! We found +Hypothesis particularly beneficial for Segpy because SEG Y is an antiquated format +that uses legacy text encodings (EBCDIC) and even a legacy floating point format +we implemented from scratch in Python.

+ +

+Hypothesis is sure to find a place in most of our future Python codebases and many +existing ones too. +

+ +
+ + +
+

+When I first heard about Hypothesis, I knew I had to include it in my two +open-source Python libraries, natsort +and fastnumbers.

+ +

Quite frankly, +I was a little appalled at the number of bugs and "holes" I found in the code. I can +now say with confidence that my libraries are more robust to "the wild." In +addition, Hypothesis gave me the confidence to expand these libraries to fully +support Unicode input, which I never would have had the stomach for without such +thorough testing capabilities. Thanks! +

+ + + +
+ +
+

+Just found out about this excellent QuickCheck for Python implementation and +ran up a few tests for my bytesize +package last night. Refuted a few hypotheses in the process. +

+ +

+Looking forward to using it with a bunch of other projects as well. +

+ +
+ + +
+

I have written a small library to serialize dicts to MariaDB's dynamic columns binary format, mariadb-dyncol. When I first +developed it, I thought I had tested it really well - there were hundreds of +test cases, some of them even taken from MariaDB's test suite itself. I was +ready to release. +

+ +

+Lucky for me, I tried Hypothesis with David at the PyCon UK sprints. Wow! It +found bug after bug after bug. Even after a first release, I thought of a way +to make the tests do more validation, which revealed a further round of bugs! +Most impressively, Hypothesis found a complicated off-by-one error in a +condition with 4095 versus 4096 bytes of data - something that I would never +have found. +

+

+Long live Hypothesis! (Or at least, property-based testing). +

+ +
+ +
+

+Adopting Hypothesis improved bidict's +test coverage and significantly increased our ability to make changes to +the code with confidence that correct behavior would be preserved. +Thank you, David, for the great testing tool. +

+ +
+ + +
+

+One extremely satisfied user here. Hypothesis is a really solid implementation +of property-based testing, adapted well to Python, and with good features +such as failure-case shrinkers. I first used it on a project where we needed +to verify that a vendor's Python and non-Python implementations of an algorithm +matched, and it found about a dozen cases that previous example-based testing +and code inspections had not. Since then I've been evangelizing for it at our firm. +

+ +
+ +

Your name goes here

+

+Want to add to the list by telling us about your Hypothesis experience? Drop us +an email at testimonials@hypothesis.works +and we'll add it to the list! +

+

+ +

diff --git a/HypothesisWorks.github.io/training/index.md b/HypothesisWorks.github.io/training/index.md new file mode 100644 index 0000000000..ea3d22d660 --- /dev/null +++ b/HypothesisWorks.github.io/training/index.md @@ -0,0 +1,62 @@ +--- +title: Training +layout: page +date: 2015-04-23 21:51 +--- + +## Why should my company get Hypothesis training? + +Hypothesis **makes your testing less labor-intensive and more thorough**. You'll find bugs you never even suspected were there in less time than your existing testing tools and methodology require. + +You don't *need* training to use Hypothesis, if you're not using Hypothesis already, why not just try it out first? + +Hypothesis has [extensive documentation](https://hypothesis.readthedocs.io/en/latest/), and a body of introductory articles are available [right here](/articles/intro/) to get you started. These resources and will help you and your team begin testing with Hypothesis in no time. Give it a spin! + +Now that you *are* using Hypothesis, ask your team two questions: + +1. Is Hypothesis useful? +2. Could we be getting more out of it? + +The answers to both will be a resounding yes. + +If you want to take your usage of Hypothesis to the next level and find +even more bugs with even less work, our training can help. + +The exact nature of the workshop depends on your needs. To find out more about our offerings, read on! + +## What do you need? + +* [I need my team to get better at testing](#i-need-my-team-to-get-better-at-testing) +* [I need my product to be better tested](#i-need-my-product-to-be-better-tested) +* [I need something else](#i-need-something-else-or-something-more) + +### I need my team to get better at testing + +In that case you want our **structured** workshop. + +* One day on-site workshop, up to 10 people. +* Video conferencing format for distributed remote teams available on request. +* Beginner friendly. +* We work through a series of examples illustrating key Hypothesis concepts. +* Attendees come away with a deeper understanding of how Hypothesis can improve their testing. + +Drop us a line at [training@hypothesis.works](mailto:training@hypothesis.works) to find out more. + +### I need my product to be better tested + +In that case you want our **exploratory** workshop. + +* One day on-site workshop, up to 10 people. +* Video conferencing format for distributed remote teams available on request. +* Some experience and familiarity with your software and Hypothesis expected. +* We work as a group to improve the test quality of your software using Hypothesis. +* Attendees come away with a better understanding of how Hypothesis fits into your specific testing needs and processes. +* You get better tested software as soon as the workshop is over! + +Drop us a line at [training@hypothesis.works](mailto:training@hypothesis.works) to find out more. + +### I need something else, or something more + +If you're looking for a customized training experience (e.g. because you want a longer or shorter course, or are looking to dive deeper into a specific topic), we are able to create a customized training that fits your needs and format requirements. + +Drop us a line at [training@hypothesis.works](mailto:training@hypothesis.works) and we can discuss the details. diff --git a/build.sh b/build.sh index ae2bfba80a..6f66cbabec 100755 --- a/build.sh +++ b/build.sh @@ -17,14 +17,14 @@ SCRIPTS="$ROOT/tooling/scripts" # shellcheck source=tooling/scripts/common.sh source "$SCRIPTS/common.sh" -if [ -n "${GITHUB_ACTIONS-}" ] ; then - # We're on GitHub Actions and already set up a suitable Python +if [ -n "${GITHUB_ACTIONS-}" ] || [ -n "${CODESPACES-}" ] ; then + # We're on GitHub Actions or Codespaces and already set up a suitable Python PYTHON=$(command -v python) else # Otherwise, we install it from scratch - # NOTE: keep this version in sync with PYMAIN in tooling - "$SCRIPTS/ensure-python.sh" 3.8.7 - PYTHON=$(pythonloc 3.8.7)/bin/python + # NOTE: tooling keeps this version in sync with ci_version in tooling + "$SCRIPTS/ensure-python.sh" 3.8.15 + PYTHON=$(pythonloc 3.8.15)/bin/python fi TOOL_REQUIREMENTS="$ROOT/requirements/tools.txt" diff --git a/conjecture-rust/.gitignore b/conjecture-rust/.gitignore new file mode 100644 index 0000000000..03314f77b5 --- /dev/null +++ b/conjecture-rust/.gitignore @@ -0,0 +1 @@ +Cargo.lock diff --git a/conjecture-rust/CHANGELOG.md b/conjecture-rust/CHANGELOG.md index 04b83c4dde..04d5337e1e 100644 --- a/conjecture-rust/CHANGELOG.md +++ b/conjecture-rust/CHANGELOG.md @@ -5,7 +5,7 @@ It also positively affects `distributions::good_bitlengths` as it does not have # Conjecture for Rust 0.6.0 (2021-01-27) -This release is required following an unsuccesful deploy of 0.5.0 due to usage of a cargo keyword that was too long. +This release is required following an unsuccessful deploy of 0.5.0 due to usage of a cargo keyword that was too long. # Conjecture for Rust 0.5.0 (2021-01-27) diff --git a/guides/documentation.rst b/guides/documentation.rst index aed344eb31..d8849d6b04 100644 --- a/guides/documentation.rst +++ b/guides/documentation.rst @@ -88,4 +88,4 @@ which should: - finish with a note of thanks from the maintainers: "Thanks to for this bug fix / feature / contribution" (depending on which it is). If this is your first contribution, - don't forget to add yourself to contributors.rst! + don't forget to add yourself to AUTHORS.rst! diff --git a/guides/testing-hypothesis.rst b/guides/testing-hypothesis.rst index b81f3fbf3a..7b5b1b6011 100644 --- a/guides/testing-hypothesis.rst +++ b/guides/testing-hypothesis.rst @@ -37,7 +37,7 @@ a starting point, not the final goal. because when it was found and fixed, someone wrote a test to make sure it couldn't come back! -The ``tests/`` directory has some notes in the README file on where various +The ``hypothesis-python/tests/`` directory has some notes in the README file on where various kinds of tests can be found or added. Go there for the practical stuff, or just ask one of the maintainers for help on a pull request! @@ -48,122 +48,8 @@ Further reading: How `SQLite is tested `_, Dan Luu writes about `fuzz testing `_ and `broken processes `_, among other things. ---------------------------------------- -Setting up a virtualenv to run tests in ---------------------------------------- - -If you want to run individual tests rather than relying on the make tasks -(which you probably will), it's easiest to do this in a virtualenv. - -The following will give you a working virtualenv for running tests in: - -.. code-block:: bash - - pip install virtualenv - python -m virtualenv testing-venv - - # On Windows: testing-venv\Scripts\activate - source testing-venv/bin/activate - - # Can also use pip install -e .[all] to get - # all optional dependencies - pip install -e . - - # Test specific dependencies. - pip install -r requirements/test.in - -Now whenever you want to run tests you can just activate the virtualenv -using ``source testing-venv/bin/activate`` or ``testing-venv\Scripts\activate`` -and all of the dependencies will be available to you and your local copy -of Hypothesis will be on the path (so any edits will be picked up automatically -and you don't need to reinstall it in the local virtualenv). - ------------- Running Tests ------------- -In order to run tests outside of the make/tox/etc set up, you'll need an -environment where Hypothesis is on the path and all of the testing dependencies -are installed. -We recommend doing this inside a virtualenv as described in the previous section. - -All testing is done using `pytest `_, -with a couple of plugins installed. For advanced usage we recommend reading the -pytest documentation, but this section will give you a primer in enough of the -common commands and arguments to get started. - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Selecting Which Files to Run -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The following invocation runs all of the tests in the file -`tests/cover/test_conjecture_engine.py`: - -.. code-block:: - - python -m pytest tests/cover/test_conjecture_engine.py - -If you want to run multiple files you can pass them all as arguments, and if -you pass a directory then it will run all files in that directory. -For example the following runs all the files in `test_conjecture_engine.py` -and `test_slippage.py` - -.. code-block:: - - python -m pytest tests/cover/test_conjecture_engine.py tests/cover/test_slippage.py - -If you were running this in bash (if you're not sure: if you're not on Windows -you probably are) you could also use the syntax: - -.. code-block:: - - python -m pytest tests/cover/test_{conjecture_engine,slippage}.py - -And the following would run all tests under `tests/cover`: - -.. code-block:: - - python -m pytest tests/cover - - -~~~~~~~~~~~ -Test Layout -~~~~~~~~~~~ - -The top level structure of the tests in Hypothesis looks as follows: - -* ``cover`` contains tests that we measure coverage for. This is intended to - be a fairly minimal and fast set of tests that still gives pretty good - confidence in the behaviour of the test suite. It is currently failing at - both "minimal" and "fast", but we're trying to move it back in that - direction. Try not to add tests to this unless they're actually to cover - some specific target. -* ``nocover`` is a general dumping ground for slower tests that aren't needed - to achieve coverage. -* ``quality`` is for expensive tests about the distribution or shrinking of - examples. These will only be run on one Python version. -* The remaining test directories are for testing specific extras modules and - should have the same name. - -As a rule of thumb when writing new tests, they should go in nocover unless -they are for a specific extras module or to deliberately target a particular -line for coverage. In the latter case, prefer fast unit tests over larger and -slower integration tests (we are not currently very good at this). - - -~~~~~~~~~~~~~~~~ -Useful Arguments -~~~~~~~~~~~~~~~~ - -Some useful arguments to pytest include: - -* You can pass ``-n 0`` to turn off ``pytest-xdist``'s parallel test execution. - Sometimes for running just a small number of tests its startup time is longer - than the time it saves (this will vary from system to system), so this can - be helpful if you find yourself waiting on test runners to start a lot. -* You can use ``-k`` to select a subset of tests to run. This matches on substrings - of the test names. For example ``-kfoo`` will only run tests that have "foo" as - a substring of their name. You can also use composite expressions here. - e.g. ``-k'foo and not bar'`` will run anything containing foo that doesn't - also contain bar. `More information on how to select tests to run can be found - in the pytest documentation `__. +Tests are run via ``build.sh``. See ``CONTRIBUTING.rst`` for more details. diff --git a/hypothesis-python/.coveragerc b/hypothesis-python/.coveragerc index a3be214d4d..1deab7668e 100644 --- a/hypothesis-python/.coveragerc +++ b/hypothesis-python/.coveragerc @@ -1,14 +1,15 @@ [run] branch = True -include = - **/.tox/*/lib/*/site-packages/hypothesis/*.py - **/.tox/*/lib/*/site-packages/hypothesis/**/*.py omit = - **/pytestplugin.py - **/strategytests.py - **/compat*.py - **/extra/__init__.py - **/.tox/*/lib/*/site-packages/hypothesis/internal/coverage.py + **/_hypothesis_ftz_detector.py + **/_hypothesis_pytestplugin.py + **/extra/array_api.py + **/extra/cli.py + **/extra/django/*.py + **/extra/ghostwriter.py + **/extra/pytestplugin.py + **/internal/scrutineer.py + **/utils/terminal.py [report] exclude_lines = @@ -19,5 +20,8 @@ exclude_lines = def __copy__ def __deepcopy__ except ImportError: + except ModuleNotFoundError: + if PYPY: if TYPE_CHECKING: + if "\w+" in sys\.modules: assert all\(.+\) diff --git a/hypothesis-python/docs/changes.rst b/hypothesis-python/docs/changes.rst index 572f8fa8e3..587ce8fb23 100644 --- a/hypothesis-python/docs/changes.rst +++ b/hypothesis-python/docs/changes.rst @@ -4,7 +4,7 @@ Changelog This is a record of all past Hypothesis releases and what went into them, in reverse chronological order. All previous releases should still be available -on `PyPI `__. +:pypi:`on PyPI `. Hypothesis 6.x @@ -18,6 +18,2179 @@ Hypothesis 6.x .. include:: ../RELEASE.rst +.. _v6.58.0: + +------------------- +6.58.0 - 2022-11-19 +------------------- + +:func:`~hypothesis.register_random` has used :mod:`weakref` since :ref:`v6.27.1`, +allowing the :class:`~random.Random`-compatible objects to be garbage-collected when +there are no other references remaining in order to avoid memory leaks. +We now raise an error or emit a warning when this seems likely to happen immediately. + +The type annotation of :func:`~hypothesis.register_random` was also widened so that +structural subtypes of :class:`~random.Random` are accepted by static typecheckers. + +.. _v6.57.1: + +------------------- +6.57.1 - 2022-11-14 +------------------- + +This patch updates some internal type annotations and fixes a formatting bug in the +:obj:`~hypothesis.Phase.explain` phase reporting. + +.. _v6.57.0: + +------------------- +6.57.0 - 2022-11-14 +------------------- + +Hypothesis now raises an error if you passed a strategy as the ``alphabet=`` +argument to :func:`~hypothesis.strategies.text`, and it generated something +which was not a length-one string. This has never been supported, we're just +adding explicit validation to catch cases like `this StackOverflow question +`__. + +.. _v6.56.4: + +------------------- +6.56.4 - 2022-10-28 +------------------- + +This patch updates some docs, and depends on :pypi:`exceptiongroup` 1.0.0 +final to avoid a bug in the previous version. + +.. _v6.56.3: + +------------------- +6.56.3 - 2022-10-17 +------------------- + +This patch teaches :func:`~hypothesis.strategies.text` to rewrite a few more +filter predicates (:issue:`3134`). You're unlikely to notice any change. + +.. _v6.56.2: + +------------------- +6.56.2 - 2022-10-10 +------------------- + +This patch updates our vendored `list of top-level domains `__, +which is used by the provisional :func:`~hypothesis.provisional.domains` strategy, and fixes some +incorrect examples in the docs for :func:`~hypothesis.extra.numpy.mutually_broadcastable_shapes`. + +.. _v6.56.1: + +------------------- +6.56.1 - 2022-10-05 +------------------- + +This patch improves the error message when Hypothesis detects "flush to zero" +mode for floating-point: we now report which package(s) enabled this, which +can make debugging much easier. See :issue:`3458` for details. + +.. _v6.56.0: + +------------------- +6.56.0 - 2022-10-02 +------------------- + +This release defines ``__bool__()`` on :class:`~hypothesis.strategies.SearchStrategy`. +It always returns ``True``, like before, but also emits a warning to help with +cases where you intended to draw a value (:issue:`3463`). + +.. _v6.55.0: + +------------------- +6.55.0 - 2022-09-29 +------------------- + +In preparation for `future versions of the Array API standard +`__, +:func:`~hypothesis.extra.array_api.make_strategies_namespace` now accepts an +optional ``api_version`` argument, which determines the version conformed to by +the returned strategies namespace. If ``None``, the version of the passed array +module ``xp`` is inferred. + +This release also introduces :func:`xps.real_dtypes`. This is currently +equivalent to the existing :func:`xps.numeric_dtypes` strategy, but exists +because the latter is expected to include complex numbers in the next version of +the standard. + +.. _v6.54.6: + +------------------- +6.54.6 - 2022-09-18 +------------------- + +If multiple explicit examples (from :func:`@example() `) +raise a Skip exception, for consistency with generated examples we now re-raise +the first instead of collecting them into an ExceptionGroup (:issue:`3453`). + +.. _v6.54.5: + +------------------- +6.54.5 - 2022-09-05 +------------------- + +This patch updates our autoformatting tools, improving our code style without any API changes. + +.. _v6.54.4: + +------------------- +6.54.4 - 2022-08-20 +------------------- + +This patch fixes some type annotations for Python 3.9 and earlier (:issue:`3397`), +and teaches :ref:`explain mode ` about certain locations it should not +bother reporting (:issue:`3439`). + +.. _v6.54.3: + +------------------- +6.54.3 - 2022-08-12 +------------------- + +This patch teaches the Ghostwriter an additional check for function +and class locations that should make it use public APIs more often. + +.. _v6.54.2: + +------------------- +6.54.2 - 2022-08-10 +------------------- + +This patch fixes our workaround for `a pytest bug where the inner exceptions in +an ExceptionGroup are not displayed `__ +(:issue:`3430`). + +.. _v6.54.1: + +------------------- +6.54.1 - 2022-08-02 +------------------- + +This patch makes ``FailedHealthCheck`` and ``DeadlineExceeded`` exceptions +picklable, for compatibility with Django's parallel test runner (:issue:`3426`). + +.. _v6.54.0: + +------------------- +6.54.0 - 2022-08-02 +------------------- + +Reporting of :obj:`multiple failing examples ` +now uses the :pep:`654` `ExceptionGroup `__ type, which is provided by the +:pypi:`exceptiongroup` backport on Python 3.10 and earlier (:issue:`3175`). +``hypothesis.errors.MultipleFailures`` is therefore deprecated. + +Failing examples and other reports are now stored as :pep:`678` exception notes, which +ensures that they will always appear together with the traceback and other information +about their respective error. + +.. _v6.53.0: + +------------------- +6.53.0 - 2022-07-25 +------------------- + +:func:`~hypothesis.extra.django.from_field` now supports ``UsernameField`` +from :mod:`django.contrib.auth.forms`. + +Thanks to Afonso Silva for reporting and working on :issue:`3417`. + +.. _v6.52.4: + +------------------- +6.52.4 - 2022-07-22 +------------------- + +This patch improves the error message when you pass filenames to the :command:`hypothesis write` +CLI, which takes the name of a module or function (e.g. :command:`hypothesis write gzip` or +:command:`hypothesis write package.some_function` rather than :command:`hypothesis write script.py`). + +Thanks to Ed Rogers for implementing this as part of the SciPy 2022 sprints! + +.. _v6.52.3: + +------------------- +6.52.3 - 2022-07-19 +------------------- + +This patch ensures that the warning for non-interactive ``.example()`` +points to your code instead of Hypothesis internals (:issue:`3403`). + +Thanks to @jameslamb for this fix. + +.. _v6.52.2: + +------------------- +6.52.2 - 2022-07-19 +------------------- + +This patch makes :func:`~hypothesis.strategies.integers` more likely to +generate boundary values for large two-sided intervals (:issue:`2942`). + +.. _v6.52.1: + +------------------- +6.52.1 - 2022-07-18 +------------------- + +This patch adds filter rewriting for :func:`math.isfinite`, :func:`math.isinf`, and :func:`math.isnan` +on :func:`~hypothesis.strategies.integers` or :func:`~hypothesis.strategies.floats` (:issue:`2701`). + +Thanks to Sam Clamons at the SciPy Sprints! + +.. _v6.52.0: + +------------------- +6.52.0 - 2022-07-18 +------------------- + +This release adds the ``allow_subnormal`` argument to :func:`~hypothesis.strategies.complex_numbers` by +applying it to each of the real and imaginary parts separately. Closes :issue:`3390`. + +Thanks to Evan Tey for this fix. + +.. _v6.51.0: + +------------------- +6.51.0 - 2022-07-17 +------------------- + +Issue a deprecation warning if a function decorated with +:func:`@composite ` +does not draw any values (:issue:`3384`). + +Thanks to Grzegorz Zieba, Rodrigo Girão, and Thomas Ball for +working on this at the EuroPython sprints! + +.. _v6.50.1: + +------------------- +6.50.1 - 2022-07-09 +------------------- + +This patch improves the error messages in :func:`@example() ` +argument validation following the recent release of :ref:`6.49.1 `. + +.. _v6.50.0: + +------------------- +6.50.0 - 2022-07-09 +------------------- + +This release allows :func:`~hypothesis.extra.numpy.from_dtype` to generate +Unicode strings which cannot be encoded in UTF-8, but are valid in Numpy +arrays (which use UTF-32). + +This logic will only be used with :pypi:`Numpy` >= 1.19, because earlier +versions have `an issue `__ +which led us to revert :ref:`Hypothesis 5.2 ` last time! + +.. _v6.49.1: + +------------------- +6.49.1 - 2022-07-05 +------------------- + +This patch fixes some inconsistency between argument handling for +:func:`@example ` and :func:`@given ` +(:issue:`2706 <2706#issuecomment-1168363177>`). + +.. _v6.49.0: + +------------------- +6.49.0 - 2022-07-04 +------------------- + +This release uses :pep:`612` :obj:`python:typing.ParamSpec` (or the +:pypi:`typing_extensions` backport) to express the first-argument-removing +behaviour of :func:`@st.composite ` +and signature-preservation of :func:`~hypothesis.strategies.functions` +to IDEs, editor plugins, and static type checkers such as :pypi:`mypy`. + +.. _v6.48.3: + +------------------- +6.48.3 - 2022-07-03 +------------------- + +:func:`hypothesis.event` now works for hashable objects which do not +support weakrefs, such as integers and tuples. + +.. _v6.48.2: + +------------------- +6.48.2 - 2022-06-29 +------------------- + +This patch tidies up some internal introspection logic, which will improve +support for positional-only arguments in a future release (:issue:`2706`). + +.. _v6.48.1: + +------------------- +6.48.1 - 2022-06-27 +------------------- + +This release automatically rewrites some simple filters, such as +``floats().filter(lambda x: x >= 10)`` to the more efficient +``floats(min_value=10)``, based on the AST of the predicate. + +We continue to recommend using the efficient form directly wherever +possible, but this should be useful for e.g. :pypi:`pandera` "``Checks``" +where you already have a simple predicate and translating manually +is really annoying. See :issue:`2701` for details. + +.. _v6.48.0: + +------------------- +6.48.0 - 2022-06-27 +------------------- + +This release raises :class:`~unittest.SkipTest` for which never executed any +examples, for example because the :obj:`~hypothesis.settings.phases` setting +excluded the :obj:`~hypothesis.Phase.explicit`, :obj:`~hypothesis.Phase.reuse`, +and :obj:`~hypothesis.Phase.generate` phases. This helps to avoid cases where +broken tests appear to pass, because they didn't actually execute (:issue:`3328`). + +.. _v6.47.5: + +------------------- +6.47.5 - 2022-06-25 +------------------- + +This patch fixes type annotations that had caused the signature of +:func:`@given ` to be partially-unknown to type-checkers for Python +versions before 3.10. + +.. _v6.47.4: + +------------------- +6.47.4 - 2022-06-23 +------------------- + +This patch fixes :func:`~hypothesis.strategies.from_type` on Python 3.11, +following `python/cpython#93754 `__. + +.. _v6.47.3: + +------------------- +6.47.3 - 2022-06-15 +------------------- + +This patch makes the :obj:`~hypothesis.HealthCheck.too_slow` health check more +consistent with long :obj:`~hypothesis.settings.deadline` tests (:issue:`3367`) +and fixes an install issue under :pypi:`pipenv` which was introduced in +:ref:`Hypothesis 6.47.2 ` (:issue:`3374`). + +.. _v6.47.2: + +------------------- +6.47.2 - 2022-06-12 +------------------- + +We now use the :pep:`654` `ExceptionGroup `__ +type - provided by the :pypi:`exceptiongroup` backport on older Pythons - +to ensure that if multiple errors are raised in teardown, they will all propagate. + +.. _v6.47.1: + +------------------- +6.47.1 - 2022-06-10 +------------------- + +Our pretty-printer no longer sorts dictionary keys, since iteration order is +stable in Python 3.7+ and this can affect reproducing examples (:issue:`3370`). +This PR was kindly supported by `Ordina Pythoneers +`__. + +.. _v6.47.0: + +------------------- +6.47.0 - 2022-06-07 +------------------- + +The :doc:`Ghostwritter ` can now write tests for +:obj:`@classmethod ` or :obj:`@staticmethod ` +methods, in addition to the existing support for functions and other callables +(:issue:`3318`). Thanks to Cheuk Ting Ho for the patch. + +.. _v6.46.11: + +-------------------- +6.46.11 - 2022-06-02 +-------------------- + +Mention :func:`hypothesis.strategies.timezones` +in the documentation of :func:`hypothesis.strategies.datetimes` for completeness. + +Thanks to George Macon for this addition. + +.. _v6.46.10: + +-------------------- +6.46.10 - 2022-06-01 +-------------------- + +This release contains some small improvements to our documentation. +Thanks to Felix Divo for his contribution! + +.. _v6.46.9: + +------------------- +6.46.9 - 2022-05-25 +------------------- + +This patch by Adrian Garcia Badaracco adds type annotations +to some private internals (:issue:`3074`). + +.. _v6.46.8: + +------------------- +6.46.8 - 2022-05-25 +------------------- + +This patch by Phillip Schanely makes changes to the +:func:`~hypothesis.strategies.floats` strategy when ``min_value`` or ``max_value`` is +present. +Hypothesis will now be capable of generating every representable value in the bounds. +You may notice that hypothesis is more likely to test values near boundaries, and values +that are very close to zero. + +These changes also support future integrations with symbolic execution tools and fuzzers +(:issue:`3086`). + +.. _v6.46.7: + +------------------- +6.46.7 - 2022-05-19 +------------------- + +This patch updates the type annotations for :func:`~hypothesis.strategies.tuples` and +:func:`~hypothesis.strategies.one_of` so that type-checkers require its arguments to be +positional-only, and so that it no longer fails under pyright-strict mode (see +:issue:`3348`). Additional changes are made to Hypothesis' internals improve pyright +scans. + +.. _v6.46.6: + +------------------- +6.46.6 - 2022-05-18 +------------------- + +This patch by Cheuk Ting Ho adds support for :pep:`655` ``Required`` and ``NotRequired`` as attributes of +:class:`~python:typing.TypedDict` in :func:`~hypothesis.strategies.from_type` (:issue:`3339`). + +.. _v6.46.5: + +------------------- +6.46.5 - 2022-05-15 +------------------- + +This patch fixes :func:`~hypothesis.extra.numpy.from_dtype` with long-precision +floating-point datatypes (typecode ``g``; see :func:`numpy:numpy.typename`). + +.. _v6.46.4: + +------------------- +6.46.4 - 2022-05-15 +------------------- + +This patch improves some error messages for custom signatures +containing invalid parameter names (:issue:`3317`). + +.. _v6.46.3: + +------------------- +6.46.3 - 2022-05-11 +------------------- + +This patch by Cheuk Ting Ho makes it an explicit error to call :func:`~hypothesis.strategies.from_type` +or :func:`~hypothesis.strategies.register_type_strategy` with types that have no runtime instances (:issue:`3280`). + +.. _v6.46.2: + +------------------- +6.46.2 - 2022-05-03 +------------------- + +This patch fixes silently dropping examples when the :func:`@example ` +decorator is applied to itself (:issue:`3319`). This was always a weird pattern, but now it +works. Thanks to Ray Sogata, Keeri Tramm, and Kevin Khuong for working on this patch! + +.. _v6.46.1: + +------------------- +6.46.1 - 2022-05-01 +------------------- + +This patch fixes a rare bug where we could incorrectly treat +:obj:`~python:inspect.Parameter.empty` as a type annotation, +if the callable had an explicitly assigned ``__signature__``. + +.. _v6.46.0: + +------------------- +6.46.0 - 2022-05-01 +------------------- + +This release adds an ``allow_nil`` argument to :func:`~hypothesis.strategies.uuids`, +which you can use to... generate the nil UUID. Thanks to Shlok Gandhi for the patch! + +.. _v6.45.4: + +------------------- +6.45.4 - 2022-05-01 +------------------- + +This patch fixes some missing imports for certain :doc:`Ghostwritten ` +tests. Thanks to Mel Seto for fixing :issue:`3316`. + +.. _v6.45.3: + +------------------- +6.45.3 - 2022-04-30 +------------------- + +This patch teaches :doc:`the Ghostwriter ` to recognize +many more common argument names (:issue:`3311`). + +.. _v6.45.2: + +------------------- +6.45.2 - 2022-04-29 +------------------- + +This patch fixes :issue:`3314`, where Hypothesis would raise an internal +error from :func:`~hypothesis.provisional.domains` or (only on Windows) +from :func:`~hypothesis.strategies.timezones` in some rare circumstances +where the installation was subtly broken. + +Thanks to Munir Abdinur for this contribution. + +.. _v6.45.1: + +------------------- +6.45.1 - 2022-04-27 +------------------- + +This release fixes deprecation warnings about ``sre_compile`` and ``sre_parse`` +imports and ``importlib.resources`` usage when running Hypothesis on Python 3.11. + +Thanks to Florian Bruhin for this contribution. + +.. _v6.45.0: + +------------------- +6.45.0 - 2022-04-22 +------------------- + +This release updates :func:`xps.indices` by introducing an ``allow_newaxis`` +argument, defaulting to ``False``. If ``allow_newaxis=True``, indices can be +generated that add dimensions to arrays, which is achieved by the indexer +containing ``None``. This change is to support a specification change that +expand dimensions via indexing (`data-apis/array-api#408 +`_). + +.. _v6.44.0: + +------------------- +6.44.0 - 2022-04-21 +------------------- + +This release adds a ``names`` argument to :func:`~hypothesis.extra.pandas.indexes` +and :func:`~hypothesis.extra.pandas.series`, so that you can create Pandas +objects with specific or varied names. + +Contributed by Sam Watts. + +.. _v6.43.3: + +------------------- +6.43.3 - 2022-04-18 +------------------- + +This patch updates the type annotations for :func:`@given ` +so that type-checkers will warn on mixed positional and keyword arguments, +as well as fixing :issue:`3296`. + +.. _v6.43.2: + +------------------- +6.43.2 - 2022-04-16 +------------------- + +Fixed a type annotation for ``pyright --strict`` (:issue:`3287`). + +.. _v6.43.1: + +------------------- +6.43.1 - 2022-04-13 +------------------- + +This patch makes it an explicit error to call +:func:`~hypothesis.strategies.register_type_strategy` with a +`Pydantic GenericModel `__ +and a callable, because ``GenericModel`` isn't actually a generic type at +runtime and so you have to register each of the "parametrized versions" +(actually subclasses!) manually. See :issue:`2940` for more details. + +.. _v6.43.0: + +------------------- +6.43.0 - 2022-04-12 +------------------- + +This release makes it an explicit error to apply +:func:`@pytest.fixture ` to a function which has +already been decorated with :func:`@given() `. Previously, +``pytest`` would convert your test to a fixture, and then never run it. + +.. _v6.42.3: + +------------------- +6.42.3 - 2022-04-10 +------------------- + +This patch fixes :func:`~hypothesis.strategies.from_type` on a :class:`~python:typing.TypedDict` +with complex annotations, defined in a file using ``from __future__ import annotations``. +Thanks to Katelyn Gigante for identifying and fixing this bug! + +.. _v6.42.2: + +------------------- +6.42.2 - 2022-04-10 +------------------- + +The Hypothesis pytest plugin was not outputting valid xunit2 nodes when +``--junit-xml`` was specified. This has been broken since Pytest 5.4, which +changed the internal API for adding nodes to the junit report. + +This also fixes the issue when using hypothesis with ``--junit-xml`` and +``pytest-xdist`` where the junit xml report would not be xunit2 compatible. +Now, when using with ``pytest-xdist``, the junit report will just omit the +```` node. + +For more details, see `this pytest issue `__, +`this pytest issue `__, +and :issue:`1935` + +Thanks to Brandon Chinn for this bug fix! + +.. _v6.42.1: + +------------------- +6.42.1 - 2022-04-10 +------------------- + +This patch fixes pretty-printing of regular expressions in Python 3.11.0a7, and +updates our vendored `list of top-level domains `__,. + +.. _v6.42.0: + +------------------- +6.42.0 - 2022-04-09 +------------------- + +This release makes ``st.functions(pure=True)`` less noisy (:issue:`3253`), +and generally improves pretty-printing of functions. + +.. _v6.41.0: + +------------------- +6.41.0 - 2022-04-01 +------------------- + +This release changes the implementation of :const:`~hypothesis.infer` to be an alias +for :obj:`python:Ellipsis`. E.g. ``@given(a=infer)`` is now equivalent to ``@given(a=...)``. Furthermore, ``@given(...)`` can now be specified so that +:func:`@given ` will infer the strategies for *all* arguments of the +decorated function based on its annotations. + +.. _v6.40.3: + +------------------- +6.40.3 - 2022-04-01 +------------------- + +This patch simplifies the repr of the strategies namespace returned in +:func:`~hypothesis.extra.array_api.make_strategies_namespace`, e.g. + +.. code-block:: pycon + + >>> from hypothesis.extra.array_api import make_strategies_namespace + >>> from numpy import array_api as xp + >>> xps = make_strategies_namespace(xp) + >>> xps + make_strategies_namespace(numpy.array_api) + +.. _v6.40.2: + +------------------- +6.40.2 - 2022-04-01 +------------------- + +Fixed :func:`~hypothesis.strategies.from_type` support for +:pep:`604` union types, like ``int | None`` (:issue:`3255`). + +.. _v6.40.1: + +------------------- +6.40.1 - 2022-04-01 +------------------- + +Fixed an internal error when ``given()`` was passed a lambda. + +.. _v6.40.0: + +------------------- +6.40.0 - 2022-03-29 +------------------- + +:doc:`The Ghostwriter ` can now write tests which check that +two or more functions are equivalent on valid inputs, *or* raise the same +type of exception for invalid inputs (:issue:`3267`). + +.. _v6.39.6: + +------------------- +6.39.6 - 2022-03-27 +------------------- + +This patch makes some quality-of-life improvements to the +:doc:`Ghostwriter `: we guess the :func:`~hypothesis.strategies.text` +strategy for arguments named ``text`` (...obvious in hindsight, eh?); +and improved the error message if you accidentally left in a +:func:`~hypothesis.strategies.nothing` or broke your :pypi:`rich` install. + +.. _v6.39.5: + +------------------- +6.39.5 - 2022-03-26 +------------------- + +This patch improves our error detection and message when Hypothesis is run +on a Python implementation without support for ``-0.0``, which is required +for the :func:`~hypothesis.strategies.floats` strategy but can be disabled by +`unsafe compiler options `__ +(:issue:`3265`). + +.. _v6.39.4: + +------------------- +6.39.4 - 2022-03-17 +------------------- + +This patch tweaks some internal formatting. There is no user-visible change. + +.. _v6.39.3: + +------------------- +6.39.3 - 2022-03-07 +------------------- + +If the :obj:`~hypothesis.Phase.shrink` phase is disabled, we now stop the +:obj:`~hypothesis.Phase.generate` phase as soon as an error is found regardless +of the value of the ``report_multiple_examples`` setting, since that's +probably what you wanted (:issue:`3244`). + +.. _v6.39.2: + +------------------- +6.39.2 - 2022-03-07 +------------------- + +This patch clarifies rare error messages in +:func:`~hypothesis.strategies.builds` (:issue:`3225`) and +:func:`~hypothesis.strategies.floats` (:issue:`3207`). + +.. _v6.39.1: + +------------------- +6.39.1 - 2022-03-03 +------------------- + +This patch fixes a regression where the bound inner function +(``your_test.hypothesis.inner_test``) would be invoked with positional +arguments rather than passing them by name, which broke +:pypi:`pytest-asyncio` (:issue:`3245`). + +.. _v6.39.0: + +------------------- +6.39.0 - 2022-03-01 +------------------- + +This release improves Hypothesis' handling of positional-only arguments, +which are now allowed :func:`@st.composite ` +strategies. + +On Python 3.8 and later, the first arguments to :func:`~hypothesis.strategies.builds` +and :func:`~hypothesis.extra.django.from_model` are now natively positional-only. +In cases which were already errors, the ``TypeError`` from incorrect usage will +therefore be raises immediately when the function is called, rather than when +the strategy object is used. + +.. _v6.38.0: + +------------------- +6.38.0 - 2022-02-26 +------------------- + +This release makes :func:`~hypothesis.strategies.floats` error *consistently* when +your floating-point hardware has been configured to violate IEEE-754 for +:wikipedia:`subnormal numbers `, instead of +only when an internal assertion was tripped (:issue:`3092`). + +If this happens to you, passing ``allow_subnormal=False`` will suppress the explicit +error. However, we strongly recommend fixing the root cause by disabling global-effect +unsafe-math compiler options instead, or at least consulting e.g. Simon Byrne's +`Beware of fast-math `__ explainer first. + +.. _v6.37.2: + +------------------- +6.37.2 - 2022-02-21 +------------------- + +This patch fixes a bug in stateful testing, where returning a single value +wrapped in :func:`~hypothesis.stateful.multiple` would be printed such that +the assigned variable was a tuple rather than the single element +(:issue:`3236`). + +.. _v6.37.1: + +------------------- +6.37.1 - 2022-02-21 +------------------- + +This patch fixes a warning under :pypi:`pytest` 7 relating to our +rich traceback display logic (:issue:`3223`). + +.. _v6.37.0: + +------------------- +6.37.0 - 2022-02-18 +------------------- + +When distinguishing multiple errors, Hypothesis now looks at the inner +exceptions of :pep:`654` ``ExceptionGroup``\ s. + +.. _v6.36.2: + +------------------- +6.36.2 - 2022-02-13 +------------------- + +This patch updates our vendored `list of top-level domains `__, +which is used by the provisional :func:`~hypothesis.provisional.domains` strategy. + +.. _v6.36.1: + +------------------- +6.36.1 - 2022-01-31 +------------------- + +This patch fixes some deprecation warnings from :pypi:`pytest` 7.0, +along with some code formatting and docs updates. + +.. _v6.36.0: + +------------------- +6.36.0 - 2022-01-19 +------------------- + +This release disallows using :obj:`python:typing.Final` +with :func:`~hypothesis.strategies.from_type` +and :func:`~hypothesis.strategies.register_type_strategy`. + +Why? +Because ``Final`` can only be used during ``class`` definition. +We don't generate class attributes. + +It also does not make sense as a runtime type on its own. + +.. _v6.35.1: + +------------------- +6.35.1 - 2022-01-17 +------------------- + +This patch fixes ``hypothesis write`` output highlighting with :pypi:`rich` +version 12.0 and later. + +.. _v6.35.0: + +------------------- +6.35.0 - 2022-01-08 +------------------- + +This release disallows using :obj:`python:typing.ClassVar` +with :func:`~hypothesis.strategies.from_type` +and :func:`~hypothesis.strategies.register_type_strategy`. + +Why? +Because ``ClassVar`` can only be used during ``class`` definition. +We don't generate class attributes. + +It also does not make sense as a runtime type on its own. + +.. _v6.34.2: + +------------------- +6.34.2 - 2022-01-05 +------------------- + +This patch updates our vendored `list of top-level domains `__, +which is used by the provisional :func:`~hypothesis.provisional.domains` strategy. + +.. _v6.34.1: + +------------------- +6.34.1 - 2021-12-31 +------------------- + +This patch fixes :issue:`3169`, an extremely rare bug which would +trigger if an internal least-recently-reused cache dropped a newly +added entry immediately after it was added. + +.. _v6.34.0: + +------------------- +6.34.0 - 2021-12-31 +------------------- + +This release fixes :issue:`3133` and :issue:`3144`, where attempting +to generate Pandas series of lists or sets would fail with confusing +errors if you did not specify ``dtype=object``. + +.. _v6.33.0: + +------------------- +6.33.0 - 2021-12-30 +------------------- + +This release disallows using :obj:`python:typing.TypeAlias` +with :func:`~hypothesis.strategies.from_type` +and :func:`~hypothesis.strategies.register_type_strategy`. + +Why? Because ``TypeAlias`` is not really a type, +it is a tag for type checkers that some expression is a type alias, +not something else. + +It does not make sense for Hypothesis to resolve it as a strategy. +References :issue:`2978`. + +.. _v6.32.1: + +------------------- +6.32.1 - 2021-12-23 +------------------- + +This patch updates our autoformatting tools, improving our code style without any API changes. + +.. _v6.32.0: + +------------------- +6.32.0 - 2021-12-23 +------------------- + +This release drops support for Python 3.6, which `reached end of life upstream +`__ on 2021-12-23. + +.. _v6.31.6: + +------------------- +6.31.6 - 2021-12-15 +------------------- + +This patch adds a temporary hook for a downstream tool, +which is not part of the public API. + +.. _v6.31.5: + +------------------- +6.31.5 - 2021-12-14 +------------------- + +This release updates our copyright headers to `use a general authorship statement and omit the year +`__. + +.. _v6.31.4: + +------------------- +6.31.4 - 2021-12-11 +------------------- + +This patch makes the ``.example()`` method more representative of +test-time data generation, albeit often at a substantial cost to +readability (:issue:`3182`). + +.. _v6.31.3: + +------------------- +6.31.3 - 2021-12-10 +------------------- + +This patch improves annotations on some of Hypothesis' internal functions, in order to +deobfuscate the signatures of some strategies. In particular, strategies shared between +:ref:`hypothesis.extra.numpy ` and +:ref:`the hypothesis.extra.array_api extra ` will benefit from this patch. + +.. _v6.31.2: + +------------------- +6.31.2 - 2021-12-10 +------------------- + +This patch fix invariants display in stateful falsifying examples (:issue:`3185`). + +.. _v6.31.1: + +------------------- +6.31.1 - 2021-12-10 +------------------- + +This patch updates :func:`xps.indices` so no flat indices are generated, i.e. +generated indices will now always explicitly cover each axes of an array if no +ellipsis is present. This is to be consistent with a specification change that +dropped support for flat indexing +(`#272 `_). + +.. _v6.31.0: + +------------------- +6.31.0 - 2021-12-09 +------------------- + +This release makes us compatible with :pypi:`Django` 4.0, in particular by adding +support for use of :mod:`zoneinfo` timezones (though we respect the new +``USE_DEPRECATED_PYTZ`` setting if you need it). + +.. _v6.30.1: + +------------------- +6.30.1 - 2021-12-05 +------------------- + +This patch updates our vendored `list of top-level domains `__, +which is used by the provisional :func:`~hypothesis.provisional.domains` strategy. + +.. _v6.30.0: + +------------------- +6.30.0 - 2021-12-03 +------------------- + +This release adds an ``allow_subnormal`` argument to the +:func:`~hypothesis.strategies.floats` strategy, which can explicitly toggle the +generation of :wikipedia:`subnormal floats ` (:issue:`3155`). +Disabling such generation is useful when testing flush-to-zero builds of +libraries. + +:func:`nps.from_dtype() ` and +:func:`xps.from_dtype` can also accept the ``allow_subnormal`` argument, and +:func:`xps.from_dtype` or :func:`xps.arrays` will disable subnormals by default +if the array module ``xp`` is detected to flush-to-zero (like is typical with +CuPy). + +.. _v6.29.3: + +------------------- +6.29.3 - 2021-12-02 +------------------- + +This patch fixes a bug in :func:`~hypothesis.extra.numpy.mutually_broadcastable_shapes`, +which restricted the patterns of singleton dimensions that could be generated for +dimensions that extended beyond ``base_shape`` (:issue:`3170`). + +.. _v6.29.2: + +------------------- +6.29.2 - 2021-12-02 +------------------- + +This patch clarifies our pretty-printing of DataFrames (:issue:`3114`). + +.. _v6.29.1: + +------------------- +6.29.1 - 2021-12-02 +------------------- + +This patch documents :func:`~hypothesis.strategies.timezones` +`Windows-only requirement `__ +for the :pypi:`tzdata` package, and ensures that +``pip install hypothesis[zoneinfo]`` will install the latest version. + +.. _v6.29.0: + +------------------- +6.29.0 - 2021-11-29 +------------------- + +This release teaches :func:`~hypothesis.strategies.builds` to use +:func:`~hypothesis.strategies.deferred` when resolving unrecognised type hints, +so that you can conveniently register strategies for recursive types +with constraints on some arguments (:issue:`3026`): + +.. code-block:: python + + class RecursiveClass: + def __init__(self, value: int, next_node: typing.Optional["SomeClass"]): + assert value > 0 + self.value = value + self.next_node = next_node + + + st.register_type_strategy( + RecursiveClass, st.builds(RecursiveClass, value=st.integers(min_value=1)) + ) + +.. _v6.28.1: + +------------------- +6.28.1 - 2021-11-28 +------------------- + +This release fixes some internal calculations related to collection sizes (:issue:`3143`). + +.. _v6.28.0: + +------------------- +6.28.0 - 2021-11-28 +------------------- + +This release modifies our :pypi:`pytest` plugin, to avoid importing Hypothesis +and therefore triggering :ref:`Hypothesis' entry points ` for +test suites where Hypothesis is installed but not actually used (:issue:`3140`). + +.. _v6.27.3: + +------------------- +6.27.3 - 2021-11-28 +------------------- + +This release fixes :issue:`3080`, where :func:`~hypothesis.strategies.from_type` +failed on unions containing :pep:`585` builtin generic types (like ``list[int]``) +in Python 3.9 and later. + +.. _v6.27.2: + +------------------- +6.27.2 - 2021-11-26 +------------------- + +This patch makes the :command:`hypothesis codemod` +:ref:`command ` somewhat faster. + +.. _v6.27.1: + +------------------- +6.27.1 - 2021-11-22 +------------------- + +This patch changes the backing datastructures of :func:`~hypothesis.register_random` +and a few internal caches to use :class:`weakref.WeakValueDictionary`. This reduces +memory usage and may improve performance when registered :class:`~random.Random` +instances are only used for a subset of your tests (:issue:`3131`). + +.. _v6.27.0: + +------------------- +6.27.0 - 2021-11-22 +------------------- + +This release teaches Hypothesis' multiple-error reporting to format tracebacks +using :pypi:`pytest` or :pypi:`better-exceptions`, if they are installed and +enabled (:issue:`3116`). + +.. _v6.26.0: + +------------------- +6.26.0 - 2021-11-21 +------------------- + +Did you know that of the 2\ :superscript:`64` possible floating-point numbers, +2\ :superscript:`53` of them are ``nan`` - and Python prints them all the same way? + +While nans *usually* have all zeros in the sign bit and mantissa, this +`isn't always true `__, +and :wikipedia:`'signaling' nans might trap or error `. +To help distinguish such errors in e.g. CI logs, Hypothesis now prints ``-nan`` for +negative nans, and adds a comment like ``# Saw 3 signaling NaNs`` if applicable. + +.. _v6.25.0: + +------------------- +6.25.0 - 2021-11-19 +------------------- + +This release adds special filtering logic to make a few special cases +like ``s.map(lambda x: x)`` and ``lists().filter(len)`` more efficient +(:issue:`2701`). + +.. _v6.24.6: + +------------------- +6.24.6 - 2021-11-18 +------------------- + +This patch makes :func:`~hypothesis.strategies.floats` generate +:wikipedia:`"subnormal" floating point numbers ` +more often, as these rare values can have strange interactions with +`unsafe compiler optimisations like -ffast-math +`__ +(:issue:`2976`). + +.. _v6.24.5: + +------------------- +6.24.5 - 2021-11-16 +------------------- + +This patch fixes a rare internal error in the :func:`~hypothesis.strategies.datetimes` +strategy, where the implementation of ``allow_imaginary=False`` crashed when checking +a time during the skipped hour of a DST transition *if* the DST offset is negative - +only true of ``Europe/Dublin``, who we presume have their reasons - and the ``tzinfo`` +object is a :pypi:`pytz` timezone (which predates :pep:`495`). + +.. _v6.24.4: + +------------------- +6.24.4 - 2021-11-15 +------------------- + +This patch gives Hypothesis it's own internal :class:`~random.Random` instance, +ensuring that test suites which reset the global random state don't induce +weird correlations between property-based tests (:issue:`2135`). + +.. _v6.24.3: + +------------------- +6.24.3 - 2021-11-13 +------------------- + +This patch updates documentation of :func:`~hypothesis.note` +(:issue:`3147`). + +.. _v6.24.2: + +------------------- +6.24.2 - 2021-11-05 +------------------- + +This patch updates internal testing for the :ref:`Array API extra ` +to be consistent with new specification changes: ``sum()`` not accepting +boolean arrays (`#234 `_), +``unique()`` split into separate functions +(`#275 `_), and treating NaNs +as distinct (`#310 `_). It has +no user visible impact. + +.. _v6.24.1: + +------------------- +6.24.1 - 2021-11-01 +------------------- + +This patch updates our vendored `list of top-level domains `__, +which is used by the provisional :func:`~hypothesis.provisional.domains` strategy. + +.. _v6.24.0: + +------------------- +6.24.0 - 2021-10-23 +------------------- + +This patch updates our vendored `list of top-level domains +`__, which is used +by the provisional :func:`~hypothesis.provisional.domains` strategy. + +(did you know that gTLDs can be both `added `__ +and `removed `__?) + +.. _v6.23.4: + +------------------- +6.23.4 - 2021-10-20 +------------------- + +This patch adds an error for when ``shapes`` in :func:`xps.arrays()` is not +passed as either a valid shape or strategy. + +.. _v6.23.3: + +------------------- +6.23.3 - 2021-10-18 +------------------- + +This patch updates our formatting with :pypi:`shed`. + +.. _v6.23.2: + +------------------- +6.23.2 - 2021-10-08 +------------------- + +This patch replaces external links to :doc:`NumPy ` API docs +with :mod:`sphinx.ext.intersphinx` cross-references. It is purely a documentation improvement. + +.. _v6.23.1: + +------------------- +6.23.1 - 2021-09-29 +------------------- + +This patch cleans up internal logic for :func:`xps.arrays()`. There is no +user-visible change. + +.. _v6.23.0: + +------------------- +6.23.0 - 2021-09-26 +------------------- + +This release follows :pypi:`pytest` in considering :class:`SystemExit` and +:class:`GeneratorExit` exceptions to be test failures, meaning that we will +shink to minimal examples and check for flakiness even though they subclass +:class:`BaseException` directly (:issue:`2223`). + +:class:`KeyboardInterrupt` continues to interrupt everything, and will be +re-raised immediately. + +.. _v6.22.0: + +------------------- +6.22.0 - 2021-09-24 +------------------- + +This release adds :class:`~hypothesis.extra.django.LiveServerTestCase` and +:class:`~hypothesis.extra.django.StaticLiveServerTestCase` for django test. +Thanks to Ivan Tham for this feature! + +.. _v6.21.6: + +------------------- +6.21.6 - 2021-09-19 +------------------- + +This patch fixes some new linter warnings such as :pypi:`flake8-bugbear`'s +``B904`` for explicit exception chaining, so tracebacks might be a bit nicer. + +.. _v6.21.5: + +------------------- +6.21.5 - 2021-09-16 +------------------- + +This release fixes ``None`` being inferred as the float64 dtype in +:func:`~xps.from_dtype()` and :func:`~xps.arrays()` from the +:ref:`Array API extra `. + +.. _v6.21.4: + +------------------- +6.21.4 - 2021-09-16 +------------------- + +This release fixes the type hint for the +:func:`@given() ` decorator +when decorating an ``async`` function (:issue:`3099`). + +.. _v6.21.3: + +------------------- +6.21.3 - 2021-09-15 +------------------- + +This release improves Ghostwritten tests for builtins (:issue:`2977`). + +.. _v6.21.2: + +------------------- +6.21.2 - 2021-09-15 +------------------- + +This release deprecates use of both ``min_dims > len(shape)`` and +``max_dims > len(shape)`` when ``allow_newaxis == False`` in +:func:`~hypothesis.extra.numpy.basic_indices` (:issue:`3091`). + +.. _v6.21.1: + +------------------- +6.21.1 - 2021-09-13 +------------------- + +This release improves the behaviour of :func:`~hypothesis.strategies.builds` +and :func:`~hypothesis.strategies.from_type` in certain situations involving +decorators (:issue:`2495` and :issue:`3029`). + +.. _v6.21.0: + +------------------- +6.21.0 - 2021-09-11 +------------------- + +This release introduces strategies for array/tensor libraries adopting the +`Array API standard `__ (:issue:`3037`). +They are available in :ref:`the hypothesis.extra.array_api extra `, +and work much like the existing :doc:`strategies for NumPy `. + +.. _v6.20.1: + +------------------- +6.20.1 - 2021-09-10 +------------------- + +This patch fixes :issue:`961`, where calling ``given()`` inline on a +bound method would fail to handle the ``self`` argument correctly. + +.. _v6.20.0: + +------------------- +6.20.0 - 2021-09-09 +------------------- + +This release allows :func:`~hypothesis.strategies.slices` to generate ``step=None``, +and fixes an off-by-one error where the ``start`` index could be equal to ``size``. +This works fine for all Python sequences and Numpy arrays, but is undefined behaviour +in the `Array API standard `__ (see :pull:`3065`). + +.. _v6.19.0: + +------------------- +6.19.0 - 2021-09-08 +------------------- + +This release makes :doc:`stateful testing ` more likely to tell you +if you do something unexpected and unsupported: + +- The :obj:`~hypothesis.HealthCheck.return_value` health check now applies to + :func:`~hypothesis.stateful.rule` and :func:`~hypothesis.stateful.initialize` + rules, if they don't have ``target`` bundles, as well as + :func:`~hypothesis.stateful.invariant`. +- Using a :func:`~hypothesis.stateful.consumes` bundle as a ``target`` is + deprecated, and will be an error in a future version. + +If existing code triggers these new checks, check for related bugs and +misunderstandings - these patterns *never* had any effect. + +.. _v6.18.0: + +------------------- +6.18.0 - 2021-09-06 +------------------- + +This release teaches :func:`~hypothesis.strategies.from_type` a neat trick: +when resolving an :obj:`python:typing.Annotated` type, if one of the annotations +is a strategy object we use that as the inferred strategy. For example: + +.. code-block:: python + + PositiveInt = Annotated[int, st.integers(min_value=1)] + +If there are multiple strategies, we use the last outer-most annotation. +See :issue:`2978` and :pull:`3082` for discussion. + +*Requires Python 3.9 or later for* +:func:`get_type_hints(..., include_extras=False) `. + +.. _v6.17.4: + +------------------- +6.17.4 - 2021-08-31 +------------------- + +This patch makes unique :func:`~hypothesis.extra.numpy.arrays` much more +efficient, especially when there are only a few valid elements - such as +for eight-bit integers (:issue:`3066`). + +.. _v6.17.3: + +------------------- +6.17.3 - 2021-08-30 +------------------- + +This patch fixes the repr of :func:`~hypothesis.extra.numpy.array_shapes`. + +.. _v6.17.2: + +------------------- +6.17.2 - 2021-08-30 +------------------- + +This patch wraps some internal helper code in our proxies decorator to prevent +mutations of method docstrings carrying over to other instances of the respective +methods. + +.. _v6.17.1: + +------------------- +6.17.1 - 2021-08-29 +------------------- + +This patch moves some internal helper code in preparation for :issue:`3065`. +There is no user-visible change, unless you depended on undocumented internals. + +.. _v6.17.0: + +------------------- +6.17.0 - 2021-08-27 +------------------- + +This release adds type annotations to the :doc:`stateful testing ` API. + +Thanks to Ruben Opdebeeck for this contribution! + +.. _v6.16.0: + +------------------- +6.16.0 - 2021-08-27 +------------------- + +This release adds the :class:`~hypothesis.strategies.DrawFn` type as a reusable +type hint for the ``draw`` argument of +:func:`@composite ` functions. + +Thanks to Ruben Opdebeeck for this contribution! + +.. _v6.15.0: + +------------------- +6.15.0 - 2021-08-22 +------------------- + +This release emits a more useful error message when :func:`@given() ` +is applied to a coroutine function, i.e. one defined using ``async def`` (:issue:`3054`). + +This was previously only handled by the generic :obj:`~hypothesis.HealthCheck.return_value` +health check, which doesn't direct you to use either :ref:`a custom executor ` +or a library such as :pypi:`pytest-trio` or :pypi:`pytest-asyncio` to handle it for you. + +.. _v6.14.9: + +------------------- +6.14.9 - 2021-08-20 +------------------- + +This patch fixes a regression in Hypothesis 6.14.8, where :func:`~hypothesis.strategies.from_type` +failed to resolve types which inherit from multiple parametrised generic types, +affecting the :pypi:`returns` package (:issue:`3060`). + +.. _v6.14.8: + +------------------- +6.14.8 - 2021-08-16 +------------------- + +This patch ensures that registering a strategy for a subclass of a a parametrised +generic type such as ``class Lines(Sequence[str]):`` will not "leak" into unrelated +strategies such as ``st.from_type(Sequence[int])`` (:issue:`2951`). +Unfortunately this fix requires :pep:`560`, meaning Python 3.7 or later. + +.. _v6.14.7: + +------------------- +6.14.7 - 2021-08-14 +------------------- + +This patch fixes :issue:`3050`, where :pypi:`attrs` classes could +cause an internal error in the :doc:`ghostwriter `. + +.. _v6.14.6: + +------------------- +6.14.6 - 2021-08-07 +------------------- + +This patch improves the error message for :issue:`3016`, where :pep:`585` +builtin generics with self-referential forward-reference strings cannot be +resolved to a strategy by :func:`~hypothesis.strategies.from_type`. + +.. _v6.14.5: + +------------------- +6.14.5 - 2021-07-27 +------------------- + +This patch fixes ``hypothesis.strategies._internal.types.is_a_new_type``. +It was failing on Python ``3.10.0b4``, where ``NewType`` is a function. + +.. _v6.14.4: + +------------------- +6.14.4 - 2021-07-26 +------------------- + +This patch fixes :func:`~hypothesis.strategies.from_type` and +:func:`~hypothesis.strategies.register_type_strategy` for +:obj:`python:typing.NewType` on Python 3.10, which changed the +underlying implementation (see :bpo:`44353` for details). + +.. _v6.14.3: + +------------------- +6.14.3 - 2021-07-18 +------------------- + +This patch updates our autoformatting tools, improving our code style without any API changes. + +.. _v6.14.2: + +------------------- +6.14.2 - 2021-07-12 +------------------- + +This patch ensures that we shorten tracebacks for tests which fail due +to inconsistent data generation between runs (i.e. raise ``Flaky``). + +.. _v6.14.1: + +------------------- +6.14.1 - 2021-07-02 +------------------- + +This patch updates some internal type annotations. +There is no user-visible change. + +.. _v6.14.0: + +------------------- +6.14.0 - 2021-06-09 +------------------- + +The :ref:`explain phase ` now requires shrinking to be enabled, +and will be automatically skipped for deadline-exceeded errors. + +.. _v6.13.14: + +-------------------- +6.13.14 - 2021-06-04 +-------------------- + +This patch improves the :func:`~hypothesis.strategies.tuples` strategy +type annotations, to preserve the element types for up to length-five +tuples (:issue:`3005`). + +As for :func:`~hypothesis.strategies.one_of`, this is the best we can do +before a `planned extension `__ +to :pep:`646` is released, hopefully in Python 3.11. + +.. _v6.13.13: + +-------------------- +6.13.13 - 2021-06-04 +-------------------- + +This patch teaches :doc:`the Ghostwriter ` how to find +:doc:`custom ufuncs ` from *any* module that defines them, +and that ``yaml.unsafe_load()`` does not undo ``yaml.safe_load()``. + +.. _v6.13.12: + +-------------------- +6.13.12 - 2021-06-03 +-------------------- + +This patch reduces the amount of internal code excluded from our test suite's +code coverage checks. + +There is no user-visible change. + +.. _v6.13.11: + +-------------------- +6.13.11 - 2021-06-02 +-------------------- + +This patch removes some old internal helper code that previously existed +to make Python 2 compatibility easier. + +There is no user-visible change. + +.. _v6.13.10: + +-------------------- +6.13.10 - 2021-05-30 +-------------------- + +This release adjusts some internal code to help make our test suite more +reliable. + +There is no user-visible change. + +.. _v6.13.9: + +------------------- +6.13.9 - 2021-05-30 +------------------- + +This patch cleans up some internal code related to filtering strategies. + +There is no user-visible change. + +.. _v6.13.8: + +------------------- +6.13.8 - 2021-05-28 +------------------- + +This patch slightly improves the performance of some internal code for +generating integers. + +.. _v6.13.7: + +------------------- +6.13.7 - 2021-05-27 +------------------- + +This patch fixes a bug in :func:`~hypothesis.strategies.from_regex` that +caused ``from_regex("", fullmatch=True)`` to unintentionally generate non-empty +strings (:issue:`4982`). + +The only strings that completely match an empty regex pattern are empty +strings. + +.. _v6.13.6: + +------------------- +6.13.6 - 2021-05-26 +------------------- + +This patch fixes a bug that caused :func:`~hypothesis.strategies.integers` +to shrink towards negative values instead of positive values in some cases. + +.. _v6.13.5: + +------------------- +6.13.5 - 2021-05-24 +------------------- + +This patch fixes rare cases where ``hypothesis write --binary-op`` could +print :doc:`reproducing instructions ` from the internal +search for an identity element. + +.. _v6.13.4: + +------------------- +6.13.4 - 2021-05-24 +------------------- + +This patch removes some unnecessary intermediate list-comprehensions, +using the latest versions of :pypi:`pyupgrade` and :pypi:`shed`. + +.. _v6.13.3: + +------------------- +6.13.3 - 2021-05-23 +------------------- + +This patch adds a ``.hypothesis`` property to invalid test functions, bringing +them inline with valid tests and fixing a bug where :pypi:`pytest-asyncio` would +swallow the real error message and mistakenly raise a version incompatibility +error. + +.. _v6.13.2: + +------------------- +6.13.2 - 2021-05-23 +------------------- + +Some of Hypothesis's numpy/pandas strategies use a ``fill`` argument to speed +up generating large arrays, by generating a single fill value and sharing that +value among many array slots instead of filling every single slot individually. + +When no ``fill`` argument is provided, Hypothesis tries to detect whether it is +OK to automatically use the ``elements`` argument as a fill strategy, so that +it can still use the faster approach. + +This patch fixes a bug that would cause that optimization to trigger in some +cases where it isn't 100% guaranteed to be OK. + +If this makes some of your numpy/pandas tests run more slowly, try adding an +explicit ``fill`` argument to the relevant strategies to ensure that Hypothesis +always uses the faster approach. + +.. _v6.13.1: + +------------------- +6.13.1 - 2021-05-20 +------------------- + +This patch strengthens some internal import-time consistency checks for the +built-in strategies. + +There is no user-visible change. + +.. _v6.13.0: + +------------------- +6.13.0 - 2021-05-18 +------------------- + +This release adds URL fragment generation to the :func:`~hypothesis.provisional.urls` +strategy (:issue:`2908`). Thanks to Pax (R. Margret) for contributing this patch at the +`PyCon US Mentored Sprints `__! + +.. _v6.12.1: + +------------------- +6.12.1 - 2021-05-17 +------------------- + +This patch fixes :issue:`2964`, where ``.map()`` and ``.filter()`` methods +were omitted from the ``repr()`` of :func:`~hypothesis.strategies.just` and +:func:`~hypothesis.strategies.sampled_from` strategies, since +:ref:`version 5.43.7 `. + +.. _v6.12.0: + +------------------- +6.12.0 - 2021-05-06 +------------------- + +This release automatically rewrites some simple filters, such as +``integers().filter(lambda x: x > 9)`` to the more efficient +``integers(min_value=10)``, based on the AST of the predicate. + +We continue to recommend using the efficient form directly wherever +possible, but this should be useful for e.g. :pypi:`pandera` "``Checks``" +where you already have a simple predicate and translating manually +is really annoying. See :issue:`2701` for ideas about floats and +simple text strategies. + +.. _v6.11.0: + +------------------- +6.11.0 - 2021-05-06 +------------------- + +:func:`hypothesis.target` now returns the ``observation`` value, +allowing it to be conveniently used inline in expressions such as +``assert target(abs(a - b)) < 0.1``. + +.. _v6.10.1: + +------------------- +6.10.1 - 2021-04-26 +------------------- + +This patch fixes a deprecation warning if you're using recent versions +of :pypi:`importlib-metadata` (:issue:`2934`), which we use to load +:ref:`third-party plugins ` such as `Pydantic's integration +`__. +On older versions of :pypi:`importlib-metadata`, there is no change and +you don't need to upgrade. + +.. _v6.10.0: + +------------------- +6.10.0 - 2021-04-17 +------------------- + +This release teaches the :doc:`Ghostwriter ` to read parameter +types from Sphinx, Google, or Numpy-style structured docstrings, and improves +some related heuristics about how to test scientific and numerical programs. + +.. _v6.9.2: + +------------------ +6.9.2 - 2021-04-15 +------------------ + +This release improves the :doc:`Ghostwriter's ` handling +of exceptions, by reading ``:raises ...:`` entries in function docstrings +and ensuring that we don't suppresss the error raised by test assertions. + +.. _v6.9.1: + +------------------ +6.9.1 - 2021-04-12 +------------------ + +This patch updates our autoformatting tools, improving our code style without any API changes. + +.. _v6.9.0: + +------------------ +6.9.0 - 2021-04-11 +------------------ + +This release teaches :func:`~hypothesis.strategies.from_type` how to see +through :obj:`python:typing.Annotated`. Thanks to Vytautas Strimaitis +for reporting and fixing :issue:`2919`! + +.. _v6.8.12: + +------------------- +6.8.12 - 2021-04-11 +------------------- + +If :pypi:`rich` is installed, the :command:`hypothesis write` command +will use it to syntax-highlight the :doc:`Ghostwritten ` +code. + +.. _v6.8.11: + +------------------- +6.8.11 - 2021-04-11 +------------------- + +This patch improves an error message from :func:`~hypothesis.strategies.builds` +when :func:`~hypothesis.strategies.from_type` would be more suitable (:issue:`2930`). + +.. _v6.8.10: + +------------------- +6.8.10 - 2021-04-11 +------------------- + +This patch updates the type annotations for :func:`~hypothesis.extra.numpy.arrays` to reflect that +``shape: SearchStrategy[int]`` is supported. + +.. _v6.8.9: + +------------------ +6.8.9 - 2021-04-07 +------------------ + +This patch fixes :func:`~hypothesis.strategies.from_type` with +:mod:`abstract types ` which have either required but +non-type-annotated arguments to ``__init__``, or where +:func:`~hypothesis.strategies.from_type` can handle some concrete +subclasses but not others. + +.. _v6.8.8: + +------------------ +6.8.8 - 2021-04-07 +------------------ + +This patch teaches :command:`hypothesis write` to check for possible roundtrips +in several more cases, such as by looking for an inverse in the module which +defines the function to test. + +.. _v6.8.7: + +------------------ +6.8.7 - 2021-04-07 +------------------ + +This patch adds a more helpful error message if you try to call +:func:`~hypothesis.strategies.sampled_from` on an :class:`~python:enum.Enum` +which has no members, but *does* have :func:`~python:dataclasses.dataclass`-style +annotations (:issue:`2923`). + +.. _v6.8.6: + +------------------ +6.8.6 - 2021-04-06 +------------------ + +The :func:`~hypothesis.strategies.fixed_dictionaries` strategy now preserves +dict iteration order instead of sorting the keys. This also affects the +pretty-printing of keyword arguments to :func:`@given() ` +(:issue:`2913`). + +.. _v6.8.5: + +------------------ +6.8.5 - 2021-04-05 +------------------ + +This patch teaches :command:`hypothesis write` to default to ghostwriting +tests with ``--style=pytest`` only if :pypi:`pytest` is installed, or +``--style=unittest`` otherwise. + +.. _v6.8.4: + +------------------ +6.8.4 - 2021-04-01 +------------------ + +This patch adds type annotations for the :class:`~hypothesis.settings` decorator, +to avoid an error when running mypy in strict mode. + +.. _v6.8.3: + +------------------ +6.8.3 - 2021-03-28 +------------------ + +This patch improves the :doc:`Ghostwriter's ` handling +of strategies to generate various fiddly types including frozensets, +keysviews, valuesviews, regex matches and patterns, and so on. + +.. _v6.8.2: + +------------------ +6.8.2 - 2021-03-27 +------------------ + +This patch fixes some internal typos. There is no user-visible change. + +.. _v6.8.1: + +------------------ +6.8.1 - 2021-03-14 +------------------ + +This patch lays more groundwork for filter rewriting (:issue:`2701`). +There is no user-visible change... yet. + +.. _v6.8.0: + +------------------ +6.8.0 - 2021-03-11 +------------------ + +This release :func:`registers ` the +remaining builtin types, and teaches :func:`~hypothesis.strategies.from_type` to +try resolving :class:`~python:typing.ForwardRef` and :class:`~python:typing.Type` +references to built-in types. + +.. _v6.7.0: + +------------------ +6.7.0 - 2021-03-10 +------------------ + +This release teaches :class:`~hypothesis.stateful.RuleBasedStateMachine` to avoid +checking :func:`~hypothesis.stateful.invariant`\ s until all +:func:`~hypothesis.stateful.initialize` rules have been run. You can enable checking +of specific invariants for incompletely initialized machines by using +``@invariant(check_during_init=True)`` (:issue:`2868`). + +In previous versions, it was possible if awkward to implement this behaviour +using :func:`~hypothesis.stateful.precondition` and an auxiliary variable. + +.. _v6.6.1: + +------------------ +6.6.1 - 2021-03-09 +------------------ + +This patch improves the error message when :func:`~hypothesis.strategies.from_type` +fails to resolve a forward-reference inside a :class:`python:typing.Type` +such as ``Type["int"]`` (:issue:`2565`). + +.. _v6.6.0: + +------------------ +6.6.0 - 2021-03-07 +------------------ + +This release makes it an explicit error to apply :func:`~hypothesis.stateful.invariant` +to a :func:`~hypothesis.stateful.rule` or :func:`~hypothesis.stateful.initialize` rule +in :doc:`stateful testing `. Such a combination had unclear semantics, +especially in combination with :func:`~hypothesis.stateful.precondition`, and was never +meant to be allowed (:issue:`2681`). + +.. _v6.5.0: + +------------------ +6.5.0 - 2021-03-07 +------------------ + +This release adds :ref:`the explain phase `, in which Hypothesis +attempts to explain *why* your test failed by pointing to suspicious lines +of code (i.e. those which were always, and only, run on failing inputs). +We plan to include "generalising" failing examples in this phase in a +future release (:issue:`2192`). + +.. _v6.4.3: + +------------------ +6.4.3 - 2021-03-04 +------------------ + +This patch fixes :issue:`2794`, where nesting :func:`~hypothesis.strategies.deferred` +strategies within :func:`~hypothesis.strategies.recursive` strategies could +trigger an internal assertion. While it was always possible to get the same +results from a more sensible strategy, the convoluted form now works too. + +.. _v6.4.2: + +------------------ +6.4.2 - 2021-03-04 +------------------ + +This patch fixes several problems with ``mypy`` when `--no-implicit-reexport `_ was activated in user projects. + +Thanks to Nikita Sobolev for fixing :issue:`2884`! + +.. _v6.4.1: + +------------------ +6.4.1 - 2021-03-04 +------------------ + +This patch fixes an exception that occurs when using type unions of +the :pypi:`typing_extensions` ``Literal`` backport on Python 3.6. + +Thanks to Ben Anhalt for identifying and fixing this bug. + +.. _v6.4.0: + +------------------ +6.4.0 - 2021-03-02 +------------------ + +This release fixes :doc:`stateful testing methods ` with multiple +:func:`~hypothesis.stateful.precondition` decorators. Previously, only the +outer-most precondition was checked (:issue:`2681`). + +.. _v6.3.4: + +------------------ +6.3.4 - 2021-02-28 +------------------ + +This patch refactors some internals of :class:`~hypothesis.stateful.RuleBasedStateMachine`. +There is no change to the public API or behaviour. + +.. _v6.3.3: + +------------------ +6.3.3 - 2021-02-26 +------------------ + +This patch moves some internal code, so that future work can avoid +creating import cycles. There is no user-visible change. + +.. _v6.3.2: + +------------------ +6.3.2 - 2021-02-25 +------------------ + +This patch enables :func:`~hypothesis.strategies.register_type_strategy` for subclasses of +:class:`python:typing.TypedDict`. Previously, :func:`~hypothesis.strategies.from_type` +would ignore the registered strategy (:issue:`2872`). + +Thanks to Ilya Lebedev for identifying and fixing this bug! + +.. _v6.3.1: + +------------------ +6.3.1 - 2021-02-24 +------------------ + +This release lays the groundwork for automatic rewriting of simple filters, +for example converting ``integers().filter(lambda x: x > 9)`` to +``integers(min_value=10)``. + +Note that this is **not supported yet**, and we will continue to recommend +writing the efficient form directly wherever possible - predicate rewriting +is provided mainly for the benefit of downstream libraries which would +otherwise have to implement it for themselves (e.g. :pypi:`pandera` and +:pypi:`icontract-hypothesis`). See :issue:`2701` for details. + .. _v6.3.0: ------------------ @@ -98,7 +2271,7 @@ that are running in multiple threads (:issue:`2717`). This patch improves the type annotations for :func:`~hypothesis.strategies.one_of`, by adding overloads to handle up to five distinct arguments as -:class:`~python:typing.Union` before falling back to :class:`~python:typing.Any`, +:obj:`~python:typing.Union` before falling back to :obj:`~python:typing.Any`, as well as annotating the ``|`` (``__or__``) operator for strategies (:issue:`2765`). .. _v6.0.2: @@ -473,7 +2646,7 @@ parameterized standard collection types, which are new in Python 3.9 ------------------- This patch fixes :func:`~hypothesis.strategies.builds`, so that when passed -:obj:`~hypothesis.infer` for an argument with a non-:class:`~python:typing.Optional` +:obj:`~hypothesis.infer` for an argument with a non-:obj:`~python:typing.Optional` type annotation and a default value of ``None`` to build a class which defines an explicit ``__signature__`` attribute, either ``None`` or that type may be generated. @@ -655,7 +2828,7 @@ The :func:`~hypothesis.target` function now accepts integers as well as floats. 5.34.1 - 2020-09-11 ------------------- -This patch adds explicit :class:`~python:typing.Optional` annotations to our public API, +This patch adds explicit :obj:`~python:typing.Optional` annotations to our public API, to better support users who run :pypi:`mypy` with ``--strict`` or ``no_implicit_optional=True``. Thanks to Krzysztof Przybyła for bringing this to our attention and writing the patch! @@ -731,7 +2904,7 @@ Previously :func:`python:typing.get_type_hints`, was used by default. If argument names varied between the ``__annotations__`` and ``__signature__``, they would not be supplied to the target. -This was particularily an issue for :pypi:`pydantic` models which use an +This was particularly an issue for :pypi:`pydantic` models which use an `alias generator `__. .. _v5.30.1: @@ -834,8 +3007,8 @@ Thanks to Zac Hatfield-Dodds and Nikita Sobolev for this feature! This patch adds two new :doc:`ghostwriters ` to test :wikipedia:`binary operations `, like :func:`python:operator.add`, -and Numpy :np-ref:`ufuncs ` and :np-ref:`gufuncs -` like :obj:`np.matmul() `. +and Numpy :doc:`ufuncs ` and :doc:`gufuncs +` like :data:`np.matmul() `. .. _v5.26.1: @@ -1237,7 +3410,7 @@ backport on PyPI. 5.16.3 - 2020-06-21 ------------------- -This patch precomputes some of the setup logic for our experimental +This patch precomputes some of the setup logic for our :ref:`external fuzzer integration ` and sets :obj:`deadline=None ` in fuzzing mode, saving around 150us on each iteration. @@ -1571,7 +3744,7 @@ scoped fixture is overridden with a higher scoped fixture. This release allows the :func:`~hypothesis.extra.numpy.array_dtypes` strategy to generate Numpy dtypes which have `field titles in addition to field names -`__. +`__. We expect this to expose latent bugs where code expects that ``set(dtype.names) == set(dtype.fields)``, though the latter may include titles. @@ -1593,7 +3766,7 @@ with a field literally named "model" (:issue:`2369`). This release adds an explicit warning for tests that are both decorated with :func:`@given(...) ` and request a -:doc:`function-scoped pytest fixture `, because such fixtures +:doc:`function-scoped pytest fixture `, because such fixtures are only executed once for *all* Hypothesis test cases and that often causes trouble (:issue:`377`). @@ -2011,8 +4184,8 @@ provided in the standard-library :mod:`python:warnings` module. ------------------- This release improves Hypothesis's management of the set of test cases it -tracks between runs. It will only do anything if you have ``Phase.target`` -enabled and an example database set. +tracks between runs. It will only do anything if you have the +:obj:`~hypothesis.Phase.target` phase enabled and an example database set. In those circumstances it should result in a more thorough and faster set of examples that are tried on each run. @@ -2479,7 +4652,7 @@ a ``__file__``, such as a :mod:`python:zipapp` (:issue:`2196`). This release adds a ``signature`` argument to :func:`~hypothesis.extra.numpy.mutually_broadcastable_shapes` (:issue:`2174`), which allows us to generate shapes which are valid for functions like -:obj:`numpy:numpy.matmul` that require shapes which are not simply broadcastable. +:data:`np.matmul() ` that require shapes which are not simply broadcastable. Thanks to everyone who has contributed to this feature over the last year, and a particular shout-out to Zac Hatfield-Dodds and Ryan Soklaski for @@ -2625,7 +4798,7 @@ chooses for ``max_dims`` is always valid (at most 32), even if you pass ``min_di This patch ensures that we only add profile information to the pytest header if running either pytest or Hypothesis in verbose mode, matching the -`builtin cache plugin `__ +`builtin cache plugin `__ (:issue:`2155`). .. _v4.42.7: @@ -2840,7 +5013,7 @@ is not a strategy, Hypothesis now tells you which one. ------------------- This release adds the :func:`~hypothesis.extra.numpy.basic_indices` strategy, -to generate `basic indexes `__ +to generate `basic indexes `__ for arrays of the specified shape (:issue:`1930`). It generates tuples containing some mix of integers, :obj:`python:slice` objects, @@ -2889,7 +5062,7 @@ some internal compatibility shims we use to support older Pythons. ------------------- This release adds the :func:`hypothesis.target` function, which implements -**experimental** support for :ref:`targeted property-based testing ` +:ref:`targeted property-based testing ` (:issue:`1779`). By calling :func:`~hypothesis.target` in your test function, Hypothesis can @@ -3854,7 +6027,7 @@ so you probably won't see much effect specifically from that. This patch removes some overhead from :func:`~hypothesis.extra.numpy.arrays` with a constant shape and dtype. The resulting performance improvement is -modest, but worthwile for small arrays. +modest, but worthwhile for small arrays. .. _v4.7.17: @@ -4175,7 +6348,7 @@ This patch updates some docstrings, but has no runtime changes. This release adds ``exclude_min`` and ``exclude_max`` arguments to :func:`~hypothesis.strategies.floats`, so that you can easily generate values from -`open or half-open intervals `_ +:wikipedia:`open or half-open intervals ` (:issue:`1622`). .. _v4.4.6: @@ -5184,7 +7357,7 @@ This is a no-op release, which implements automatic DOI minting and code archival of Hypothesis via `Zenodo `_. Thanks to CERN and the EU *Horizon 2020* programme for providing this service! -Check our :gh-file:`CITATION` file for details, or head right on over to +Check our :gh-file:`CITATION.cff` file for details, or head right on over to `doi.org/10.5281/zenodo.1412597 `_ .. _v3.71.3: @@ -5449,7 +7622,7 @@ This is a docs-only patch, fixing some typos and formatting issues. This change fixes a small bug in how the core engine caches the results of previously-tried inputs. The effect is unlikely to be noticeable, but it might -avoid unnecesary work in some cases. +avoid unnecessary work in some cases. .. _v3.68.1: @@ -6883,7 +9056,7 @@ that have previously been missed. -------------------- This patch avoids creating debug statements when debugging is disabled. -Profiling suggests this is a 5-10% performance improvement (:pull:`1040`). +Profiling suggests this is a 5-10% performance improvement (:issue:`1040`). .. _v3.44.9: @@ -6951,7 +9124,7 @@ the copyright headers in our source to include 2018. 3.44.4 - 2017-12-23 ------------------- -This release fixes :issue:`1044`, which slowed tests by up to 6% +This release fixes :issue:`1041`, which slowed tests by up to 6% due to broken caching. .. _v3.44.3: @@ -7468,7 +9641,7 @@ unnoticeable. Previously it was large and became much larger in :ref:`3.30.4 making it part of the public API. It also updates how the example database is used. Principally: -* A ``Phases.reuse`` argument will now correctly control whether examples +* The :obj:`~hypothesis.Phase.reuse` phase will now correctly control whether examples from the database are run (it previously did exactly the wrong thing and controlled whether examples would be *saved*). * Hypothesis will no longer try to rerun *all* previously failing examples. diff --git a/hypothesis-python/docs/community.rst b/hypothesis-python/docs/community.rst index 57acbfe5d5..c35fe02b91 100644 --- a/hypothesis-python/docs/community.rst +++ b/hypothesis-python/docs/community.rst @@ -4,20 +4,16 @@ Community The Hypothesis community is small for the moment but is full of excellent people who can answer your questions and help you out. Please do join us. -The two major places for community discussion are: +The major place for community discussion is `the mailing list `_. -* `The mailing list `_. -* An IRC channel, #hypothesis on freenode, which is more active than the mailing list. - -Feel free to use these to ask for help, provide feedback, or discuss anything remotely +Feel free to use it to ask for help, provide feedback, or discuss anything remotely Hypothesis related at all. If you post a question on Stack Overflow, please use the `python-hypothesis `__ tag! -Please note that `the Hypothesis code of conduct `_ +Please note that :gh-file:`the Hypothesis code of conduct ` applies in all Hypothesis community spaces. -If you would like to cite Hypothesis, please consider `our suggested citation -`_. +If you would like to cite Hypothesis, please consider :gh-file:`our suggested citation `. If you like repo badges, we suggest the following badge, which you can add with reStructuredText or Markdown, respectively: diff --git a/hypothesis-python/docs/conf.py b/hypothesis-python/docs/conf.py index 12ad6aa244..5a1d3edacb 100644 --- a/hypothesis-python/docs/conf.py +++ b/hypothesis-python/docs/conf.py @@ -1,21 +1,17 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import datetime import os import sys +import types import sphinx_rtd_theme @@ -31,6 +27,7 @@ "sphinx.ext.viewcode", "sphinx.ext.intersphinx", "hoverxref.extension", + "sphinx_codeautolink", "sphinx_selective_exclude.eager_only", ] @@ -59,8 +56,22 @@ def setup(app): if os.path.isfile(os.path.join(os.path.dirname(__file__), "..", "RELEASE.rst")): app.tags.add("has_release_file") + # patch in mock array_api namespace so we can autodoc it + from hypothesis.extra.array_api import ( + RELEASED_VERSIONS, + make_strategies_namespace, + mock_xp, + ) + + mod = types.ModuleType("xps") + mod.__dict__.update( + make_strategies_namespace(mock_xp, api_version=RELEASED_VERSIONS[-1]).__dict__ + ) + assert "xps" not in sys.modules + sys.modules["xps"] = mod -language = None + +language = "en" exclude_patterns = ["_build"] @@ -71,6 +82,17 @@ def setup(app): # See https://sphinx-hoverxref.readthedocs.io/en/latest/configuration.html hoverxref_auto_ref = True hoverxref_domains = ["py"] +hoverxref_role_types = { + "attr": "tooltip", + "class": "tooltip", + "const": "tooltip", + "exc": "tooltip", + "func": "tooltip", + "meth": "tooltip", + "mod": "tooltip", + "obj": "tooltip", + "ref": "tooltip", +} intersphinx_mapping = { "python": ("https://docs.python.org/3/", None), @@ -81,24 +103,32 @@ def setup(app): "dateutil": ("https://dateutil.readthedocs.io/en/stable/", None), "redis": ("https://redis-py.readthedocs.io/en/stable/", None), "attrs": ("https://www.attrs.org/en/stable/", None), + "sphinx": ("https://www.sphinx-doc.org/en/master/", None), } -autodoc_mock_imports = ["numpy", "pandas", "redis"] +autodoc_mock_imports = ["numpy", "pandas", "redis", "django"] + +codeautolink_autodoc_inject = False +codeautolink_global_preface = """ +from hypothesis import * +import hypothesis.strategies as st +from hypothesis.strategies import * +""" # This config value must be a dictionary of external sites, mapping unique # short alias names to a base URL and a prefix. # See http://sphinx-doc.org/ext/extlinks.html _repo = "https://github.com/HypothesisWorks/hypothesis/" extlinks = { - "commit": (_repo + "commit/%s", "commit "), - "gh-file": (_repo + "blob/master/%s", ""), - "gh-link": (_repo + "%s", ""), - "issue": (_repo + "issues/%s", "issue #"), - "pull": (_repo + "pull/%s", "pull request #"), - "pypi": ("https://pypi.org/project/%s", ""), - "bpo": ("https://bugs.python.org/issue%s", "bpo-"), - "np-ref": ("https://numpy.org/doc/stable/reference/%s", ""), - "wikipedia": ("https://en.wikipedia.org/wiki/%s", ""), + "commit": (_repo + "commit/%s", "commit %s"), + "gh-file": (_repo + "blob/master/%s", "%s"), + "gh-link": (_repo + "%s", "%s"), + "issue": (_repo + "issues/%s", "issue #%s"), + "pull": (_repo + "pull/%s", "pull request #%s"), + "pypi": ("https://pypi.org/project/%s/", "%s"), + "bpo": ("https://bugs.python.org/issue%s", "bpo-%s"), + "xp-ref": ("https://data-apis.org/array-api/latest/API_specification/%s", "%s"), + "wikipedia": ("https://en.wikipedia.org/wiki/%s", "%s"), } # -- Options for HTML output ---------------------------------------------- diff --git a/hypothesis-python/docs/data.rst b/hypothesis-python/docs/data.rst index 78e81eb160..1c19b317cd 100644 --- a/hypothesis-python/docs/data.rst +++ b/hypothesis-python/docs/data.rst @@ -84,7 +84,8 @@ e.g.: [-25527, -24245, -23118, -93, -70, -7, 0, 39, 40, 65, 88, 112, 6189, 9480, 19469, 27256, 32526, 1566924430] Note that many things that you might use mapping for can also be done with -:func:`~hypothesis.strategies.builds`. +:func:`~hypothesis.strategies.builds`, and if you find yourself indexing +into a tuple within ``.map()`` it's probably time to use that instead. .. _filtering: @@ -194,8 +195,7 @@ for your data type, returns a new strategy for it. So for example: ... from pprint import pprint >>> json = recursive( ... none() | booleans() | floats() | text(printable), - ... lambda children: lists(children, 1) - ... | dictionaries(text(printable), children, min_size=1), + ... lambda children: lists(children) | dictionaries(text(printable), children), ... ) >>> pprint(json.example()) [[1.175494351e-38, ']', 1.9, True, False, '.M}Xl', ''], True] diff --git a/hypothesis-python/docs/details.rst b/hypothesis-python/docs/details.rst index c45958783e..b143a64a06 100644 --- a/hypothesis-python/docs/details.rst +++ b/hypothesis-python/docs/details.rst @@ -31,7 +31,7 @@ intermediate steps of your test. That's where the ``note`` function comes in: ... def test_shuffle_is_noop(ls, r): ... ls2 = list(ls) ... r.shuffle(ls2) - ... note("Shuffle: %r" % (ls2)) + ... note(f"Shuffle: {ls2!r}") ... assert ls == ls2 ... >>> try: @@ -43,7 +43,7 @@ intermediate steps of your test. That's where the ``note`` function comes in: Shuffle: [1, 0] ls != ls2 -The note is printed in the final run of the test in order to include any +The note is printed for the minimal failing example of the test in order to include any additional information you might need in your test. @@ -120,7 +120,7 @@ You can also mark custom events in a test using the ``event`` function: @given(st.integers().filter(lambda x: x % 2 == 0)) def test_even_integers(i): - event("i mod 3 = %d" % (i % 3,)) + event(f"i mod 3 = {i%3}") You will then see output like: @@ -252,7 +252,7 @@ Here's what happens if we try to run this: @given(lists(integers())) def test_sum_is_positive(xs): - assume(len(xs) > 1) + assume(len(xs) > 10) assume(all(x > 0 for x in xs)) print(xs) assert sum(xs) > 0 @@ -413,11 +413,26 @@ high confidence in the system being tested than random PBT. (`Löscher and Sagonas `__) This is not *always* a good idea - for example calculating the search metric -might take time better spent running more uniformly-random test cases - but -Hypothesis has **experimental** support for targeted PBT you may wish to try. +might take time better spent running more uniformly-random test cases, or your +target metric might accidentally lead Hypothesis *away* from bugs - but if +there is a natural metric like "floating-point error", "load factor" or +"queue length", we encourage you to experiment with targeted testing. .. autofunction:: hypothesis.target +.. code-block:: python + + from hypothesis import given, strategies as st, target + + + @given(st.floats(0, 1e100), st.floats(0, 1e100), st.floats(0, 1e100)) + def test_associativity_with_target(a, b, c): + ab_c = (a + b) + c + a_bc = a + (b + c) + difference = abs(ab_c - a_bc) + target(difference) # Without this, the test almost always passes + assert difference < 2.0 + We recommend that users also skim the papers introducing targeted PBT; from `ISSTA 2017 `__ and `ICST 2018 `__. @@ -555,11 +570,11 @@ This is based on introspection, *not* magic, and therefore has well-defined limits. :func:`~hypothesis.strategies.builds` will check the signature of the -``target`` (using :func:`~python:inspect.getfullargspec`). +``target`` (using :func:`~python:inspect.signature`). If there are required arguments with type annotations and no strategy was passed to :func:`~hypothesis.strategies.builds`, :func:`~hypothesis.strategies.from_type` is used to fill them in. -You can also pass the special value :const:`hypothesis.infer` as a keyword +You can also pass the value ``...`` (``Ellipsis``) as a keyword argument, to force this inference for arguments with a default value. .. code-block:: pycon @@ -574,21 +589,38 @@ argument, to force this inference for arguments with a default value. :func:`@given ` does not perform any implicit inference for required arguments, as this would break compatibility with pytest fixtures. -:const:`~hypothesis.infer` can be used as a keyword argument to explicitly -fill in an argument from its type annotation. +``...`` (:obj:`python:Ellipsis`), can be used as a keyword argument to explicitly fill +in an argument from its type annotation. You can also use the ``hypothesis.infer`` +alias if writing a literal ``...`` seems too weird. .. code:: python - @given(a=infer) + @given(a=...) # or @given(a=infer) def test(a: int): pass # is equivalent to - @given(a=integers()) + @given(a=from_type(int)) def test(a): pass + +``@given(...)`` can also be specified to fill all arguments from their type annotations. + +.. code:: python + + @given(...) + def test(a: int, b: str): + pass + + + # is equivalent to + @given(a=..., b=...) + def test(a, b): + pass + + ~~~~~~~~~~~ Limitations ~~~~~~~~~~~ @@ -599,10 +631,10 @@ Hypothesis does not inspect :pep:`484` type comments at runtime. While will only work if you manually create the ``__annotations__`` attribute (e.g. by using ``@annotations(...)`` and ``@returns(...)`` decorators). -The :mod:`python:typing` module is provisional and has a number of internal -changes between Python 3.5.0 and 3.6.1, including at minor versions. These -are all supported on a best-effort basis, but you may encounter problems with -an old version of the module. Please report them to us, and consider +The :mod:`python:typing` module changes between different Python releases, +including at minor versions. These +are all supported on a best-effort basis, +but you may encounter problems. Please report them to us, and consider updating to a newer version of Python as a workaround. @@ -683,10 +715,12 @@ providing extra information and convenient access to config options. :ref:`display test and data generation statistics `. - ``pytest --hypothesis-profile=`` can be used to :ref:`load a settings profile `. - ``pytest --hypothesis-verbosity=`` can be used to +- ``pytest --hypothesis-verbosity=`` can be used to :ref:`override the current verbosity level `. - ``pytest --hypothesis-seed=`` can be used to :ref:`reproduce a failure with a particular seed `. +- ``pytest --hypothesis-explain`` can be used to + :ref:`temporarily enable the explain phase `. Finally, all tests that are defined with Hypothesis automatically have ``@pytest.mark.hypothesis`` applied to them. See :ref:`here for information @@ -764,7 +798,7 @@ replay, shrink, deduplicate, and report whatever errors were discovered. - The :obj:`~hypothesis.settings.database` setting *is* used by fuzzing mode - adding failures to the database to be replayed when you next run your tests - is our preferred reporting mechanism and reponse to + is our preferred reporting mechanism and response to `the 'fuzzer taming' problem `__. - The :obj:`~hypothesis.settings.verbosity` and :obj:`~hypothesis.settings.stateful_step_count` settings work as usual. diff --git a/hypothesis-python/docs/development.rst b/hypothesis-python/docs/development.rst index f90a907cd6..551403722e 100644 --- a/hypothesis-python/docs/development.rst +++ b/hypothesis-python/docs/development.rst @@ -2,10 +2,12 @@ Ongoing Hypothesis development ============================== -Hypothesis development is managed by me, `David R. MacIver `_. -I am the primary author of Hypothesis. +Hypothesis development is managed by `David R. MacIver `_ +and `Zac Hatfield-Dodds `_, respectively the first author and lead +maintainer. -*However*, I no longer do unpaid feature development on Hypothesis. My roles as leader of the project are: +*However*, these roles don't include unpaid feature development on Hypothesis. +Our roles as leaders of the project are: 1. Helping other people do feature development on Hypothesis 2. Fixing bugs and other code health issues @@ -17,7 +19,8 @@ I am the primary author of Hypothesis. So all new features must either be sponsored or implemented by someone else. That being said, the maintenance team takes an active role in shepherding pull requests and helping people write a new feature (see :gh-file:`CONTRIBUTING.rst` for -details and :pull:`154` for an example of how the process goes). This isn't +details and :gh-link:`these examples of how the process goes +`). This isn't "patches welcome", it's "we will help you write a patch". diff --git a/hypothesis-python/docs/django.rst b/hypothesis-python/docs/django.rst index 4b40d859e7..e4202ca0b7 100644 --- a/hypothesis-python/docs/django.rst +++ b/hypothesis-python/docs/django.rst @@ -13,7 +13,9 @@ if you're still getting security patches, you can test with Hypothesis. Using it is quite straightforward: All you need to do is subclass :class:`hypothesis.extra.django.TestCase` or -:class:`hypothesis.extra.django.TransactionTestCase` +:class:`hypothesis.extra.django.TransactionTestCase` or +:class:`~hypothesis.extra.django.LiveServerTestCase` or +:class:`~hypothesis.extra.django.StaticLiveServerTestCase` and you can use :func:`@given ` as normal, and the transactions will be per example rather than per test function as they would be if you used :func:`@given ` with a normal @@ -23,6 +25,8 @@ on these classes that do not use :func:`@given ` will be run as normal. .. class:: hypothesis.extra.django.TransactionTestCase +.. class:: hypothesis.extra.django.LiveServerTestCase +.. class:: hypothesis.extra.django.StaticLiveServerTestCase We recommend avoiding :class:`~hypothesis.extra.django.TransactionTestCase` unless you really have to run each test case in a database transaction. @@ -37,8 +41,8 @@ a strategy for Django models: .. autofunction:: hypothesis.extra.django.from_model -For example, using `the trivial django project we have for testing -`_: +For example, using :gh-file:`the trivial django project we have for testing +`: .. code-block:: pycon @@ -67,11 +71,6 @@ that pass validation. Sometimes that will mean that we fail a :class:`~hypothesis.HealthCheck` because of the filtering, so let's explicitly pass a strategy to skip validation at the strategy level: -.. note:: - Inference from validators will be much more powerful when :issue:`1116` - is implemented, but there will always be some edge cases that require you - to pass an explicit strategy. - .. code-block:: pycon >>> from hypothesis.strategies import integers diff --git a/hypothesis-python/docs/endorsements.rst b/hypothesis-python/docs/endorsements.rst index 3c3e223f2c..c366793e1a 100644 --- a/hypothesis-python/docs/endorsements.rst +++ b/hypothesis-python/docs/endorsements.rst @@ -3,8 +3,8 @@ Testimonials ============ This is a page for listing people who are using Hypothesis and how excited they -are about that. If that's you and your name is not on the list, `this file is in -Git `_ +are about that. If that's you and your name is not on the list, +:gh-file:`this file is in Git ` and I'd love it if you sent me a pull request to fix that. --------------------------------------------------------------------------------------- @@ -116,7 +116,7 @@ Thank you, David, for the great testing tool. Hypothesis is the single most powerful tool in my toolbox for working with algorithmic code, or any software that produces predictable output from a wide range of sources. When using it with -`Priority `_, Hypothesis consistently found +`Priority `_, Hypothesis consistently found errors in my assumptions and extremely subtle bugs that would have taken months of real-world use to locate. In some cases, Hypothesis found subtle deviations from the correct output of the algorithm that may never have been noticed at diff --git a/hypothesis-python/docs/extras.rst b/hypothesis-python/docs/extras.rst index 9521adb4ff..1368febaf5 100644 --- a/hypothesis-python/docs/extras.rst +++ b/hypothesis-python/docs/extras.rst @@ -28,19 +28,19 @@ There are separate pages for :doc:`django` and :doc:`numpy`. .. automodule:: hypothesis.extra.dpcontracts :members: - .. tip:: +.. tip:: - For new projects, we recommend using either :pypi:`deal` or :pypi:`icontract` - and :pypi:`icontract-hypothesis` over :pypi:`dpcontracts`. - They're generally more powerful tools for design-by-contract programming, - and have substantially nicer Hypothesis integration too! + For new projects, we recommend using either :pypi:`deal` or :pypi:`icontract` + and :pypi:`icontract-hypothesis` over :pypi:`dpcontracts`. + They're generally more powerful tools for design-by-contract programming, + and have substantially nicer Hypothesis integration too! .. automodule:: hypothesis.extra.lark :members: Example grammars, which may provide a useful starting point for your tests, can be found `in the Lark repository `__ -and in `this third-party collection `__. +and in `this third-party collection `__. .. automodule:: hypothesis.extra.pytz :members: diff --git a/hypothesis-python/docs/healthchecks.rst b/hypothesis-python/docs/healthchecks.rst index 37316f8402..84f3b1040a 100644 --- a/hypothesis-python/docs/healthchecks.rst +++ b/hypothesis-python/docs/healthchecks.rst @@ -23,8 +23,7 @@ The argument for this parameter is a list with elements drawn from any of the class-level attributes of the HealthCheck class. Using a value of ``HealthCheck.all()`` will disable all health checks. -.. module:: hypothesis -.. autoclass:: HealthCheck +.. autoclass:: hypothesis.HealthCheck :undoc-members: :inherited-members: :exclude-members: all diff --git a/hypothesis-python/docs/numpy.rst b/hypothesis-python/docs/numpy.rst index 11b1426c5c..e707e13c43 100644 --- a/hypothesis-python/docs/numpy.rst +++ b/hypothesis-python/docs/numpy.rst @@ -8,7 +8,7 @@ Hypothesis for the scientific stack numpy ----- -Hypothesis offers a number of strategies for `NumPy `_ testing, +Hypothesis offers a number of strategies for `NumPy `_ testing, available in the ``hypothesis[numpy]`` :doc:`extra `. It lives in the ``hypothesis.extra.numpy`` package. @@ -47,8 +47,50 @@ Supported versions There is quite a lot of variation between pandas versions. We only commit to supporting the latest version of pandas, but older minor versions are supported on a "best effort" basis. Hypothesis is currently tested against -and confirmed working with every Pandas minor version from 0.25 through to 1.1. +and confirmed working with every Pandas minor version from 1.0 through to 1.5. Releases that are not the latest patch release of their minor version are not tested or officially supported, but will probably also work unless you hit a pandas bug. + +.. _array-api: + +--------- +Array API +--------- + +.. tip:: + The Array API standard is not yet finalised, so this module will make breaking + changes if necessary to support newer versions of the standard. + +Hypothesis offers strategies for `Array API `_ adopting +libraries in the ``hypothesis.extra.array_api`` package. See :issue:`3037` for +more details. If you want to test with :pypi:`CuPy`, :pypi:`Dask`, :pypi:`JAX`, +:pypi:`MXNet`, :pypi:`PyTorch `, :pypi:`TensorFlow`, or :pypi:`Xarray` - +or just ``numpy.array_api`` - this is the extension for you! + +.. autofunction:: hypothesis.extra.array_api.make_strategies_namespace + +The resulting namespace contains all our familiar strategies like +:func:`~xps.arrays` and :func:`~xps.from_dtype`, but based on the Array API +standard semantics and returning objects from the ``xp`` module: + +.. + TODO: for next released xp version, include complex_dtypes here + +.. automodule:: xps + :members: + from_dtype, + arrays, + array_shapes, + scalar_dtypes, + boolean_dtypes, + numeric_dtypes, + real_dtypes, + integer_dtypes, + unsigned_integer_dtypes, + floating_dtypes, + valid_tuple_axes, + broadcastable_shapes, + mutually_broadcastable_shapes, + indices, diff --git a/hypothesis-python/docs/packaging.rst b/hypothesis-python/docs/packaging.rst index 5bca356a6f..1aff697c4f 100644 --- a/hypothesis-python/docs/packaging.rst +++ b/hypothesis-python/docs/packaging.rst @@ -67,7 +67,7 @@ If you want to test Hypothesis as part of your packaging you will probably not w Hypothesis itself uses for running its tests, because it has a lot of logic for installing and testing against different versions of Python. -The tests must be run with fairly recent tooling; check the :gh-file:`requirements/` +The tests must be run with fairly recent tooling; check the :gh-link:`tree/master/requirements/` directory for details. The organisation of the tests is described in the :gh-file:`hypothesis-python/tests/README.rst`. @@ -76,6 +76,6 @@ The organisation of the tests is described in the :gh-file:`hypothesis-python/te Examples -------- -* `arch linux `_ +* `arch linux `_ * `fedora `_ * `gentoo `_ diff --git a/hypothesis-python/docs/quickstart.rst b/hypothesis-python/docs/quickstart.rst index ef74d4baf0..6f1d00ae5f 100644 --- a/hypothesis-python/docs/quickstart.rst +++ b/hypothesis-python/docs/quickstart.rst @@ -9,9 +9,8 @@ Hypothesis. An example ---------- -Suppose we've written a `run length encoding -`_ system and we want to test -it out. +Suppose we've written a :wikipedia:`run length encoding ` +system and we want to test it out. We have the following code which I took straight from the `Rosetta Code `_ wiki (OK, I @@ -87,7 +86,7 @@ Anyway, this test immediately finds a bug in the code: Hypothesis correctly points out that this code is simply wrong if called on an empty string. -If we fix that by just adding the following code to the beginning of the function +If we fix that by just adding the following code to the beginning of our ``encode`` function then Hypothesis tells us the code is correct (by doing nothing as you'd expect a passing test to). @@ -116,7 +115,8 @@ of data are valid inputs, or to ensure that particular edge cases such as although Hypothesis will :doc:`remember failing examples `, we don't recommend distributing that database. -It's also worth noting that both example and given support keyword arguments as +It's also worth noting that both :func:`@example ` and +:func:`@given ` support keyword arguments as well as positional. The following would have worked just as well: .. code:: python @@ -161,12 +161,6 @@ values are enough to set the count to a number different from one, followed by another distinct value which should have reset the count but in this case didn't. -The examples Hypothesis provides are valid Python code you can run. Any -arguments that you explicitly provide when calling the function are not -generated by Hypothesis, and if you explicitly provide *all* the arguments -Hypothesis will just call the underlying function once rather than -running it multiple times. - ---------- Installing ---------- diff --git a/hypothesis-python/docs/reproducing.rst b/hypothesis-python/docs/reproducing.rst index 6ef53efb02..17ee797544 100644 --- a/hypothesis-python/docs/reproducing.rst +++ b/hypothesis-python/docs/reproducing.rst @@ -57,7 +57,7 @@ styles will work as expected: @example("Hello world") @example(x="Some very long string") def test_some_code(x): - assert True + pass from unittest import TestCase @@ -68,7 +68,7 @@ styles will work as expected: @example("Hello world") @example(x="Some very long string") def test_some_code(self, x): - assert True + pass As with ``@given``, it is not permitted for a single example to be a mix of positional and keyword arguments. @@ -140,7 +140,7 @@ as follows: ... pass ... Falsifying example: test(f=nan) - + You can reproduce this example by temporarily adding @reproduce_failure(..., b'AAAA//AAAAAAAAEA') as a decorator on your test case Adding the suggested decorator to the test should reproduce the failure (as diff --git a/hypothesis-python/docs/settings.rst b/hypothesis-python/docs/settings.rst index cbdf77a686..757c782568 100644 --- a/hypothesis-python/docs/settings.rst +++ b/hypothesis-python/docs/settings.rst @@ -52,7 +52,7 @@ Available settings Controlling what runs ~~~~~~~~~~~~~~~~~~~~~ -Hypothesis divides tests into five logically distinct phases: +Hypothesis divides tests into logically distinct phases: 1. Running explicit examples :ref:`provided with the @example decorator `. 2. Rerunning a selection of previously failing examples to reproduce a previously seen error @@ -60,18 +60,17 @@ Hypothesis divides tests into five logically distinct phases: 4. Mutating examples for :ref:`targeted property-based testing `. 5. Attempting to shrink an example found in previous phases (other than phase 1 - explicit examples cannot be shrunk). This turns potentially large and complicated examples which may be hard to read into smaller and simpler ones. +6. Attempting to explain the cause of the failure, by identifying suspicious lines of code + (e.g. the earliest lines which are never run on passing inputs, and always run on failures). + This relies on :func:`python:sys.settrace`, and is therefore automatically disabled on + PyPy or if you are using :pypi:`coverage` or a debugger. If there are no clearly + suspicious lines of code, :pep:`we refuse the temptation to guess <20>`. The phases setting provides you with fine grained control over which of these run, with each phase corresponding to a value on the :class:`~hypothesis.Phase` enum: -.. class:: hypothesis.Phase - -1. ``Phase.explicit`` controls whether explicit examples are run. -2. ``Phase.reuse`` controls whether previous examples will be reused. -3. ``Phase.generate`` controls whether new examples will be generated. -4. ``Phase.target`` controls whether examples will be mutated for targeting. -5. ``Phase.shrink`` controls whether examples will be shrunk. - +.. autoclass:: hypothesis.Phase + :members: The phases argument accepts a collection with any subset of these. e.g. ``settings(phases=[Phase.generate, Phase.shrink])`` will generate new examples @@ -121,7 +120,7 @@ falsifying example. debug is basically verbose but a bit more so. You probably don't want it. If you are using :pypi:`pytest`, you may also need to -:doc:`disable output capturing for passing tests `. +:doc:`disable output capturing for passing tests `. ------------------------- Building settings objects diff --git a/hypothesis-python/docs/stateful.rst b/hypothesis-python/docs/stateful.rst index 0728899b41..c75005e797 100644 --- a/hypothesis-python/docs/stateful.rst +++ b/hypothesis-python/docs/stateful.rst @@ -9,12 +9,20 @@ not just data but entire tests. You specify a number of primitive actions that can be combined together, and then Hypothesis will try to find sequences of those actions that result in a failure. +.. tip:: + + Before reading this reference documentation, we recommend reading + `How not to Die Hard with Hypothesis `__ + and `An Introduction to Rule-Based Stateful Testing `__, + in that order. The implementation details will make more sense once you've seen + them used in practice, and know *why* each method or decorator is available. + .. note:: This style of testing is often called *model-based testing*, but in Hypothesis is called *stateful testing* (mostly for historical reasons - the original implementation of this idea in Hypothesis was more closely based on - `ScalaCheck's stateful testing `_ + `ScalaCheck's stateful testing `_ where the name is more apt). Both of these names are somewhat misleading: You don't really need any sort of formal model of your code to use this, and it can be just as useful for pure APIs diff --git a/hypothesis-python/docs/strategies.rst b/hypothesis-python/docs/strategies.rst index af1c1af83f..98522e1b2d 100644 --- a/hypothesis-python/docs/strategies.rst +++ b/hypothesis-python/docs/strategies.rst @@ -21,7 +21,7 @@ External strategies Some packages provide strategies directly: * :pypi:`hypothesis-fspaths` - strategy to generate filesystem paths. -* :pypi:`hypothesis-geojson` - strategy to generate `GeoJson `_. +* :pypi:`hypothesis-geojson` - strategy to generate `GeoJson `_. * :pypi:`hypothesis-geometry` - strategies to generate geometric objects. * :pypi:`hs-dbus-signature` - strategy to generate arbitrary `D-Bus signatures `_. @@ -72,7 +72,7 @@ concurrent programs. :pypi:`pymtl3` is "an open-source Python-based hardware generation, simulation, and verification framework with multi-level hardware modeling support", which ships with Hypothesis integrations to check that all of those levels are -eqivalent, from function-level to register-transfer level and even to hardware. +equivalent, from function-level to register-transfer level and even to hardware. :pypi:`libarchimedes` makes it easy to use Hypothesis in `the Hy language `_, a Lisp embedded in Python. @@ -128,13 +128,14 @@ test is using Hypothesis: .. _entry-points: -------------------------------------------------- -Registering strategies via setuptools entry points +Hypothesis integration via setuptools entry points -------------------------------------------------- If you would like to ship Hypothesis strategies for a custom type - either as part of the upstream library, or as a third-party extension, there's a catch: :func:`~hypothesis.strategies.from_type` only works after the corresponding -call to :func:`~hypothesis.strategies.register_type_strategy`. This means that +call to :func:`~hypothesis.strategies.register_type_strategy`, and you'll have +the same problem with :func:`~hypothesis.register_random`. This means that either - you have to try importing Hypothesis to register the strategy when *your* @@ -182,7 +183,7 @@ and then tell ``setuptools`` that this is your ``"hypothesis"`` entry point: And that's all it takes! .. note:: - On Python 3.6 and 3.7, where the ``importlib.metadata`` module + On Python 3.7, where the ``importlib.metadata`` module is not in the standard library, loading entry points requires either the :pypi:`importlib_metadata` (preferred) or :pypi:`setuptools` (fallback) package to be installed. diff --git a/hypothesis-python/docs/support.rst b/hypothesis-python/docs/support.rst index 286cfca958..e6ab532f99 100644 --- a/hypothesis-python/docs/support.rst +++ b/hypothesis-python/docs/support.rst @@ -10,7 +10,8 @@ sees them. For bugs and enhancements, please file an issue on the :issue:`GitHub issue tracker <>`. Note that as per the :doc:`development policy `, enhancements will probably not get -implemented unless you're willing to pay for development or implement them yourself (with assistance from me). Bugs +implemented unless you're willing to pay for development or implement them yourself +(with assistance from the maintainers). Bugs will tend to get fixed reasonably promptly, though it is of course on a best effort basis. To see the versions of Python, optional dependencies, test runners, and operating systems Hypothesis diff --git a/hypothesis-python/docs/supported.rst b/hypothesis-python/docs/supported.rst index 40f5f78715..08e26e88f2 100644 --- a/hypothesis-python/docs/supported.rst +++ b/hypothesis-python/docs/supported.rst @@ -25,11 +25,10 @@ changes in patch releases. Python versions --------------- -Hypothesis is supported and tested on CPython 3.6+, i.e. -`all versions of CPython with upstream support `_, - -Hypothesis also supports the latest PyPy for Python 3.6. -32-bit builds of CPython also work, though they are currently only tested on Windows. +Hypothesis is supported and tested on CPython 3.7+, i.e. +`all versions of CPython with upstream support `_, +along with PyPy for the same versions. +32-bit builds of CPython also work, though we only test them on Windows. In general Hypothesis does not officially support anything except the latest patch release of any version of Python it supports. Earlier releases should work @@ -80,7 +79,7 @@ In terms of what's actually *known* to work: and avoid poor interactions with Pytest fixtures. * Nose works fine with Hypothesis, and this is tested as part of the CI. ``yield`` based tests simply won't work. - * Integration with Django's testing requires use of the :ref:`hypothesis-django` package. + * Integration with Django's testing requires use of the :ref:`hypothesis-django` extra. The issue is that in Django's tests' normal mode of execution it will reset the database once per test rather than once per example, which is not what you want. * :pypi:`Coverage` works out of the box with Hypothesis; our own test suite has @@ -99,6 +98,6 @@ Regularly verifying this ------------------------ Everything mentioned above as explicitly supported is checked on every commit -with `GitHub Actions `__. +with :gh-link:`GitHub Actions `. Our continuous delivery pipeline runs all of these checks before publishing each release, so when we say they're supported we really mean it. diff --git a/hypothesis-python/docs/usage.rst b/hypothesis-python/docs/usage.rst index 71f2863f88..b916ae5334 100644 --- a/hypothesis-python/docs/usage.rst +++ b/hypothesis-python/docs/usage.rst @@ -8,21 +8,21 @@ The only inclusion criterion right now is that if it's a Python library then it should be available on PyPI. You can find hundreds more from `the Hypothesis page at libraries.io -`_, and `thousands on GitHub -`_. +`_, and :gh-file:`thousands on GitHub `. Hypothesis has `over 100,000 downloads per week `__, -and was used by `more than 4% of Python users surveyed by the PSF in 2018 -`__. +and was used by `more than 4% of Python users surveyed by the PSF in 2020 +`__. * `aur `_ -* `argon2_cffi `_ +* `argon2_cffi `_ +* `array-api-tests `_ * `attrs `_ * `axelrod `_ * `bidict `_ -* `binaryornot `_ -* `brotlipy `_ +* `binaryornot `_ +* `brotlicffi `_ * :pypi:`chardet` -* `cmph-cffi `_ +* :pypi:`cmph-cffi` * `cryptography `_ * `dbus-signature-pyparsing `_ * `dry-python/returns `_ @@ -31,29 +31,38 @@ and was used by `more than 4% of Python users surveyed by the PSF in 2018 * `flownetpy `_ * `funsize `_ * `fusion-index `_ -* `hyper-h2 `_ +* `hyper-h2 `_ * `into-dbus-python `_ +* `ivy `_ * `justbases `_ * `justbytes `_ * `loris `_ * `mariadb-dyncol `_ +* `MDAnalysis `_ * `mercurial `_ +* `napari `_ * `natsort `_ +* `numpy `_ +* `pandas `_ +* `pandera `_ * `poliastro `_ * `pretext `_ * `priority `_ * `PyCEbox `_ -* `PyPy `_ +* `PyPy `_ * `pyrsistent `_ * `python-humble-utils `_ * `pyudev `_ * `qutebrowser `_ * `RubyMarshal `_ * `Segpy `_ +* `sgkit `_ * `simoa `_ * `srt `_ * `tchannel `_ * `vdirsyncer `_ * `wcag-contrast-ratio `_ +* `xarray `_ * `yacluster `_ * `yturl `_ +* `zenml `_ diff --git a/hypothesis-python/examples/example_hypothesis_entrypoint/example_hypothesis_entrypoint.py b/hypothesis-python/examples/example_hypothesis_entrypoint/example_hypothesis_entrypoint.py index 4757b21714..89d77e8a9b 100644 --- a/hypothesis-python/examples/example_hypothesis_entrypoint/example_hypothesis_entrypoint.py +++ b/hypothesis-python/examples/example_hypothesis_entrypoint/example_hypothesis_entrypoint.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """This example demonstrates a setuptools entry point. diff --git a/hypothesis-python/examples/example_hypothesis_entrypoint/setup.py b/hypothesis-python/examples/example_hypothesis_entrypoint/setup.py index 1bf3b9a7f3..7f175f9771 100644 --- a/hypothesis-python/examples/example_hypothesis_entrypoint/setup.py +++ b/hypothesis-python/examples/example_hypothesis_entrypoint/setup.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """Minimal setup.py to register an entrypoint.""" @@ -25,7 +20,7 @@ description="Minimal setup.py to register an entrypoint.", packages=setuptools.find_packages(), install_requires=["hypothesis"], - python_requires=">=3.6", + python_requires=">=3.7", entry_points={ "hypothesis": ["_ = example_hypothesis_entrypoint:_hypothesis_setup_hook"] }, diff --git a/hypothesis-python/examples/example_hypothesis_entrypoint/test_entrypoint.py b/hypothesis-python/examples/example_hypothesis_entrypoint/test_entrypoint.py index 18c500c418..84c26bbb39 100644 --- a/hypothesis-python/examples/example_hypothesis_entrypoint/test_entrypoint.py +++ b/hypothesis-python/examples/example_hypothesis_entrypoint/test_entrypoint.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from example_hypothesis_entrypoint import MyCustomType diff --git a/hypothesis-python/examples/test_binary_search.py b/hypothesis-python/examples/test_binary_search.py index 6d4e8512fd..c264c95880 100644 --- a/hypothesis-python/examples/test_binary_search.py +++ b/hypothesis-python/examples/test_binary_search.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """This file demonstrates testing a binary search. diff --git a/hypothesis-python/examples/test_rle.py b/hypothesis-python/examples/test_rle.py index 541e1328a1..0094610a4d 100644 --- a/hypothesis-python/examples/test_rle.py +++ b/hypothesis-python/examples/test_rle.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """This example demonstrates testing a run length encoding scheme. That is, we take a sequence and represent it by a shorter sequence where each 'run' of diff --git a/hypothesis-python/pyrightconfig.json b/hypothesis-python/pyrightconfig.json new file mode 100644 index 0000000000..4cfcacc926 --- /dev/null +++ b/hypothesis-python/pyrightconfig.json @@ -0,0 +1,4 @@ +{ + "include": ["src"], + "typeCheckingMode": "strict" +} diff --git a/hypothesis-python/scripts/basic-test.sh b/hypothesis-python/scripts/basic-test.sh index 268399b639..8ad96b1f7f 100755 --- a/hypothesis-python/scripts/basic-test.sh +++ b/hypothesis-python/scripts/basic-test.sh @@ -13,7 +13,7 @@ for k, v in sorted(dict(os.environ).items()): pip install . -PYTEST="python -m pytest -n2" +PYTEST="python -bb -X dev -m pytest -nauto" # Run all the no-extra-dependency tests for this version (except slow nocover tests) $PYTEST tests/cover tests/pytest @@ -27,29 +27,36 @@ pip install ".[dpcontracts]" $PYTEST tests/dpcontracts/ pip uninstall -y dpcontracts -pip install fakeredis +pip install "$(grep 'fakeredis==' ../requirements/coverage.txt)" $PYTEST tests/redis/ pip uninstall -y redis fakeredis -pip install typing_extensions +pip install "$(grep 'typing-extensions==' ../requirements/coverage.txt)" $PYTEST tests/typing_extensions/ -if [ "$(python -c 'import sys; print(sys.version_info[:2] <= (3, 7))')" = "False" ] ; then +if [ "$(python -c 'import sys; print(sys.version_info[:2] == (3, 7))')" = "False" ] ; then # Required by importlib_metadata backport, which we don't want to break pip uninstall -y typing_extensions fi pip install ".[lark]" $PYTEST tests/lark/ -pip install lark-parser==0.7.1 +pip install "$(grep 'lark-parser==' ../requirements/coverage.txt)" $PYTEST tests/lark/ pip uninstall -y lark-parser -if [ "$(python -c 'import platform; print(platform.python_implementation() != "PyPy")')" = "True" ] ; then +if [ "$(python -c $'import platform, sys; print(sys.version_info.releaselevel == \'final\' and platform.python_implementation() != "PyPy")')" = "True" ] ; then pip install ".[codemods,cli]" $PYTEST tests/codemods/ pip uninstall -y libcst click - pip install black numpy + if [ "$(python -c 'import sys; print(sys.version_info[:2] == (3, 7))')" = "True" ] ; then + # Per NEP-29, this is the last version to support Python 3.7 + pip install numpy==1.21.5 + else + pip install "$(grep 'numpy==' ../requirements/coverage.txt)" + fi + + pip install "$(grep 'black==' ../requirements/coverage.txt)" $PYTEST tests/ghostwriter/ pip uninstall -y black numpy fi @@ -62,7 +69,10 @@ $PYTEST tests/nocover/ # Run some tests without docstrings or assertions, to catch bugs # like issue #822 in one of the test decorators. See also #1541. -PYTHONOPTIMIZE=2 $PYTEST tests/cover/test_testdecorators.py +PYTHONOPTIMIZE=2 $PYTEST \ + -W'ignore:assertions not in test modules or plugins will be ignored because assert statements are not executed by the underlying Python interpreter:pytest.PytestConfigWarning' \ + -W'ignore:Module already imported so cannot be rewritten:pytest.PytestAssertRewriteWarning' \ + tests/cover/test_testdecorators.py if [ "$(python -c 'import platform; print(platform.python_implementation())')" != "PyPy" ]; then pip install .[django] @@ -70,9 +80,10 @@ if [ "$(python -c 'import platform; print(platform.python_implementation())')" ! HYPOTHESIS_DJANGO_USETZ=FALSE python -m tests.django.manage test tests.django pip uninstall -y django pytz - pip install numpy + pip install "$(grep 'numpy==' ../requirements/coverage.txt)" + $PYTEST tests/array_api $PYTEST tests/numpy - pip install pandas + pip install "$(grep 'pandas==' ../requirements/coverage.txt)" $PYTEST tests/pandas fi diff --git a/hypothesis-python/scripts/validate_branch_check.py b/hypothesis-python/scripts/validate_branch_check.py index a180d7c1f5..e97e097f3f 100644 --- a/hypothesis-python/scripts/validate_branch_check.py +++ b/hypothesis-python/scripts/validate_branch_check.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import json import sys @@ -26,6 +21,10 @@ for d in data: checks[d["name"]].add(d["value"]) + if not checks: + print("No branches found in the branch-check file?") + sys.exit(1) + always_true = [] always_false = [] @@ -42,17 +41,20 @@ if failure: print("Some branches were not properly covered.") - print() if always_true: - print("The following were always True:") print() + print("The following were always True:") for c in always_true: print(f" * {c}") if always_false: - print("The following were always False:") print() + print("The following were always False:") for c in always_false: print(f" * {c}") if failure: sys.exit(1) + + print( + f"""Successfully validated {len(checks)} branch{"es" if len(checks) > 1 else ""}.""" + ) diff --git a/hypothesis-python/setup.cfg b/hypothesis-python/setup.cfg index 0e358611ad..a151fbeff0 100644 --- a/hypothesis-python/setup.cfg +++ b/hypothesis-python/setup.cfg @@ -1,3 +1,3 @@ [metadata] # This includes the license file in the wheel. -license_file = LICENSE.txt +license_files = LICENSE.txt diff --git a/hypothesis-python/setup.py b/hypothesis-python/setup.py index ce07a91bac..62b5180018 100644 --- a/hypothesis-python/setup.py +++ b/hypothesis-python/setup.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import os import sys @@ -19,9 +14,10 @@ import setuptools -if sys.version_info[:2] < (3, 6): +if sys.version_info[:2] < (3, 7): raise Exception( - "This version of Python is too old to install new versions of Hypothesis. " + "You are trying to install Hypothesis using Python " + f"{sys.version.split()[0]}, but it requires Python 3.7 or later." "Update `pip` and `setuptools`, try again, and you will automatically " "get the latest compatible version of Hypothesis instead. " "See also https://python3statement.org/practicalities/" @@ -35,19 +31,18 @@ def local_file(name): SOURCE = local_file("src") README = local_file("README.rst") -setuptools_version = tuple(map(int, setuptools.__version__.split(".")[:2])) +setuptools_version = tuple(map(int, setuptools.__version__.split(".")[:1])) -if setuptools_version < (36, 2): +if setuptools_version < (42,): # Warning only - very bad if uploading bdist but fine if installing sdist. warnings.warn( - "This version of setuptools is too old to correctly store " - "conditional dependencies in binary wheels. For more info, see: " - "https://hynek.me/articles/conditional-python-dependencies/" + "This version of setuptools is too old to handle license_files " + "metadata key. For more info, see: " + "https://setuptools.pypa.io/en/latest/userguide/declarative_config.html#metadata" ) -# Assignment to placate pyflakes. The actual version is from the exec that -# follows. +# Assignment to placate pyflakes. The actual version is from the exec that follows. __version__ = None with open(local_file("src/hypothesis/version.py")) as o: @@ -57,54 +52,60 @@ def local_file(name): extras = { - "cli": ["click>=7.0", "black>=19.10b0"], + "cli": ["click>=7.0", "black>=19.10b0", "rich>=9.0.0"], "codemods": ["libcst>=0.3.16"], "ghostwriter": ["black>=19.10b0"], "pytz": ["pytz>=2014.1"], "dateutil": ["python-dateutil>=1.4"], "lark": ["lark-parser>=0.6.5"], "numpy": ["numpy>=1.9.0"], - "pandas": ["pandas>=0.25"], + "pandas": ["pandas>=1.0"], "pytest": ["pytest>=4.6"], "dpcontracts": ["dpcontracts>=0.4"], "redis": ["redis>=3.0.0"], # zoneinfo is an odd one: every dependency is conditional, because they're # only necessary on old versions of Python or Windows systems. "zoneinfo": [ - "tzdata>=2020.4 ; sys_platform == 'win32'", + "tzdata>=2022.6 ; sys_platform == 'win32'", "backports.zoneinfo>=0.2.1 ; python_version<'3.9'", - "importlib_resources>=3.3.0 ; python_version<'3.7'", ], # We only support Django versions with upstream support - see # https://www.djangoproject.com/download/#supported-versions - "django": ["pytz>=2014.1", "django>=2.2"], + # We also leave the choice of timezone library to the user, since it + # might be zoneinfo or pytz depending on version and configuration. + "django": ["django>=3.2"], } extras["all"] = sorted( - set(sum(extras.values(), ["importlib_metadata ; python_version<'3.8'"])) + set(sum(extras.values(), ["importlib_metadata>=3.6; python_version<'3.8'"])) ) setuptools.setup( name="hypothesis", version=__version__, - author="David R. MacIver", + author="David R. MacIver and Zac Hatfield-Dodds", author_email="david@drmaciver.com", packages=setuptools.find_packages(SOURCE), package_dir={"": SOURCE}, package_data={"hypothesis": ["py.typed", "vendor/tlds-alpha-by-domain.txt"]}, - url="https://github.com/HypothesisWorks/hypothesis/tree/master/hypothesis-python", + url="https://hypothesis.works", project_urls={ - "Website": "https://hypothesis.works", + "Source": "https://github.com/HypothesisWorks/hypothesis/tree/master/hypothesis-python", + "Changelog": "https://hypothesis.readthedocs.io/en/latest/changes.html", "Documentation": "https://hypothesis.readthedocs.io", "Issues": "https://github.com/HypothesisWorks/hypothesis/issues", }, - license="MPL v2", + license="MPL-2.0", description="A library for property-based testing", zip_safe=False, extras_require=extras, - install_requires=["attrs>=19.2.0", "sortedcontainers>=2.1.0,<3.0.0"], - python_requires=">=3.6", + install_requires=[ + "attrs>=19.2.0", + "exceptiongroup>=1.0.0 ; python_version<'3.11'", + "sortedcontainers>=2.1.0,<3.0.0", + ], + python_requires=">=3.7", classifiers=[ "Development Status :: 5 - Production/Stable", "Framework :: Hypothesis", @@ -117,17 +118,19 @@ def local_file(name): "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Education :: Testing", "Topic :: Software Development :: Testing", "Typing :: Typed", ], + py_modules=["_hypothesis_pytestplugin", "_hypothesis_ftz_detector"], entry_points={ - "pytest11": ["hypothesispytest = hypothesis.extra.pytestplugin"], + "pytest11": ["hypothesispytest = _hypothesis_pytestplugin"], "console_scripts": ["hypothesis = hypothesis.extra.cli:main"], }, long_description=open(README).read(), diff --git a/hypothesis-python/src/_hypothesis_ftz_detector.py b/hypothesis-python/src/_hypothesis_ftz_detector.py new file mode 100644 index 0000000000..0a0378ae83 --- /dev/null +++ b/hypothesis-python/src/_hypothesis_ftz_detector.py @@ -0,0 +1,149 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +""" +This is a toolkit for determining which module set the "flush to zero" flag. + +For details, see the docstring and comments in `identify_ftz_culprit()`. This module +is defined outside the main Hypothesis namespace so that we can avoid triggering +import of Hypothesis itself from each subprocess which must import the worker function. +""" + +import importlib +import sys + +KNOWN_EVER_CULPRITS = ( + # https://moyix.blogspot.com/2022/09/someones-been-messing-with-my-subnormals.html + # fmt: off + "archive-pdf-tools", "bgfx-python", "bicleaner-ai-glove", "BTrees", "cadbiom", + "ctranslate2", "dyNET", "dyNET38", "gevent", "glove-python-binary", "higra", + "hybridq", "ikomia", "ioh", "jij-cimod", "lavavu", "lavavu-osmesa", "MulticoreTSNE", + "neural-compressor", "nwhy", "openjij", "openturns", "perfmetrics", "pHashPy", + "pyace-lite", "pyapr", "pycompadre", "pycompadre-serial", "PyKEP", "pykep", + "pylimer-tools", "pyqubo", "pyscf", "PyTAT", "python-prtree", "qiskit-aer", + "qiskit-aer-gpu", "RelStorage", "sail-ml", "segmentation", "sente", "sinr", + "snapml", "superman", "symengine", "systran-align", "texture-tool", "tsne-mp", + "xcsf", + # fmt: on +) + + +def flush_to_zero(): + # If this subnormal number compares equal to zero we have a problem + return 2.0**-1073 == 0 + + +def run_in_process(fn, *args): + import multiprocessing as mp + + mp.set_start_method("spawn", force=True) + q = mp.Queue() + p = mp.Process(target=target, args=(q, fn, *args)) + p.start() + retval = q.get() + p.join() + return retval + + +def target(q, fn, *args): + q.put(fn(*args)) + + +def always_imported_modules(): + return flush_to_zero(), set(sys.modules) + + +def modules_imported_by(mod): + """Return the set of modules imported transitively by mod.""" + before = set(sys.modules) + try: + importlib.import_module(mod) + except Exception: + return None, set() + imports = set(sys.modules) - before + return flush_to_zero(), imports + + +# We don't want to redo all the expensive process-spawning checks when we've already +# done them, so we cache known-good packages and a known-FTZ result if we have one. +KNOWN_FTZ = None +CHECKED_CACHE = set() + + +def identify_ftz_culprits(): + """Find the modules in sys.modules which cause "mod" to be imported.""" + # If we've run this function before, return the same result. + global KNOWN_FTZ + if KNOWN_FTZ: + return KNOWN_FTZ + # Start by determining our baseline: the FTZ and sys.modules state in a fresh + # process which has only imported this module and nothing else. + always_enables_ftz, always_imports = run_in_process(always_imported_modules) + if always_enables_ftz: + raise RuntimeError("Python is always in FTZ mode, even without imports!") + CHECKED_CACHE.update(always_imports) + + # Next, we'll search through sys.modules looking for a package (or packages) such + # that importing them in a new process sets the FTZ state. As a heuristic, we'll + # start with packages known to have ever enabled FTZ, then top-level packages as + # a way to eliminate large fractions of the search space relatively quickly. + def key(name): + """Prefer known-FTZ modules, then top-level packages, then alphabetical.""" + return (name not in KNOWN_EVER_CULPRITS, name.count("."), name) + + # We'll track the set of modules to be checked, and those which do trigger FTZ. + candidates = set(sys.modules) - CHECKED_CACHE + triggering_modules = {} + while candidates: + mod = min(candidates, key=key) + candidates.discard(mod) + enables_ftz, imports = run_in_process(modules_imported_by, mod) + imports -= CHECKED_CACHE + if enables_ftz: + triggering_modules[mod] = imports + candidates &= imports + else: + candidates -= imports + CHECKED_CACHE.update(imports) + + # We only want to report the 'top level' packages which enable FTZ - for example, + # if the enabling code is in `a.b`, and `a` in turn imports `a.b`, we prefer to + # report `a`. On the other hand, if `a` does _not_ import `a.b`, as is the case + # for `hypothesis.extra.*` modules, then `a` will not be in `triggering_modules` + # and we'll report `a.b` here instead. + prefixes = tuple(n + "." for n in triggering_modules) + result = {k for k in triggering_modules if not k.startswith(prefixes)} + + # Suppose that `bar` enables FTZ, and `foo` imports `bar`. At this point we're + # tracking both, but only want to report the latter. + for a in sorted(result): + for b in sorted(result): + if a in triggering_modules[b] and b not in triggering_modules[a]: + result.discard(b) + + # There may be a cyclic dependency which that didn't handle, or simply two + # separate modules which both enable FTZ. We already gave up comprehensive + # reporting for speed above (`candidates &= imports`), so we'll also buy + # simpler reporting by arbitrarily selecting the alphabetically first package. + KNOWN_FTZ = min(result) # Cache the result - it's likely this will trigger again! + return KNOWN_FTZ + + +if __name__ == "__main__": + # This would be really really annoying to write automated tests for, so I've + # done some manual exploratory testing: `pip install grequests gevent==21.12.0`, + # and call print() as desired to observe behavior. + import grequests # noqa + + # To test without skipping to a known answer, uncomment the following line and + # change the last element of key from `name` to `-len(name)` so that we check + # grequests before gevent. + ## KNOWN_EVER_CULPRITS = [c for c in KNOWN_EVER_CULPRITS if c != "gevent"] + print(identify_ftz_culprits()) diff --git a/hypothesis-python/src/_hypothesis_pytestplugin.py b/hypothesis-python/src/_hypothesis_pytestplugin.py new file mode 100644 index 0000000000..afafd52ce4 --- /dev/null +++ b/hypothesis-python/src/_hypothesis_pytestplugin.py @@ -0,0 +1,371 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +""" +The pytest plugin for Hypothesis. + +We move this from the old location at `hypothesis.extra.pytestplugin` so that it +can be loaded by Pytest without importing Hypothesis. In turn, this means that +Hypothesis will not load our own third-party plugins (with associated side-effects) +unless and until the user explicitly runs `import hypothesis`. + +See https://github.com/HypothesisWorks/hypothesis/issues/3140 for details. +""" + +import base64 +import sys +from inspect import signature + +import pytest + +try: + from _pytest.junitxml import xml_key +except ImportError: + xml_key = "_xml" # type: ignore + +LOAD_PROFILE_OPTION = "--hypothesis-profile" +VERBOSITY_OPTION = "--hypothesis-verbosity" +PRINT_STATISTICS_OPTION = "--hypothesis-show-statistics" +SEED_OPTION = "--hypothesis-seed" +EXPLAIN_OPTION = "--hypothesis-explain" + +_VERBOSITY_NAMES = ["quiet", "normal", "verbose", "debug"] +_ALL_OPTIONS = [ + LOAD_PROFILE_OPTION, + VERBOSITY_OPTION, + PRINT_STATISTICS_OPTION, + SEED_OPTION, + EXPLAIN_OPTION, +] +_FIXTURE_MSG = """Function-scoped fixture {0!r} used by {1!r} + +Function-scoped fixtures are not reset between examples generated by +`@given(...)`, which is often surprising and can cause subtle test bugs. + +If you were expecting the fixture to run separately for each generated example, +then unfortunately you will need to find a different way to achieve your goal +(e.g. using a similar context manager instead of a fixture). + +If you are confident that your test will work correctly even though the +fixture is not reset between generated examples, you can suppress this health +check to assure Hypothesis that you understand what you are doing. +""" + +STATS_KEY = "_hypothesis_stats" + + +class StoringReporter: + def __init__(self, config): + assert "hypothesis" in sys.modules + from hypothesis.reporting import default + + self.report = default + self.config = config + self.results = [] + + def __call__(self, msg): + if self.config.getoption("capture", "fd") == "no": + self.report(msg) + if not isinstance(msg, str): + msg = repr(msg) + self.results.append(msg) + + +# Avoiding distutils.version.LooseVersion due to +# https://github.com/HypothesisWorks/hypothesis/issues/2490 +if tuple(map(int, pytest.__version__.split(".")[:2])) < (4, 6): # pragma: no cover + import warnings + + PYTEST_TOO_OLD_MESSAGE = """ + You are using pytest version %s. Hypothesis tests work with any test + runner, but our pytest plugin requires pytest 4.6 or newer. + Note that the pytest developers no longer support your version either! + Disabling the Hypothesis pytest plugin... + """ + warnings.warn(PYTEST_TOO_OLD_MESSAGE % (pytest.__version__,)) + +else: + + def pytest_addoption(parser): + group = parser.getgroup("hypothesis", "Hypothesis") + group.addoption( + LOAD_PROFILE_OPTION, + action="store", + help="Load in a registered hypothesis.settings profile", + ) + group.addoption( + VERBOSITY_OPTION, + action="store", + choices=_VERBOSITY_NAMES, + help="Override profile with verbosity setting specified", + ) + group.addoption( + PRINT_STATISTICS_OPTION, + action="store_true", + help="Configure when statistics are printed", + default=False, + ) + group.addoption( + SEED_OPTION, + action="store", + help="Set a seed to use for all Hypothesis tests", + ) + group.addoption( + EXPLAIN_OPTION, + action="store_true", + help="Enable the `explain` phase for failing Hypothesis tests", + default=False, + ) + + def _any_hypothesis_option(config): + return bool(any(config.getoption(opt) for opt in _ALL_OPTIONS)) + + def pytest_report_header(config): + if not ( + config.option.verbose >= 1 + or "hypothesis" in sys.modules + or _any_hypothesis_option(config) + ): + return None + + from hypothesis import Verbosity, settings + + if config.option.verbose < 1 and settings.default.verbosity < Verbosity.verbose: + return None + settings_str = settings.default.show_changed() + if settings_str != "": + settings_str = f" -> {settings_str}" + return f"hypothesis profile {settings._current_profile!r}{settings_str}" + + def pytest_configure(config): + config.addinivalue_line("markers", "hypothesis: Tests which use hypothesis.") + if not _any_hypothesis_option(config): + return + from hypothesis import Phase, Verbosity, core, settings + + profile = config.getoption(LOAD_PROFILE_OPTION) + if profile: + settings.load_profile(profile) + verbosity_name = config.getoption(VERBOSITY_OPTION) + if verbosity_name and verbosity_name != settings.default.verbosity.name: + verbosity_value = Verbosity[verbosity_name] + name = f"{settings._current_profile}-with-{verbosity_name}-verbosity" + # register_profile creates a new profile, exactly like the current one, + # with the extra values given (in this case 'verbosity') + settings.register_profile(name, verbosity=verbosity_value) + settings.load_profile(name) + if ( + config.getoption(EXPLAIN_OPTION) + and Phase.explain not in settings.default.phases + ): + name = f"{settings._current_profile}-with-explain-phase" + phases = settings.default.phases + (Phase.explain,) + settings.register_profile(name, phases=phases) + settings.load_profile(name) + + seed = config.getoption(SEED_OPTION) + if seed is not None: + try: + seed = int(seed) + except ValueError: + pass + core.global_force_seed = seed + + @pytest.hookimpl(hookwrapper=True) + def pytest_runtest_call(item): + __tracebackhide__ = True + if not (hasattr(item, "obj") and "hypothesis" in sys.modules): + yield + return + + from hypothesis import core + from hypothesis.internal.detection import is_hypothesis_test + + ## See https://github.com/pytest-dev/pytest/issues/9159 + # TODO: add `pytest_version >= (7, 2) or` once the issue above is fixed. + core.pytest_shows_exceptiongroups = ( + item.config.getoption("tbstyle", "auto") == "native" + ) + core.running_under_pytest = True + + if not is_hypothesis_test(item.obj): + # If @given was not applied, check whether other hypothesis + # decorators were applied, and raise an error if they were. + # We add this frame of indirection to enable __tracebackhide__. + def raise_hypothesis_usage_error(msg): + raise InvalidArgument(msg) + + if getattr(item.obj, "is_hypothesis_strategy_function", False): + from hypothesis.errors import InvalidArgument + + raise_hypothesis_usage_error( + f"{item.nodeid} is a function that returns a Hypothesis strategy, " + "but pytest has collected it as a test function. This is useless " + "as the function body will never be executed. To define a test " + "function, use @given instead of @composite." + ) + message = "Using `@%s` on a test without `@given` is completely pointless." + for name, attribute in [ + ("example", "hypothesis_explicit_examples"), + ("seed", "_hypothesis_internal_use_seed"), + ("settings", "_hypothesis_internal_settings_applied"), + ("reproduce_example", "_hypothesis_internal_use_reproduce_failure"), + ]: + if hasattr(item.obj, attribute): + from hypothesis.errors import InvalidArgument + + raise_hypothesis_usage_error(message % (name,)) + yield + else: + from hypothesis import HealthCheck, settings + from hypothesis.internal.escalation import current_pytest_item + from hypothesis.internal.healthcheck import fail_health_check + from hypothesis.reporting import with_reporter + from hypothesis.statistics import collector, describe_statistics + + # Retrieve the settings for this test from the test object, which + # is normally a Hypothesis wrapped_test wrapper. If this doesn't + # work, the test object is probably something weird + # (e.g a stateful test wrapper), so we skip the function-scoped + # fixture check. + settings = getattr(item.obj, "_hypothesis_internal_use_settings", None) + + # Check for suspicious use of function-scoped fixtures, but only + # if the corresponding health check is not suppressed. + if ( + settings is not None + and HealthCheck.function_scoped_fixture + not in settings.suppress_health_check + ): + # Warn about function-scoped fixtures, excluding autouse fixtures because + # the advice is probably not actionable and the status quo seems OK... + # See https://github.com/HypothesisWorks/hypothesis/issues/377 for detail. + argnames = None + for fx_defs in item._request._fixturemanager.getfixtureinfo( + node=item, func=item.function, cls=None + ).name2fixturedefs.values(): + if argnames is None: + argnames = frozenset(signature(item.function).parameters) + for fx in fx_defs: + if fx.argname in argnames: + active_fx = item._request._get_active_fixturedef(fx.argname) + if active_fx.scope == "function": + fail_health_check( + settings, + _FIXTURE_MSG.format(fx.argname, item.nodeid), + HealthCheck.function_scoped_fixture, + ) + + if item.get_closest_marker("parametrize") is not None: + # Give every parametrized test invocation a unique database key + key = item.nodeid.encode() + item.obj.hypothesis.inner_test._hypothesis_internal_add_digest = key + + store = StoringReporter(item.config) + + def note_statistics(stats): + stats["nodeid"] = item.nodeid + item.hypothesis_statistics = describe_statistics(stats) + + with collector.with_value(note_statistics): + with with_reporter(store): + with current_pytest_item.with_value(item): + yield + if store.results: + item.hypothesis_report_information = list(store.results) + + def _stash_get(config, key, default): + if hasattr(config, "stash"): + # pytest 7 + return config.stash.get(key, default) + elif hasattr(config, "_store"): + # pytest 5.4 + return config._store.get(key, default) + else: + return getattr(config, key, default) + + @pytest.hookimpl(hookwrapper=True) + def pytest_runtest_makereport(item, call): + report = (yield).get_result() + if hasattr(item, "hypothesis_report_information"): + report.sections.append( + ("Hypothesis", "\n".join(item.hypothesis_report_information)) + ) + if hasattr(item, "hypothesis_statistics") and report.when == "teardown": + stats = item.hypothesis_statistics + stats_base64 = base64.b64encode(stats.encode()).decode() + + name = "hypothesis-statistics-" + item.nodeid + + # Include hypothesis information to the junit XML report. + # + # Note that when `pytest-xdist` is enabled, `xml_key` is not present in the + # stash, so we don't add anything to the junit XML report in that scenario. + # https://github.com/pytest-dev/pytest/issues/7767#issuecomment-1082436256 + xml = _stash_get(item.config, xml_key, None) + if xml: + xml.add_global_property(name, stats_base64) + + # If there's a terminal report, include our summary stats for each test + terminalreporter = item.config.pluginmanager.getplugin("terminalreporter") + if terminalreporter is not None: + # ideally, we would store this on terminalreporter.config.stash, but + # pytest-xdist doesn't copy that back to the controller + report.__dict__[STATS_KEY] = stats + + # If there's an HTML report, include our summary stats for each test + pytest_html = item.config.pluginmanager.getplugin("html") + if pytest_html is not None: # pragma: no cover + report.extra = getattr(report, "extra", []) + [ + pytest_html.extras.text(stats, name="Hypothesis stats") + ] + + def pytest_terminal_summary(terminalreporter): + if terminalreporter.config.getoption(PRINT_STATISTICS_OPTION): + terminalreporter.section("Hypothesis Statistics") + for reports in terminalreporter.stats.values(): + for report in reports: + stats = report.__dict__.get(STATS_KEY) + if stats: + terminalreporter.write_line(stats + "\n\n") + + def pytest_collection_modifyitems(items): + if "hypothesis" not in sys.modules: + return + + from hypothesis.internal.detection import is_hypothesis_test + + for item in items: + if isinstance(item, pytest.Function) and is_hypothesis_test(item.obj): + item.add_marker("hypothesis") + + # Monkeypatch some internals to prevent applying @pytest.fixture() to a + # function which has already been decorated with @hypothesis.given(). + # (the reverse case is already an explicit error in Hypothesis) + # We do this here so that it catches people on old Pytest versions too. + from _pytest import fixtures + + def _ban_given_call(self, function): + if "hypothesis" in sys.modules: + from hypothesis.internal.detection import is_hypothesis_test + + if is_hypothesis_test(function): + raise RuntimeError( + f"Can't apply @pytest.fixture() to {function.__name__} because " + "it is already decorated with @hypothesis.given()" + ) + return _orig_call(self, function) + + _orig_call = fixtures.FixtureFunctionMarker.__call__ + fixtures.FixtureFunctionMarker.__call__ = _ban_given_call # type: ignore + + +def load(): + """Required for `pluggy` to load a plugin from setuptools entrypoints.""" diff --git a/hypothesis-python/src/hypothesis/__init__.py b/hypothesis-python/src/hypothesis/__init__.py index 5bdf8dfe5d..e8a3873140 100644 --- a/hypothesis-python/src/hypothesis/__init__.py +++ b/hypothesis-python/src/hypothesis/__init__.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """Hypothesis is a library for writing unit tests which are parametrized by some source of data. diff --git a/hypothesis-python/src/hypothesis/_error_if_old.py b/hypothesis-python/src/hypothesis/_error_if_old.py index 3cc3df73f6..176c74e27b 100644 --- a/hypothesis-python/src/hypothesis/_error_if_old.py +++ b/hypothesis-python/src/hypothesis/_error_if_old.py @@ -1,28 +1,23 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import sys from hypothesis.version import __version__ message = """ -Hypothesis {} requires Python 3.6 or later. +Hypothesis {} requires Python 3.7 or later. This can only happen if your packaging toolchain is older than python_requires. See https://packaging.python.org/guides/distributing-packages-using-setuptools/ """ -if sys.version_info[:3] < (3, 6): # pragma: no cover +if sys.version_info[:2] < (3, 7): # pragma: no cover raise Exception(message.format(__version__)) diff --git a/hypothesis-python/src/hypothesis/_settings.py b/hypothesis-python/src/hypothesis/_settings.py index 62fd858c0e..4743a666fd 100644 --- a/hypothesis-python/src/hypothesis/_settings.py +++ b/hypothesis-python/src/hypothesis/_settings.py @@ -1,19 +1,14 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER -"""A module controlling settings for Hypothesis to use in falsification. +"""The settings module configures runtime options for Hypothesis. Either an explicit settings object can be used or the default object on this module can be modified. @@ -25,7 +20,7 @@ import os import warnings from enum import Enum, IntEnum, unique -from typing import TYPE_CHECKING, Any, Collection, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Collection, Dict, List, Optional, TypeVar, Union import attr @@ -44,7 +39,9 @@ __all__ = ["settings"] -all_settings = {} # type: Dict[str, Setting] +all_settings: Dict[str, "Setting"] = {} + +T = TypeVar("T") class settingsProperty: @@ -90,11 +87,11 @@ def __doc__(self): class settingsMeta(type): - def __init__(self, *args, **kwargs): + def __init__(cls, *args, **kwargs): super().__init__(*args, **kwargs) @property - def default(self): + def default(cls): v = default_variable.value if v is not None: return v @@ -103,10 +100,10 @@ def default(self): assert default_variable.value is not None return default_variable.value - def _assign_default_internal(self, value): + def _assign_default_internal(cls, value): default_variable.value = value - def __setattr__(self, name, value): + def __setattr__(cls, name, value): if name == "default": raise AttributeError( "Cannot assign to the property settings.default - " @@ -119,20 +116,19 @@ def __setattr__(self, name, value): "settings with settings.load_profile, or use @settings(...) " "to decorate your test instead." ) - return type.__setattr__(self, name, value) + return super().__setattr__(name, value) class settings(metaclass=settingsMeta): - """A settings object controls a variety of parameters that are used in - falsification. These may control both the falsification strategy and the - details of the data that is generated. + """A settings object configures options including verbosity, runtime controls, + persistence, determinism, and more. Default values are picked up from the settings.default object and changes made there will be picked up in newly created settings. """ __definitions_are_locked = False - _profiles = {} # type: dict + _profiles: Dict[str, "settings"] = {} __module__ = "hypothesis" def __getattr__(self, name): @@ -151,19 +147,19 @@ def __init__( # The intended use is "like **kwargs, but more tractable for tooling". max_examples: int = not_set, # type: ignore derandomize: bool = not_set, # type: ignore - database: Union[None, "ExampleDatabase"] = not_set, # type: ignore + database: Optional["ExampleDatabase"] = not_set, # type: ignore verbosity: "Verbosity" = not_set, # type: ignore phases: Collection["Phase"] = not_set, # type: ignore stateful_step_count: int = not_set, # type: ignore report_multiple_bugs: bool = not_set, # type: ignore suppress_health_check: Collection["HealthCheck"] = not_set, # type: ignore - deadline: Union[None, int, float, datetime.timedelta] = not_set, # type: ignore + deadline: Union[int, float, datetime.timedelta, None] = not_set, # type: ignore print_blob: bool = not_set, # type: ignore ) -> None: if parent is not None: check_type(settings, parent, "parent") if derandomize not in (not_set, False): - if database not in (not_set, None): + if database not in (not_set, None): # type: ignore raise InvalidArgument( "derandomize=True implies database=None, so passing " f"database={database!r} too is invalid." @@ -181,12 +177,18 @@ def __init__( else: object.__setattr__(self, setting.name, setting.validator(value)) - def __call__(self, test): + def __call__(self, test: T) -> T: """Make the settings object (self) an attribute of the test. The settings are later discovered by looking them up on the test itself. """ - if not callable(test): + # Aliasing as Any avoids mypy errors (attr-defined) when accessing and + # setting custom attributes on the decorated function or class. + _test: Any = test + + # Using the alias here avoids a mypy error (return-value) later when + # ``test`` is returned, because this check results in type refinement. + if not callable(_test): raise InvalidArgument( "settings objects can be called as a decorator with @given, " f"but decorated test={test!r} is not callable." @@ -194,7 +196,7 @@ def __call__(self, test): if inspect.isclass(test): from hypothesis.stateful import RuleBasedStateMachine - if issubclass(test, RuleBasedStateMachine): + if issubclass(_test, RuleBasedStateMachine): attr_name = "_hypothesis_internal_settings_applied" if getattr(test, attr_name, False): raise InvalidArgument( @@ -203,25 +205,25 @@ def __call__(self, test): "instead." ) setattr(test, attr_name, True) - test.TestCase.settings = self - return test + _test.TestCase.settings = self + return test # type: ignore else: raise InvalidArgument( "@settings(...) can only be used as a decorator on " "functions, or on subclasses of RuleBasedStateMachine." ) - if hasattr(test, "_hypothesis_internal_settings_applied"): + if hasattr(_test, "_hypothesis_internal_settings_applied"): # Can't use _hypothesis_internal_use_settings as an indicator that # @settings was applied, because @given also assigns that attribute. descr = get_pretty_function_description(test) raise InvalidArgument( f"{descr} has already been decorated with a settings object.\n" - f" Previous: {test._hypothesis_internal_use_settings!r}\n" + f" Previous: {_test._hypothesis_internal_use_settings!r}\n" f" This: {self!r}" ) - test._hypothesis_internal_use_settings = self - test._hypothesis_internal_settings_applied = True + _test._hypothesis_internal_use_settings = self + _test._hypothesis_internal_settings_applied = True return test @classmethod @@ -361,7 +363,12 @@ def _max_examples_validator(x): validator=_max_examples_validator, description=""" Once this many satisfying examples have been considered without finding any -counter-example, falsification will terminate. +counter-example, Hypothesis will stop looking. + +Note that we might call your test function fewer times if we find a bug early +or can tell that we've exhausted the search space; or more if we discard some +examples due to use of .filter(), assume(), or a few other things that can +prevent the test case from completing successfully. The default value is chosen to suit a workflow where the test will be part of a suite that is regularly executed locally or on a CI server, balancing total @@ -427,11 +434,12 @@ def _validate_database(db): @unique class Phase(IntEnum): - explicit = 0 - reuse = 1 - generate = 2 - target = 3 - shrink = 4 + explicit = 0 #: controls whether explicit examples are run. + reuse = 1 #: controls whether previous examples will be reused. + generate = 2 #: controls whether new examples will be generated. + target = 3 #: controls whether examples will be mutated for targeting. + shrink = 4 #: controls whether examples will be shrunk. + explain = 5 #: controls whether Hypothesis attempts to explain test failures. def __repr__(self): return f"Phase.{self.name}" @@ -452,8 +460,14 @@ def all(cls) -> List["HealthCheck"]: return list(HealthCheck) data_too_large = 1 - """Check for when the typical size of the examples you are generating - exceeds the maximum allowed size too often.""" + """Checks if too many examples are aborted for being too large. + + This is measured by the number of random choices that Hypothesis makes + in order to generate something, not the size of the generated object. + For example, choosing a 100MB object from a predefined list would take + only a few bits, while generating 10KB of JSON from scratch might trigger + this health check. + """ filter_too_much = 2 """Check for when the test is filtering out too many examples, either @@ -476,12 +490,12 @@ def all(cls) -> List["HealthCheck"]: method defined by :class:`python:unittest.TestCase` (i.e. not a test).""" function_scoped_fixture = 9 - """Check if :func:`@given ` has been applied to a test + """Checks if :func:`@given ` has been applied to a test with a pytest function-scoped fixture. Function-scoped fixtures run once for the whole function, not once per example, and this is usually not what you want. - Because of this limitation, tests that need to need to set up or reset + Because of this limitation, tests that need to set up or reset state for every example need to do so manually within the test itself, typically using an appropriate context manager. @@ -523,7 +537,9 @@ def _validate_phases(phases): settings._define_setting( "phases", - default=tuple(Phase), + # We leave the `explain` phase disabled by default, for speed and brevity + # TODO: consider default-enabling this in CI? + default=_validate_phases(set(Phase) - {Phase.explain}), description=( "Control which phases should be run. " "See :ref:`the full documentation for more details `" @@ -585,7 +601,7 @@ class duration(datetime.timedelta): def __repr__(self): ms = self.total_seconds() * 1000 - return "timedelta(milliseconds={!r})".format(int(ms) if ms == int(ms) else ms) + return f"timedelta(milliseconds={int(ms) if ms == int(ms) else ms!r})" def _validate_deadline(x): diff --git a/hypothesis-python/src/hypothesis/configuration.py b/hypothesis-python/src/hypothesis/configuration.py index 5f6629eaf4..8f20aec7ea 100644 --- a/hypothesis-python/src/hypothesis/configuration.py +++ b/hypothesis-python/src/hypothesis/configuration.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import os diff --git a/hypothesis-python/src/hypothesis/control.py b/hypothesis-python/src/hypothesis/control.py index f06df34486..ebf2e53031 100644 --- a/hypothesis-python/src/hypothesis/control.py +++ b/hypothesis-python/src/hypothesis/control.py @@ -1,24 +1,19 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import math -import traceback from typing import NoReturn, Union from hypothesis import Verbosity, settings -from hypothesis.errors import CleanupFailed, InvalidArgument, UnsatisfiedAssumption +from hypothesis.errors import InvalidArgument, UnsatisfiedAssumption +from hypothesis.internal.compat import BaseExceptionGroup from hypothesis.internal.conjecture.data import ConjectureData from hypothesis.internal.validation import check_type from hypothesis.reporting import report, verbose_report @@ -79,18 +74,16 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, tb): self.assign_variable.__exit__(exc_type, exc_value, tb) - if self.close() and exc_type is None: - raise CleanupFailed() - - def close(self): - any_failed = False + errors = [] for task in self.tasks: try: task() - except BaseException: - any_failed = True - report(traceback.format_exc()) - return any_failed + except BaseException as err: + errors.append(err) + if errors: + if len(errors) == 1: + raise errors[0] from exc_value + raise BaseExceptionGroup("Cleanup failed", errors) from exc_value def cleanup(teardown): @@ -116,7 +109,7 @@ def should_note(): def note(value: str) -> None: - """Report this value in the final execution.""" + """Report this value for the minimal failing example.""" if should_note(): report(value) @@ -135,7 +128,7 @@ def event(value: str) -> None: context.data.note_event(value) -def target(observation: Union[int, float], *, label: str = "") -> None: +def target(observation: Union[int, float], *, label: str = "") -> Union[int, float]: """Calling this function with an ``int`` or ``float`` observation gives it feedback with which to guide our search for inputs that will cause an error, in addition to all the usual heuristics. Observations must always be finite. @@ -165,11 +158,6 @@ def target(observation: Union[int, float], *, label: str = "") -> None: and immediately obvious by around ten thousand examples *per label* used by your test. - .. note:: - ``hypothesis.target`` is considered experimental, and may be radically - changed or even removed in a future version. If you find it useful, - please let us know so we can share and build on that success! - :ref:`statistics` include the best score seen for each label, which can help avoid `the threshold problem `__ when the minimal @@ -182,13 +170,18 @@ def target(observation: Union[int, float], *, label: str = "") -> None: context = _current_build_context.value if context is None: - raise InvalidArgument("Calling target() outside of a test is invalid.") + raise InvalidArgument( + "Calling target() outside of a test is invalid. " + "Consider guarding this call with `if currently_in_test_context(): ...`" + ) verbose_report(f"Saw target(observation={observation!r}, label={label!r})") if label in context.data.target_observations: raise InvalidArgument( - "Calling target(%r, label=%r) would overwrite target(%r, label=%r)" - % (observation, label, context.data.target_observations[label], label) + f"Calling target({observation!r}, label={label!r}) would overwrite " + f"target({context.data.target_observations[label]!r}, label={label!r})" ) else: context.data.target_observations[label] = observation + + return observation diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py index f7ff9bbef5..d6c784c99f 100644 --- a/hypothesis-python/src/hypothesis/core.py +++ b/hypothesis-python/src/hypothesis/core.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """This module provides the core primitives of Hypothesis, such as given.""" @@ -20,17 +15,29 @@ import datetime import inspect import io -import random as rnd_module import sys import time -import traceback import types +import unittest import warnings import zlib -from inspect import getfullargspec +from collections import defaultdict +from functools import partial from io import StringIO from random import Random -from typing import Any, BinaryIO, Callable, Hashable, List, Optional, TypeVar, Union +from typing import ( + TYPE_CHECKING, + Any, + BinaryIO, + Callable, + Coroutine, + Hashable, + List, + Optional, + TypeVar, + Union, + overload, +) from unittest import TestCase import attr @@ -53,37 +60,42 @@ HypothesisDeprecationWarning, HypothesisWarning, InvalidArgument, - MultipleFailures, NoSuchExample, + StopTest, Unsatisfiable, UnsatisfiedAssumption, ) -from hypothesis.executors import new_style_executor +from hypothesis.executors import default_new_style_executor, new_style_executor from hypothesis.internal.compat import ( + PYPY, + BaseExceptionGroup, bad_django_TestCase, get_type_hints, int_from_bytes, - qualname, ) -from hypothesis.internal.conjecture.data import ConjectureData, StopTest -from hypothesis.internal.conjecture.engine import ConjectureRunner, sort_key +from hypothesis.internal.conjecture.data import ConjectureData, Status +from hypothesis.internal.conjecture.engine import ConjectureRunner +from hypothesis.internal.conjecture.shrinker import sort_key from hypothesis.internal.entropy import deterministic_PRNG from hypothesis.internal.escalation import ( escalate_hypothesis_internal_error, + format_exception, get_interesting_origin, get_trimmed_traceback, ) from hypothesis.internal.healthcheck import fail_health_check from hypothesis.internal.reflection import ( - arg_string, convert_positional_arguments, define_function_signature, function_digest, get_pretty_function_description, + get_signature, impersonate, is_mock, proxies, + repr_call, ) +from hypothesis.internal.scrutineer import Tracer, explanatory_lines from hypothesis.reporting import ( current_verbosity, report, @@ -92,20 +104,30 @@ ) from hypothesis.statistics import describe_targets, note_statistics from hypothesis.strategies._internal.collections import TupleStrategy +from hypothesis.strategies._internal.misc import NOTHING from hypothesis.strategies._internal.strategies import ( Ex, MappedSearchStrategy, SearchStrategy, ) -from hypothesis.utils.conventions import InferType, infer from hypothesis.vendor.pretty import RepresentationPrinter from hypothesis.version import __version__ +if sys.version_info >= (3, 10): # pragma: no cover + from types import EllipsisType as EllipsisType +elif TYPE_CHECKING: + from builtins import ellipsis as EllipsisType +else: + EllipsisType = type(Ellipsis) + + TestFunc = TypeVar("TestFunc", bound=Callable) running_under_pytest = False +pytest_shows_exceptiongroups = True global_force_seed = None +_hypothesis_global_random = None @attr.s() @@ -123,12 +145,15 @@ def example(*args: Any, **kwargs: Any) -> Callable[[TestFunc], TestFunc]: if not (args or kwargs): raise InvalidArgument("An example must provide at least one argument") + hypothesis_explicit_examples: List[Example] = [] + def accept(test): if not hasattr(test, "hypothesis_explicit_examples"): - test.hypothesis_explicit_examples = [] + test.hypothesis_explicit_examples = hypothesis_explicit_examples test.hypothesis_explicit_examples.append(Example(tuple(args), kwargs)) return test + accept.hypothesis_explicit_examples = hypothesis_explicit_examples # type: ignore return accept @@ -192,15 +217,17 @@ def decode_failure(blob): try: buffer = base64.b64decode(blob) except Exception: - raise InvalidArgument(f"Invalid base64 encoded string: {blob!r}") + raise InvalidArgument(f"Invalid base64 encoded string: {blob!r}") from None prefix = buffer[:1] if prefix == b"\0": return buffer[1:] elif prefix == b"\1": try: return zlib.decompress(buffer[1:]) - except zlib.error: - raise InvalidArgument(f"Invalid zlib compression for blob {blob!r}") + except zlib.error as err: + raise InvalidArgument( + f"Invalid zlib compression for blob {blob!r}" + ) from err else: raise InvalidArgument( f"Could not decode blob {blob!r}: Invalid start byte {prefix!r}" @@ -210,7 +237,7 @@ def decode_failure(blob): class WithRunner(MappedSearchStrategy): def __init__(self, base, runner): assert runner is not None - MappedSearchStrategy.__init__(self, base) + super().__init__(base) self.runner = runner def do_draw(self, data): @@ -221,7 +248,21 @@ def __repr__(self): return f"WithRunner({self.mapped_strategy!r}, runner={self.runner!r})" -def is_invalid_test(name, original_argspec, given_arguments, given_kwargs): +def _invalid(message, *, exc=InvalidArgument, test, given_kwargs): + @impersonate(test) + def wrapped_test(*arguments, **kwargs): # pragma: no cover # coverage limitation + raise exc(message) + + wrapped_test.is_hypothesis_test = True + wrapped_test.hypothesis = HypothesisHandle( + inner_test=test, + get_fuzz_target=wrapped_test, + given_kwargs=given_kwargs, + ) + return wrapped_test + + +def is_invalid_test(test, original_sig, given_arguments, given_kwargs): """Check the arguments to ``@given`` for basic usage constraints. Most errors are not raised immediately; instead we return a dummy test @@ -229,60 +270,57 @@ def is_invalid_test(name, original_argspec, given_arguments, given_kwargs): When the user runs a subset of tests (e.g via ``pytest -k``), errors will only be reported for tests that actually ran. """ - - def invalid(message): - def wrapped_test(*arguments, **kwargs): - raise InvalidArgument(message) - - wrapped_test.is_hypothesis_test = True - return wrapped_test + invalid = partial(_invalid, test=test, given_kwargs=given_kwargs) if not (given_arguments or given_kwargs): return invalid("given must be called with at least one argument") - if given_arguments and any( - [original_argspec.varargs, original_argspec.varkw, original_argspec.kwonlyargs] - ): + params = list(original_sig.parameters.values()) + pos_params = [p for p in params if p.kind is p.POSITIONAL_OR_KEYWORD] + kwonly_params = [p for p in params if p.kind is p.KEYWORD_ONLY] + if given_arguments and params != pos_params: return invalid( "positional arguments to @given are not supported with varargs, " - "varkeywords, or keyword-only arguments" + "varkeywords, positional-only, or keyword-only arguments" ) - if len(given_arguments) > len(original_argspec.args): - args = tuple(given_arguments) + if len(given_arguments) > len(pos_params): return invalid( - "Too many positional arguments for %s() were passed to @given " - "- expected at most %d arguments, but got %d %r" - % (name, len(original_argspec.args), len(args), args) + f"Too many positional arguments for {test.__name__}() were passed to " + f"@given - expected at most {len(pos_params)} " + f"arguments, but got {len(given_arguments)} {given_arguments!r}" ) - if infer in given_arguments: + if ... in given_arguments: return invalid( - "infer was passed as a positional argument to @given, " - "but may only be passed as a keyword argument" + "... was passed as a positional argument to @given, but may only be " + "passed as a keyword argument or as the sole argument of @given" ) if given_arguments and given_kwargs: return invalid("cannot mix positional and keyword arguments to @given") extra_kwargs = [ - k - for k in given_kwargs - if k not in original_argspec.args + original_argspec.kwonlyargs + k for k in given_kwargs if k not in {p.name for p in pos_params + kwonly_params} ] - if extra_kwargs and not original_argspec.varkw: + if extra_kwargs and (params == [] or params[-1].kind is not params[-1].VAR_KEYWORD): arg = extra_kwargs[0] return invalid( - "%s() got an unexpected keyword argument %r, from `%s=%r` in @given" - % (name, arg, arg, given_kwargs[arg]) + f"{test.__name__}() got an unexpected keyword argument {arg!r}, " + f"from `{arg}={given_kwargs[arg]!r}` in @given" ) - if original_argspec.defaults or original_argspec.kwonlydefaults: + if any(p.default is not p.empty for p in params): return invalid("Cannot apply @given to a function with defaults.") - missing = [repr(kw) for kw in original_argspec.kwonlyargs if kw not in given_kwargs] - if missing: + + # This case would raise Unsatisfiable *anyway*, but by detecting it here we can + # provide a much more helpful error message for people e.g. using the Ghostwriter. + empty = [ + f"{s!r} (arg {idx})" for idx, s in enumerate(given_arguments) if s is NOTHING + ] + [f"{name}={s!r}" for name, s in given_kwargs.items() if s is NOTHING] + if empty: + strats = "strategies" if len(empty) > 1 else "strategy" return invalid( - "Missing required kwarg{}: {}".format( - "s" if len(missing) > 1 else "", ", ".join(missing) - ) + f"Cannot generate examples from empty {strats}: " + ", ".join(empty), + exc=Unsatisfiable, ) @@ -294,8 +332,9 @@ class ArtificialDataForExample(ConjectureData): provided by @example. """ - def __init__(self, kwargs): + def __init__(self, args, kwargs): self.__draws = 0 + self.__args = args self.__kwargs = kwargs super().__init__(max_length=0, prefix=b"", random=None) @@ -309,36 +348,61 @@ def draw(self, strategy): # first positional arguments then keyword arguments. When building this # object already converted all positional arguments to keyword arguments, # so this is the correct format to return. - return (), self.__kwargs + return self.__args, self.__kwargs -def execute_explicit_examples(state, wrapped_test, arguments, kwargs): - original_argspec = getfullargspec(state.test) +def execute_explicit_examples(state, wrapped_test, arguments, kwargs, original_sig): + posargs = [ + p.name + for p in original_sig.parameters.values() + if p.kind is p.POSITIONAL_OR_KEYWORD + ] for example in reversed(getattr(wrapped_test, "hypothesis_explicit_examples", ())): - example_kwargs = dict(original_argspec.kwonlydefaults or {}) + # All of this validation is to check that @example() got "the same" arguments + # as @given, i.e. corresponding to the same parameters, even though they might + # be any mixture of positional and keyword arguments. if example.args: - if len(example.args) > len(original_argspec.args): + assert not example.kwargs + if any( + p.kind is p.POSITIONAL_ONLY for p in original_sig.parameters.values() + ): raise InvalidArgument( - "example has too many arguments for test. " - "Expected at most %d but got %d" - % (len(original_argspec.args), len(example.args)) + "Cannot pass positional arguments to @example() when decorating " + "a test function which has positional-only parameters." ) - example_kwargs.update( - dict(zip(original_argspec.args[-len(example.args) :], example.args)) - ) + if len(example.args) > len(posargs): + raise InvalidArgument( + "example has too many arguments for test. Expected at most " + f"{len(posargs)} but got {len(example.args)}" + ) + example_kwargs = dict(zip(posargs[-len(example.args) :], example.args)) else: - example_kwargs.update(example.kwargs) + example_kwargs = dict(example.kwargs) + given_kws = ", ".join( + repr(k) for k in sorted(wrapped_test.hypothesis._given_kwargs) + ) + example_kws = ", ".join(repr(k) for k in sorted(example_kwargs)) + if given_kws != example_kws: + raise InvalidArgument( + f"Inconsistent args: @given() got strategies for {given_kws}, " + f"but @example() got arguments for {example_kws}" + ) from None + + # This is certainly true because the example_kwargs exactly match the params + # reserved by @given(), which are then remove from the function signature. + assert set(example_kwargs).isdisjoint(kwargs) + example_kwargs.update(kwargs) + if Phase.explicit not in state.settings.phases: continue - example_kwargs.update(kwargs) with local_settings(state.settings): fragments_reported = [] try: with with_reporter(fragments_reported.append): state.execute_once( - ArtificialDataForExample(example_kwargs), + ArtificialDataForExample(arguments, example_kwargs), is_final=True, print_example=True, ) @@ -372,7 +436,11 @@ def execute_explicit_examples(state, wrapped_test, arguments, kwargs): err = new yield (fragments_reported, err) - if state.settings.report_multiple_bugs: + if ( + state.settings.report_multiple_bugs + and pytest_shows_exceptiongroups + and not isinstance(err, skip_exceptions_to_reraise()) + ): continue break finally: @@ -399,20 +467,24 @@ def get_random_for_wrapped_test(test, wrapped_test): elif global_force_seed is not None: return Random(global_force_seed) else: - seed = rnd_module.getrandbits(128) + global _hypothesis_global_random + if _hypothesis_global_random is None: + _hypothesis_global_random = Random() + seed = _hypothesis_global_random.getrandbits(128) wrapped_test._hypothesis_internal_use_generated_seed = seed return Random(seed) -def process_arguments_to_given(wrapped_test, arguments, kwargs, given_kwargs, argspec): +def process_arguments_to_given(wrapped_test, arguments, kwargs, given_kwargs, params): selfy = None arguments, kwargs = convert_positional_arguments(wrapped_test, arguments, kwargs) # If the test function is a method of some kind, the bound object # will be the first named argument if there are any, otherwise the # first vararg (if any). - if argspec.args: - selfy = kwargs.get(argspec.args[0]) + posargs = [p.name for p in params.values() if p.kind is p.POSITIONAL_OR_KEYWORD] + if posargs: + selfy = kwargs.get(posargs[0]) elif arguments: selfy = arguments[0] @@ -458,11 +530,11 @@ def skip_exceptions_to_reraise(): # and more importantly it avoids possible side-effects :-) if "unittest" in sys.modules: exceptions.add(sys.modules["unittest"].SkipTest) - if "unittest2" in sys.modules: # pragma: no cover + if "unittest2" in sys.modules: exceptions.add(sys.modules["unittest2"].SkipTest) - if "nose" in sys.modules: # pragma: no cover + if "nose" in sys.modules: exceptions.add(sys.modules["nose"].SkipTest) - if "_pytest" in sys.modules: # pragma: no branch + if "_pytest" in sys.modules: exceptions.add(sys.modules["_pytest"].outcomes.Skipped) return tuple(sorted(exceptions, key=str)) @@ -473,24 +545,28 @@ def failure_exceptions_to_catch(): This is intended to cover most common test runners; if you would like another to be added please open an issue or pull request. """ - exceptions = [Exception] - if "_pytest" in sys.modules: # pragma: no branch + # While SystemExit and GeneratorExit are instances of BaseException, we also + # expect them to be deterministic - unlike KeyboardInterrupt - and so we treat + # them as standard exceptions, check for flakiness, etc. + # See https://github.com/HypothesisWorks/hypothesis/issues/2223 for details. + exceptions = [Exception, SystemExit, GeneratorExit] + if "_pytest" in sys.modules: exceptions.append(sys.modules["_pytest"].outcomes.Failed) return tuple(exceptions) -def new_given_argspec(original_argspec, given_kwargs): - """Make an updated argspec for the wrapped test.""" - new_args = [a for a in original_argspec.args if a not in given_kwargs] - new_kwonlyargs = [a for a in original_argspec.kwonlyargs if a not in given_kwargs] - annots = { - k: v - for k, v in original_argspec.annotations.items() - if k in new_args + new_kwonlyargs - } - annots["return"] = None - return original_argspec._replace( - args=new_args, kwonlyargs=new_kwonlyargs, annotations=annots +def new_given_signature(original_sig, given_kwargs): + """Make an updated signature for the wrapped test.""" + return original_sig.replace( + parameters=[ + p + for p in original_sig.parameters.values() + if not ( + p.name in given_kwargs + and p.kind in (p.POSITIONAL_OR_KEYWORD, p.KEYWORD_ONLY) + ) + ], + return_annotation=None, ) @@ -503,12 +579,10 @@ def __init__( self.settings = settings self.last_exception = None self.falsifying_examples = () - self.__was_flaky = False self.random = random - self.__warned_deadline = False self.__test_runtime = None + self.ever_executed = False - self.__had_seed = wrapped_test._hypothesis_internal_use_seed self.is_find = getattr(wrapped_test, "_hypothesis_internal_is_find", False) self.wrapped_test = wrapped_test @@ -520,6 +594,9 @@ def __init__( self.files_to_propagate = set() self.failed_normally = False + self.failed_due_to_deadline = False + + self.explain_traces = defaultdict(set) def execute_once( self, data, print_example=False, is_final=False, expected_failure=None @@ -535,9 +612,10 @@ def execute_once( swallowed the corresponding control exception. """ + self.ever_executed = True data.is_find = self.is_find - text_repr = [None] + text_repr = None if self.settings.deadline is None: test = self.test else: @@ -570,7 +648,8 @@ def run(data): # Generate all arguments to the test function. args, kwargs = data.draw(self.search_strategy) if expected_failure is not None: - text_repr[0] = arg_string(test, args, kwargs) + nonlocal text_repr + text_repr = repr_call(test, args, kwargs) if print_example or current_verbosity() >= Verbosity.verbose: output = StringIO() @@ -589,31 +668,12 @@ def run(data): for v in args: printer.pretty(v) # We add a comma unconditionally because - # generated arguments will always be - # kwargs, so there will always be more - # to come. + # generated arguments will always be kwargs, + # so there will always be more to come. printer.text(",") printer.breakable() - # We need to make sure to print these in the - # argument order for Python 2 and older versions - # of Python 3.5. In modern versions this isn't - # an issue because kwargs is ordered. - arg_order = { - v: i - for i, v in enumerate( - getfullargspec(self.test).args - ) - } - for i, (k, v) in enumerate( - sorted( - kwargs.items(), - key=lambda t: ( - arg_order.get(t[0], float("inf")), - t[0], - ), - ) - ): + for i, (k, v) in enumerate(kwargs.items()): printer.text(k) printer.text("=") printer.pretty(v) @@ -639,14 +699,12 @@ def run(data): and self.__test_runtime is not None ): report( - ( - "Unreliable test timings! On an initial run, this " - "test took %.2fms, which exceeded the deadline of " - "%.2fms, but on a subsequent run it took %.2f ms, " - "which did not. If you expect this sort of " - "variability in your test timings, consider turning " - "deadlines off for this test by setting deadline=None." - ) + "Unreliable test timings! On an initial run, this " + "test took %.2fms, which exceeded the deadline of " + "%.2fms, but on a subsequent run it took %.2f ms, " + "which did not. If you expect this sort of " + "variability in your test timings, consider turning " + "deadlines off for this test by setting deadline=None." % ( exception.runtime.total_seconds() * 1000, self.settings.deadline.total_seconds() * 1000, @@ -655,13 +713,10 @@ def run(data): ) else: report("Failed to reproduce exception. Expected: \n" + traceback) - self.__flaky( - ( - "Hypothesis %s(%s) produces unreliable results: Falsified" - " on the first call but did not on a subsequent one" - ) - % (test.__name__, text_repr[0]) - ) + raise Flaky( + f"Hypothesis {text_repr} produces unreliable results: " + "Falsified on the first call but did not on a subsequent one" + ) from exception return result def _execute_once_for_engine(self, data): @@ -673,15 +728,33 @@ def _execute_once_for_engine(self, data): ``StopTest`` must be a fatal error, and should stop the entire engine. """ try: - result = self.execute_once(data) + trace = frozenset() + if ( + self.failed_normally + and not self.failed_due_to_deadline + and Phase.shrink in self.settings.phases + and Phase.explain in self.settings.phases + and sys.gettrace() is None + and not PYPY + ): # pragma: no cover + # This is in fact covered by our *non-coverage* tests, but due to the + # settrace() contention *not* by our coverage tests. Ah well. + tracer = Tracer() + try: + sys.settrace(tracer.trace) + result = self.execute_once(data) + if data.status == Status.VALID: + self.explain_traces[None].add(frozenset(tracer.branches)) + finally: + sys.settrace(None) + trace = frozenset(tracer.branches) + else: + result = self.execute_once(data) if result is not None: fail_health_check( self.settings, - ( - "Tests run under @given should return None, but " - "%s returned %r instead." - ) - % (self.test.__name__, result), + "Tests run under @given should return None, but " + f"{self.test.__name__} returned {result!r} instead.", HealthCheck.return_value, ) except UnsatisfiedAssumption: @@ -708,7 +781,7 @@ def _execute_once_for_engine(self, data): # This can happen if an error occurred in a finally # block somewhere, suppressing our original StopTest. # We raise a new one here to resume normal operation. - raise StopTest(data.testcounter) + raise StopTest(data.testcounter) from e else: # The test failed by raising an exception, so we inform the # engine that this test run was interesting. This is the normal @@ -716,13 +789,20 @@ def _execute_once_for_engine(self, data): tb = get_trimmed_traceback() info = data.extra_information - info.__expected_traceback = "".join( - traceback.format_exception(type(e), e, tb) - ) + info.__expected_traceback = format_exception(e, tb) info.__expected_exception = e verbose_report(info.__expected_traceback) - data.mark_interesting(get_interesting_origin(e)) + self.failed_normally = True + + interesting_origin = get_interesting_origin(e) + if trace: # pragma: no cover + # Trace collection is explicitly disabled under coverage. + self.explain_traces[interesting_origin].add(trace) + if interesting_origin[0] == DeadlineExceeded: + self.failed_due_to_deadline = True + self.explain_traces.clear() + data.mark_interesting(interesting_origin) def run_engine(self): """Run the test function many times, on database input and generated @@ -759,109 +839,101 @@ def run_engine(self): ) else: if runner.valid_examples == 0: - raise Unsatisfiable( - "Unable to satisfy assumptions of hypothesis %s." - % (get_pretty_function_description(self.test),) - ) + rep = get_pretty_function_description(self.test) + raise Unsatisfiable(f"Unable to satisfy assumptions of {rep}") if not self.falsifying_examples: return - elif not self.settings.report_multiple_bugs: + elif not (self.settings.report_multiple_bugs and pytest_shows_exceptiongroups): # Pretend that we only found one failure, by discarding the others. del self.falsifying_examples[:-1] # The engine found one or more failures, so we need to reproduce and # report them. - self.failed_normally = True - - flaky = 0 + errors_to_report = [] - if runner.best_observed_targets: - for line in describe_targets(runner.best_observed_targets): - report(line) - report("") + report_lines = describe_targets(runner.best_observed_targets) + if report_lines: + report_lines.append("") + explanations = explanatory_lines(self.explain_traces, self.settings) for falsifying_example in self.falsifying_examples: info = falsifying_example.extra_information + fragments = [] ran_example = ConjectureData.for_buffer(falsifying_example.buffer) - self.__was_flaky = False assert info.__expected_exception is not None try: - self.execute_once( - ran_example, - print_example=not self.is_find, - is_final=True, - expected_failure=( - info.__expected_exception, - info.__expected_traceback, - ), - ) - except (UnsatisfiedAssumption, StopTest): - report(traceback.format_exc()) - self.__flaky( + with with_reporter(fragments.append): + self.execute_once( + ran_example, + print_example=not self.is_find, + is_final=True, + expected_failure=( + info.__expected_exception, + info.__expected_traceback, + ), + ) + except (UnsatisfiedAssumption, StopTest) as e: + err = Flaky( "Unreliable assumption: An example which satisfied " - "assumptions on the first run now fails it." + "assumptions on the first run now fails it.", ) + err.__cause__ = err.__context__ = e + errors_to_report.append((fragments, err)) except BaseException as e: - if len(self.falsifying_examples) <= 1: - # There is only one failure, so we can report it by raising - # it directly. - raise - - # We are reporting multiple failures, so we need to manually - # print each exception's stack trace and information. - tb = get_trimmed_traceback() - report("".join(traceback.format_exception(type(e), e, tb))) - - finally: # pragma: no cover - # Mostly useful for ``find`` and ensuring that objects that - # hold on to a reference to ``data`` know that it's now been - # finished and they shouldn't attempt to draw more data from - # it. - ran_example.freeze() + # If we have anything for explain-mode, this is the time to report. + fragments.extend(explanations[falsifying_example.interesting_origin]) + errors_to_report.append( + (fragments, e.with_traceback(get_trimmed_traceback())) + ) - # This section is in fact entirely covered by the tests in - # test_reproduce_failure, but it seems to trigger a lovely set - # of coverage bugs: The branches show up as uncovered (despite - # definitely being covered - you can add an assert False else - # branch to verify this and see it fail - and additionally the - # second branch still complains about lack of coverage even if - # you add a pragma: no cover to it! - # See https://bitbucket.org/ned/coveragepy/issues/623/ + finally: + # Whether or not replay actually raised the exception again, we want + # to print the reproduce_failure decorator for the failing example. if self.settings.print_blob: - report( - ( - "\nYou can reproduce this example by temporarily " - "adding @reproduce_failure(%r, %r) as a decorator " - "on your test case" - ) + fragments.append( + "\nYou can reproduce this example by temporarily adding " + "@reproduce_failure(%r, %r) as a decorator on your test case" % (__version__, encode_failure(falsifying_example.buffer)) ) - if self.__was_flaky: - flaky += 1 + # Mostly useful for ``find`` and ensuring that objects that + # hold on to a reference to ``data`` know that it's now been + # finished and they can't draw more data from it. + ran_example.freeze() + _raise_to_user(errors_to_report, self.settings, report_lines) - # If we only have one example then we should have raised an error or - # flaky prior to this point. - assert len(self.falsifying_examples) > 1 - if flaky > 0: - raise Flaky( - f"Hypothesis found {len(self.falsifying_examples)} distinct failures, " - f"but {flaky} of them exhibited some sort of flaky behaviour." - ) - else: - raise MultipleFailures( - f"Hypothesis found {len(self.falsifying_examples)} distinct failures." - ) +def add_note(exc, note): + try: + exc.add_note(note) + except AttributeError: + if not hasattr(exc, "__notes__"): + exc.__notes__ = [] + exc.__notes__.append(note) + + +def _raise_to_user(errors_to_report, settings, target_lines, trailer=""): + """Helper function for attaching notes and grouping multiple errors.""" + if settings.verbosity >= Verbosity.normal: + for fragments, err in errors_to_report: + for note in fragments: + add_note(err, note) + + if len(errors_to_report) == 1: + _, the_error_hypothesis_found = errors_to_report[0] + else: + assert errors_to_report + the_error_hypothesis_found = BaseExceptionGroup( + f"Hypothesis found {len(errors_to_report)} distinct failures{trailer}.", + [e for _, e in errors_to_report], + ) - def __flaky(self, message): - if len(self.falsifying_examples) <= 1: - raise Flaky(message) - else: - self.__was_flaky = True - report("Flaky example! " + message) + if settings.verbosity >= Verbosity.normal: + for line in target_lines: + add_note(the_error_hypothesis_found, line) + raise the_error_hypothesis_found @contextlib.contextmanager @@ -911,8 +983,6 @@ def fuzz_one_input( Returns None if buffer invalid for the strategy, canonical pruned bytes if the buffer was valid, and leaves raised exceptions alone. - - Note: this feature is experimental and may change or be removed. """ # Note: most users, if they care about fuzzer performance, will access the # property and assign it to a local variable to move the attribute lookup @@ -925,10 +995,30 @@ def fuzz_one_input( return self.__cached_target +@overload +def given( + *_given_arguments: Union[SearchStrategy[Any], EllipsisType], +) -> Callable[ + [Callable[..., Optional[Coroutine[Any, Any, None]]]], Callable[..., None] +]: # pragma: no cover + ... + + +@overload +def given( + **_given_kwargs: Union[SearchStrategy[Any], EllipsisType], +) -> Callable[ + [Callable[..., Optional[Coroutine[Any, Any, None]]]], Callable[..., None] +]: # pragma: no cover + ... + + def given( - *_given_arguments: Union[SearchStrategy, InferType], - **_given_kwargs: Union[SearchStrategy, InferType], -) -> Callable[[Callable[..., None]], Callable[..., None]]: + *_given_arguments: Union[SearchStrategy[Any], EllipsisType], + **_given_kwargs: Union[SearchStrategy[Any], EllipsisType], +) -> Callable[ + [Callable[..., Optional[Coroutine[Any, Any, None]]]], Callable[..., None] +]: """A decorator for turning a test function that accepts arguments into a randomized test. @@ -943,10 +1033,18 @@ def run_test_as_given(test): given_arguments = tuple(_given_arguments) given_kwargs = dict(_given_kwargs) - original_argspec = getfullargspec(test) + original_sig = get_signature(test) + if given_arguments == (Ellipsis,) and not given_kwargs: + # user indicated that they want to infer all arguments + given_kwargs = { + p.name: Ellipsis + for p in original_sig.parameters.values() + if p.kind in (p.POSITIONAL_OR_KEYWORD, p.KEYWORD_ONLY) + } + given_arguments = () check_invalid = is_invalid_test( - test.__name__, original_argspec, given_arguments, given_kwargs + test, original_sig, given_arguments, given_kwargs ) # If the argument check found problems, return a dummy test function @@ -958,38 +1056,32 @@ def run_test_as_given(test): # positional arguments into keyword arguments for simplicity. if given_arguments: assert not given_kwargs - for name, strategy in zip( - reversed(original_argspec.args), reversed(given_arguments) - ): - given_kwargs[name] = strategy + posargs = [ + p.name + for p in original_sig.parameters.values() + if p.kind is p.POSITIONAL_OR_KEYWORD + ] + given_kwargs = dict(zip(posargs[::-1], given_arguments[::-1])) # These have been converted, so delete them to prevent accidental use. del given_arguments - argspec = new_given_argspec(original_argspec, given_kwargs) + new_signature = new_given_signature(original_sig, given_kwargs) # Use type information to convert "infer" arguments into appropriate strategies. - if infer in given_kwargs.values(): + if ... in given_kwargs.values(): hints = get_type_hints(test) - for name in [name for name, value in given_kwargs.items() if value is infer]: + for name in [name for name, value in given_kwargs.items() if value is ...]: if name not in hints: - # As usual, we want to emit this error when the test is executed, - # not when it's decorated. - - @impersonate(test) - @define_function_signature(test.__name__, test.__doc__, argspec) - def wrapped_test(*arguments, **kwargs): - __tracebackhide__ = True - raise InvalidArgument( - "passed %s=infer for %s, but %s has no type annotation" - % (name, test.__name__, name) - ) - - return wrapped_test - + return _invalid( + f"passed {name}=... for {test.__name__}, but {name} has " + "no type annotation", + test=test, + given_kwargs=given_kwargs, + ) given_kwargs[name] = st.from_type(hints[name]) @impersonate(test) - @define_function_signature(test.__name__, test.__doc__, argspec) + @define_function_signature(test.__name__, test.__doc__, new_signature) def wrapped_test(*arguments, **kwargs): # Tell pytest to omit the body of this function from tracebacks __tracebackhide__ = True @@ -998,15 +1090,12 @@ def wrapped_test(*arguments, **kwargs): if getattr(test, "is_hypothesis_test", False): raise InvalidArgument( - ( - "You have applied @given to the test %s more than once, which " - "wraps the test several times and is extremely slow. A " - "similar effect can be gained by combining the arguments " - "of the two calls to given. For example, instead of " - "@given(booleans()) @given(integers()), you could write " - "@given(booleans(), integers())" - ) - % (test.__name__,) + f"You have applied @given to the test {test.__name__} more than " + "once, which wraps the test several times and is extremely slow. " + "A similar effect can be gained by combining the arguments " + "of the two calls to given. For example, instead of " + "@given(booleans()) @given(integers()), you could write " + "@given(booleans(), integers())" ) settings = wrapped_test._hypothesis_internal_use_settings @@ -1014,26 +1103,43 @@ def wrapped_test(*arguments, **kwargs): random = get_random_for_wrapped_test(test, wrapped_test) processed_args = process_arguments_to_given( - wrapped_test, arguments, kwargs, given_kwargs, argspec + wrapped_test, arguments, kwargs, given_kwargs, new_signature.parameters ) arguments, kwargs, test_runner, search_strategy = processed_args + if ( + inspect.iscoroutinefunction(test) + and test_runner is default_new_style_executor + ): + # See https://github.com/HypothesisWorks/hypothesis/issues/3054 + # If our custom executor doesn't handle coroutines, or we return an + # awaitable from a non-async-def function, we just rely on the + # return_value health check. This catches most user errors though. + raise InvalidArgument( + "Hypothesis doesn't know how to run async test functions like " + f"{test.__name__}. You'll need to write a custom executor, " + "or use a library like pytest-asyncio or pytest-trio which can " + "handle the translation for you.\n See https://hypothesis." + "readthedocs.io/en/latest/details.html#custom-function-execution" + ) + runner = getattr(search_strategy, "runner", None) if isinstance(runner, TestCase) and test.__name__ in dir(TestCase): msg = ( - "You have applied @given to the method %s, which is " + f"You have applied @given to the method {test.__name__}, which is " "used by the unittest runner but is not itself a test." - " This is not useful in any way." % test.__name__ + " This is not useful in any way." ) fail_health_check(settings, msg, HealthCheck.not_a_test_method) if bad_django_TestCase(runner): # pragma: no cover # Covered by the Django tests, but not the pytest coverage task raise InvalidArgument( - "You have applied @given to a method on %s, but this " + "You have applied @given to a method on " + f"{type(runner).__qualname__}, but this " "class does not inherit from the supported versions in " "`hypothesis.extra.django`. Use the Hypothesis variants " "to ensure that each example is run in a separate " - "database transaction." % qualname(type(runner)) + "database transaction." ) state = StateForActualGivenExecution( @@ -1049,12 +1155,10 @@ def wrapped_test(*arguments, **kwargs): expected_version, failure = reproduce_failure if expected_version != __version__: raise InvalidArgument( - ( - "Attempting to reproduce a failure from a different " - "version of Hypothesis. This failure is from %s, but " - "you are currently running %r. Please change your " - "Hypothesis version to a matching one." - ) + "Attempting to reproduce a failure from a different " + "version of Hypothesis. This failure is from %s, but " + "you are currently running %r. Please change your " + "Hypothesis version to a matching one." % (expected_version, __version__) ) try: @@ -1072,47 +1176,50 @@ def wrapped_test(*arguments, **kwargs): "The shape of the test data has changed in some way " "from where this blob was defined. Are you sure " "you're running the same test?" - ) + ) from None except UnsatisfiedAssumption: raise DidNotReproduce( "The test data failed to satisfy an assumption in the " "test. Have you added it since this blob was " "generated?" - ) + ) from None # There was no @reproduce_failure, so start by running any explicit # examples from @example decorators. errors = list( - execute_explicit_examples(state, wrapped_test, arguments, kwargs) + execute_explicit_examples( + state, wrapped_test, arguments, kwargs, original_sig + ) ) - with local_settings(state.settings): - if len(errors) > 1: - # If we're not going to report multiple bugs, we would have - # stopped running explicit examples at the first failure. - assert state.settings.report_multiple_bugs - for fragments, err in errors: - for f in fragments: - report(f) - tb_lines = traceback.format_exception( - type(err), err, err.__traceback__ - ) - report("".join(tb_lines)) - raise MultipleFailures( - f"Hypothesis found {len(errors)} failures in explicit examples." - ) - elif errors: - fragments, the_error_hypothesis_found = errors[0] - for f in fragments: - report(f) - raise the_error_hypothesis_found + if errors: + # If we're not going to report multiple bugs, we would have + # stopped running explicit examples at the first failure. + assert len(errors) == 1 or state.settings.report_multiple_bugs + + # If an explicit example raised a 'skip' exception, ensure it's never + # wrapped up in an exception group. Because we break out of the loop + # immediately on finding a skip, if present it's always the last error. + if isinstance(errors[-1][1], skip_exceptions_to_reraise()): + # Covered by `test_issue_3453_regression`, just in a subprocess. + del errors[:-1] # pragma: no cover + + _raise_to_user(errors, state.settings, [], " in explicit examples") # If there were any explicit examples, they all ran successfully. # The next step is to use the Conjecture engine to run the test on # many different inputs. + ran_explicit_examples = Phase.explicit in state.settings.phases and getattr( + wrapped_test, "hypothesis_explicit_examples", () + ) + SKIP_BECAUSE_NO_EXAMPLES = unittest.SkipTest( + "Hypothesis has been told to run no examples for this test." + ) if not ( Phase.reuse in settings.phases or Phase.generate in settings.phases ): + if not ran_explicit_examples: + raise SKIP_BECAUSE_NO_EXAMPLES return try: @@ -1127,7 +1234,7 @@ def wrapped_test(*arguments, **kwargs): state.run_engine() except BaseException as e: # The exception caught here should either be an actual test - # failure (or MultipleFailures), or some kind of fatal error + # failure (or BaseExceptionGroup), or some kind of fatal error # that caused the engine to stop. generated_seed = wrapped_test._hypothesis_internal_use_generated_seed @@ -1153,10 +1260,15 @@ def wrapped_test(*arguments, **kwargs): # which will actually appear in tracebacks is as clear as # possible - "raise the_error_hypothesis_found". the_error_hypothesis_found = e.with_traceback( - get_trimmed_traceback() + None + if isinstance(e, BaseExceptionGroup) + else get_trimmed_traceback() ) raise the_error_hypothesis_found + if not (ran_explicit_examples or state.ever_executed): + raise SKIP_BECAUSE_NO_EXAMPLES + def _get_fuzz_target() -> Callable[ [Union[bytes, bytearray, memoryview, BinaryIO]], Optional[bytes] ]: @@ -1175,7 +1287,7 @@ def _get_fuzz_target() -> Callable[ ) random = get_random_for_wrapped_test(test, wrapped_test) _args, _kwargs, test_runner, search_strategy = process_arguments_to_given( - wrapped_test, (), {}, given_kwargs, argspec + wrapped_test, (), {}, given_kwargs, new_signature.parameters ) assert not _args assert not _kwargs @@ -1262,12 +1374,12 @@ def find( if not isinstance(specifier, SearchStrategy): raise InvalidArgument( - "Expected SearchStrategy but got %r of type %s" - % (specifier, type(specifier).__name__) + f"Expected SearchStrategy but got {specifier!r} of " + f"type {type(specifier).__name__}" ) specifier.validate() - last = [] # type: List[Ex] + last: List[Ex] = [] @settings @given(specifier) @@ -1279,8 +1391,11 @@ def test(v): if random is not None: test = seed(random.getrandbits(64))(test) - test._hypothesis_internal_is_find = True - test._hypothesis_internal_database_key = database_key + # Aliasing as Any avoids mypy errors (attr-defined) when accessing and + # setting custom attributes on the decorated function or class. + _test: Any = test + _test._hypothesis_internal_is_find = True + _test._hypothesis_internal_database_key = database_key try: test() diff --git a/hypothesis-python/src/hypothesis/database.py b/hypothesis-python/src/hypothesis/database.py index 9d709c4a91..c77a1e78fa 100644 --- a/hypothesis-python/src/hypothesis/database.py +++ b/hypothesis-python/src/hypothesis/database.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import abc import binascii @@ -19,7 +14,7 @@ import sys import warnings from hashlib import sha384 -from typing import Iterable +from typing import Dict, Iterable from hypothesis.configuration import mkdir_p, storage_directory from hypothesis.errors import HypothesisException, HypothesisWarning @@ -86,7 +81,7 @@ def __call__(self, *args, **kwargs): # This code only runs if Sphinx has already been imported; and it would live in our # docs/conf.py except that we would also like it to work for anyone documenting # downstream ExampleDatabase subclasses too. -if "sphinx" in sys.modules: # pragma: no cover +if "sphinx" in sys.modules: try: from sphinx.ext.autodoc import _METACLASS_CALL_BLACKLIST @@ -187,7 +182,7 @@ class DirectoryBasedExampleDatabase(ExampleDatabase): def __init__(self, path: str) -> None: self.path = path - self.keypaths = {} # type: dict + self.keypaths: Dict[str, str] = {} def __repr__(self) -> str: return f"DirectoryBasedExampleDatabase({self.path!r})" diff --git a/hypothesis-python/src/hypothesis/entry_points.py b/hypothesis-python/src/hypothesis/entry_points.py index 00f0919260..00e2e11ede 100644 --- a/hypothesis-python/src/hypothesis/entry_points.py +++ b/hypothesis-python/src/hypothesis/entry_points.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """Run all functions registered for the "hypothesis" entry point. @@ -29,8 +24,14 @@ import importlib_metadata # type: ignore # mypy thinks this is a redefinition def get_entry_points(): - yield from importlib_metadata.entry_points().get("hypothesis", []) - + try: + eps = importlib_metadata.entry_points(group="hypothesis") + except TypeError: + # Load-time selection requires Python >= 3.10 or importlib_metadata >= 3.6, + # so we'll retain this fallback logic for some time to come. See also + # https://importlib-metadata.readthedocs.io/en/latest/using.html + eps = importlib_metadata.entry_points().get("hypothesis", []) + yield from eps except ImportError: # But if we're not on Python >= 3.8 and the importlib_metadata backport diff --git a/hypothesis-python/src/hypothesis/errors.py b/hypothesis-python/src/hypothesis/errors.py index c342d8ea5e..06b77de29d 100644 --- a/hypothesis-python/src/hypothesis/errors.py +++ b/hypothesis-python/src/hypothesis/errors.py @@ -1,25 +1,20 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER class HypothesisException(Exception): """Generic parent class for exceptions thrown by Hypothesis.""" -class CleanupFailed(HypothesisException): - """At least one cleanup task failed and no other exception was raised.""" +class _Trimmable(HypothesisException): + """Hypothesis can trim these tracebacks even if they're raised internally.""" class UnsatisfiedAssumption(HypothesisException): @@ -40,7 +35,7 @@ def __init__(self, condition_string, extra=""): super().__init__(f"No examples found of condition {condition_string}{extra}") -class Unsatisfiable(HypothesisException): +class Unsatisfiable(_Trimmable): """We ran out of time or examples before we could find enough examples which satisfy the assumptions of this hypothesis. @@ -53,7 +48,7 @@ class Unsatisfiable(HypothesisException): """ -class Flaky(HypothesisException): +class Flaky(_Trimmable): """This function appears to fail non-deterministically: We have seen it fail when passed this example at least once, but a subsequent invocation did not fail. @@ -70,7 +65,7 @@ class Flaky(HypothesisException): """ -class InvalidArgument(HypothesisException, TypeError): +class InvalidArgument(_Trimmable, TypeError): """Used to indicate that the arguments to a Hypothesis function were in some manner incorrect.""" @@ -80,7 +75,7 @@ class ResolutionFailed(InvalidArgument): Type inference is best-effort, so this only happens when an annotation exists but could not be resolved for a required argument - to the target of ``builds()``, or where the user passed ``infer``. + to the target of ``builds()``, or where the user passed ``...``. """ @@ -88,7 +83,7 @@ class InvalidState(HypothesisException): """The system is not in a state where you were allowed to do that.""" -class InvalidDefinition(HypothesisException, TypeError): +class InvalidDefinition(_Trimmable, TypeError): """Used to indicate that a class definition was not well put together and has something wrong with it.""" @@ -97,13 +92,8 @@ class HypothesisWarning(HypothesisException, Warning): """A generic warning issued by Hypothesis.""" -class FailedHealthCheck(HypothesisWarning): - """Raised when a test fails a preliminary healthcheck that occurs before - execution.""" - - def __init__(self, message, check): - super().__init__(message) - self.health_check = check +class FailedHealthCheck(_Trimmable): + """Raised when a test fails a healthcheck.""" class NonInteractiveExampleWarning(HypothesisWarning): @@ -129,12 +119,21 @@ class Frozen(HypothesisException): after freeze() has been called.""" -class MultipleFailures(HypothesisException): - """Indicates that Hypothesis found more than one distinct bug when testing - your code.""" +def __getattr__(name): + if name == "MultipleFailures": + from hypothesis._settings import note_deprecation + from hypothesis.internal.compat import BaseExceptionGroup + note_deprecation( + "MultipleFailures is deprecated; use the builtin `BaseExceptionGroup` type " + "instead, or `exceptiongroup.BaseExceptionGroup` before Python 3.11", + since="2022-08-02", + has_codemod=False, # This would be a great PR though! + ) + return BaseExceptionGroup -class DeadlineExceeded(HypothesisException): + +class DeadlineExceeded(_Trimmable): """Raised when an individual test body has taken too long to run.""" def __init__(self, runtime, deadline): @@ -145,6 +144,9 @@ def __init__(self, runtime, deadline): self.runtime = runtime self.deadline = deadline + def __reduce__(self): + return (type(self), (self.runtime, self.deadline)) + class StopTest(BaseException): """Raised when a test should stop running and return control to diff --git a/hypothesis-python/src/hypothesis/executors.py b/hypothesis-python/src/hypothesis/executors.py index 275f6b9012..67895467e4 100644 --- a/hypothesis-python/src/hypothesis/executors.py +++ b/hypothesis-python/src/hypothesis/executors.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER def default_executor(function): # pragma: nocover diff --git a/hypothesis-python/src/hypothesis/extra/__init__.py b/hypothesis-python/src/hypothesis/extra/__init__.py index e61e3c2872..fcb1ac6538 100644 --- a/hypothesis-python/src/hypothesis/extra/__init__.py +++ b/hypothesis-python/src/hypothesis/extra/__init__.py @@ -1,14 +1,9 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER diff --git a/hypothesis-python/src/hypothesis/extra/_array_helpers.py b/hypothesis-python/src/hypothesis/extra/_array_helpers.py new file mode 100644 index 0000000000..5e6718eea8 --- /dev/null +++ b/hypothesis-python/src/hypothesis/extra/_array_helpers.py @@ -0,0 +1,689 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import re +from typing import NamedTuple, Optional, Tuple, Union + +from hypothesis import assume, strategies as st +from hypothesis.errors import InvalidArgument +from hypothesis.internal.conjecture import utils as cu +from hypothesis.internal.coverage import check_function +from hypothesis.internal.validation import check_type, check_valid_interval +from hypothesis.strategies._internal.utils import defines_strategy +from hypothesis.utils.conventions import UniqueIdentifier, not_set + +__all__ = [ + "NDIM_MAX", + "Shape", + "BroadcastableShapes", + "BasicIndex", + "check_argument", + "order_check", + "check_valid_dims", + "array_shapes", + "valid_tuple_axes", + "broadcastable_shapes", + "mutually_broadcastable_shapes", + "MutuallyBroadcastableShapesStrategy", + "BasicIndexStrategy", +] + + +Shape = Tuple[int, ...] +# We silence flake8 here because it disagrees with mypy about `ellipsis` (`type(...)`) +BasicIndex = Tuple[Union[int, slice, None, "ellipsis"], ...] # noqa: F821 + + +class BroadcastableShapes(NamedTuple): + input_shapes: Tuple[Shape, ...] + result_shape: Shape + + +@check_function +def check_argument(condition, fail_message, *f_args, **f_kwargs): + if not condition: + raise InvalidArgument(fail_message.format(*f_args, **f_kwargs)) + + +@check_function +def order_check(name, floor, min_, max_): + if floor > min_: + raise InvalidArgument(f"min_{name} must be at least {floor} but was {min_}") + if min_ > max_: + raise InvalidArgument(f"min_{name}={min_} is larger than max_{name}={max_}") + + +# 32 is a dimension limit specific to NumPy, and does not necessarily apply to +# other array/tensor libraries. Historically these strategies were built for the +# NumPy extra, so it's nice to keep these limits, and it's seemingly unlikely +# someone would want to generate >32 dim arrays anyway. +# See https://github.com/HypothesisWorks/hypothesis/pull/3067. +NDIM_MAX = 32 + + +@check_function +def check_valid_dims(dims, name): + if dims > NDIM_MAX: + raise InvalidArgument( + f"{name}={dims}, but Hypothesis does not support arrays with " + f"more than {NDIM_MAX} dimensions" + ) + + +@defines_strategy() +def array_shapes( + *, + min_dims: int = 1, + max_dims: Optional[int] = None, + min_side: int = 1, + max_side: Optional[int] = None, +) -> st.SearchStrategy[Shape]: + """Return a strategy for array shapes (tuples of int >= 1). + + * ``min_dims`` is the smallest length that the generated shape can possess. + * ``max_dims`` is the largest length that the generated shape can possess, + defaulting to ``min_dims + 2``. + * ``min_side`` is the smallest size that a dimension can possess. + * ``max_side`` is the largest size that a dimension can possess, + defaulting to ``min_side + 5``. + """ + check_type(int, min_dims, "min_dims") + check_type(int, min_side, "min_side") + check_valid_dims(min_dims, "min_dims") + + if max_dims is None: + max_dims = min(min_dims + 2, NDIM_MAX) + check_type(int, max_dims, "max_dims") + check_valid_dims(max_dims, "max_dims") + + if max_side is None: + max_side = min_side + 5 + check_type(int, max_side, "max_side") + + order_check("dims", 0, min_dims, max_dims) + order_check("side", 0, min_side, max_side) + + return st.lists( + st.integers(min_side, max_side), min_size=min_dims, max_size=max_dims + ).map(tuple) + + +@defines_strategy() +def valid_tuple_axes( + ndim: int, + *, + min_size: int = 0, + max_size: Optional[int] = None, +) -> st.SearchStrategy[Tuple[int, ...]]: + """All tuples will have a length >= ``min_size`` and <= ``max_size``. The default + value for ``max_size`` is ``ndim``. + + Examples from this strategy shrink towards an empty tuple, which render most + sequential functions as no-ops. + + The following are some examples drawn from this strategy. + + .. code-block:: pycon + + >>> [valid_tuple_axes(3).example() for i in range(4)] + [(-3, 1), (0, 1, -1), (0, 2), (0, -2, 2)] + + ``valid_tuple_axes`` can be joined with other strategies to generate + any type of valid axis object, i.e. integers, tuples, and ``None``: + + .. code-block:: python + + any_axis_strategy = none() | integers(-ndim, ndim - 1) | valid_tuple_axes(ndim) + + """ + check_type(int, ndim, "ndim") + check_type(int, min_size, "min_size") + if max_size is None: + max_size = ndim + check_type(int, max_size, "max_size") + order_check("size", 0, min_size, max_size) + check_valid_interval(max_size, ndim, "max_size", "ndim") + + axes = st.integers(0, max(0, 2 * ndim - 1)).map( + lambda x: x if x < ndim else x - 2 * ndim + ) + + return st.lists( + axes, min_size=min_size, max_size=max_size, unique_by=lambda x: x % ndim + ).map(tuple) + + +@defines_strategy() +def broadcastable_shapes( + shape: Shape, + *, + min_dims: int = 0, + max_dims: Optional[int] = None, + min_side: int = 1, + max_side: Optional[int] = None, +) -> st.SearchStrategy[Shape]: + """Return a strategy for shapes that are broadcast-compatible with the + provided shape. + + Examples from this strategy shrink towards a shape with length ``min_dims``. + The size of an aligned dimension shrinks towards size ``1``. The size of an + unaligned dimension shrink towards ``min_side``. + + * ``shape`` is a tuple of integers. + * ``min_dims`` is the smallest length that the generated shape can possess. + * ``max_dims`` is the largest length that the generated shape can possess, + defaulting to ``max(len(shape), min_dims) + 2``. + * ``min_side`` is the smallest size that an unaligned dimension can possess. + * ``max_side`` is the largest size that an unaligned dimension can possess, + defaulting to 2 plus the size of the largest aligned dimension. + + The following are some examples drawn from this strategy. + + .. code-block:: pycon + + >>> [broadcastable_shapes(shape=(2, 3)).example() for i in range(5)] + [(1, 3), (), (2, 3), (2, 1), (4, 1, 3), (3, )] + + """ + check_type(tuple, shape, "shape") + check_type(int, min_side, "min_side") + check_type(int, min_dims, "min_dims") + check_valid_dims(min_dims, "min_dims") + + strict_check = max_side is None or max_dims is None + + if max_dims is None: + max_dims = min(max(len(shape), min_dims) + 2, NDIM_MAX) + check_type(int, max_dims, "max_dims") + check_valid_dims(max_dims, "max_dims") + + if max_side is None: + max_side = max(shape[-max_dims:] + (min_side,)) + 2 + check_type(int, max_side, "max_side") + + order_check("dims", 0, min_dims, max_dims) + order_check("side", 0, min_side, max_side) + + if strict_check: + dims = max_dims + bound_name = "max_dims" + else: + dims = min_dims + bound_name = "min_dims" + + # check for unsatisfiable min_side + if not all(min_side <= s for s in shape[::-1][:dims] if s != 1): + raise InvalidArgument( + f"Given shape={shape}, there are no broadcast-compatible " + f"shapes that satisfy: {bound_name}={dims} and min_side={min_side}" + ) + + # check for unsatisfiable [min_side, max_side] + if not ( + min_side <= 1 <= max_side or all(s <= max_side for s in shape[::-1][:dims]) + ): + raise InvalidArgument( + f"Given base_shape={shape}, there are no broadcast-compatible " + f"shapes that satisfy all of {bound_name}={dims}, " + f"min_side={min_side}, and max_side={max_side}" + ) + + if not strict_check: + # reduce max_dims to exclude unsatisfiable dimensions + for n, s in zip(range(max_dims), shape[::-1]): + if s < min_side and s != 1: + max_dims = n + break + elif not (min_side <= 1 <= max_side or s <= max_side): + max_dims = n + break + + return MutuallyBroadcastableShapesStrategy( + num_shapes=1, + base_shape=shape, + min_dims=min_dims, + max_dims=max_dims, + min_side=min_side, + max_side=max_side, + ).map(lambda x: x.input_shapes[0]) + + +# See https://numpy.org/doc/stable/reference/c-api/generalized-ufuncs.html +# Implementation based on numpy.lib.function_base._parse_gufunc_signature +# with minor upgrades to handle numeric and optional dimensions. Examples: +# +# add (),()->() binary ufunc +# sum1d (i)->() reduction +# inner1d (i),(i)->() vector-vector multiplication +# matmat (m,n),(n,p)->(m,p) matrix multiplication +# vecmat (n),(n,p)->(p) vector-matrix multiplication +# matvec (m,n),(n)->(m) matrix-vector multiplication +# matmul (m?,n),(n,p?)->(m?,p?) combination of the four above +# cross1d (3),(3)->(3) cross product with frozen dimensions +# +# Note that while no examples of such usage are given, Numpy does allow +# generalised ufuncs that have *multiple output arrays*. This is not +# currently supported by Hypothesis - please contact us if you would use it! +# +# We are unsure if gufuncs allow frozen dimensions to be optional, but it's +# easy enough to support here - and so we will unless we learn otherwise. +_DIMENSION = r"\w+\??" # Note that \w permits digits too! +_SHAPE = r"\((?:{0}(?:,{0})".format(_DIMENSION) + r"{0,31})?\)" +_ARGUMENT_LIST = "{0}(?:,{0})*".format(_SHAPE) +_SIGNATURE = rf"^{_ARGUMENT_LIST}->{_SHAPE}$" +_SIGNATURE_MULTIPLE_OUTPUT = r"^{0}->{0}$".format(_ARGUMENT_LIST) + + +class _GUfuncSig(NamedTuple): + input_shapes: Tuple[Shape, ...] + result_shape: Shape + + +def _hypothesis_parse_gufunc_signature(signature, all_checks=True): + # Disable all_checks to better match the Numpy version, for testing + if not re.match(_SIGNATURE, signature): + if re.match(_SIGNATURE_MULTIPLE_OUTPUT, signature): + raise InvalidArgument( + "Hypothesis does not yet support generalised ufunc signatures " + "with multiple output arrays - mostly because we don't know of " + "anyone who uses them! Please get in touch with us to fix that." + f"\n (signature={signature!r})" + ) + if re.match( + ( + # Taken from np.lib.function_base._SIGNATURE + r"^\((?:\w+(?:,\w+)*)?\)(?:,\((?:\w+(?:,\w+)*)?\))*->" + r"\((?:\w+(?:,\w+)*)?\)(?:,\((?:\w+(?:,\w+)*)?\))*$" + ), + signature, + ): + raise InvalidArgument( + f"signature={signature!r} matches Numpy's regex for gufunc signatures, " + f"but contains shapes with more than {NDIM_MAX} dimensions and is thus invalid." + ) + raise InvalidArgument(f"{signature!r} is not a valid gufunc signature") + input_shapes, output_shapes = ( + tuple(tuple(re.findall(_DIMENSION, a)) for a in re.findall(_SHAPE, arg_list)) + for arg_list in signature.split("->") + ) + assert len(output_shapes) == 1 + result_shape = output_shapes[0] + if all_checks: + # Check that there are no names in output shape that do not appear in inputs. + # (kept out of parser function for easier generation of test values) + # We also disallow frozen optional dimensions - this is ambiguous as there is + # no way to share an un-named dimension between shapes. Maybe just padding? + # Anyway, we disallow it pending clarification from upstream. + frozen_optional_err = ( + "Got dimension %r, but handling of frozen optional dimensions " + "is ambiguous. If you known how this should work, please " + "contact us to get this fixed and documented (signature=%r)." + ) + only_out_err = ( + "The %r dimension only appears in the output shape, and is " + "not frozen, so the size is not determined (signature=%r)." + ) + names_in = {n.strip("?") for shp in input_shapes for n in shp} + names_out = {n.strip("?") for n in result_shape} + for shape in input_shapes + (result_shape,): + for name in shape: + try: + int(name.strip("?")) + if "?" in name: + raise InvalidArgument(frozen_optional_err % (name, signature)) + except ValueError: + if name.strip("?") in (names_out - names_in): + raise InvalidArgument( + only_out_err % (name, signature) + ) from None + return _GUfuncSig(input_shapes=input_shapes, result_shape=result_shape) + + +@defines_strategy() +def mutually_broadcastable_shapes( + *, + num_shapes: Union[UniqueIdentifier, int] = not_set, + signature: Union[UniqueIdentifier, str] = not_set, + base_shape: Shape = (), + min_dims: int = 0, + max_dims: Optional[int] = None, + min_side: int = 1, + max_side: Optional[int] = None, +) -> st.SearchStrategy[BroadcastableShapes]: + """Return a strategy for a specified number of shapes N that are + mutually-broadcastable with one another and with the provided base shape. + + * ``num_shapes`` is the number of mutually broadcast-compatible shapes to generate. + * ``base_shape`` is the shape against which all generated shapes can broadcast. + The default shape is empty, which corresponds to a scalar and thus does + not constrain broadcasting at all. + * ``min_dims`` is the smallest length that the generated shape can possess. + * ``max_dims`` is the largest length that the generated shape can possess, + defaulting to ``max(len(shape), min_dims) + 2``. + * ``min_side`` is the smallest size that an unaligned dimension can possess. + * ``max_side`` is the largest size that an unaligned dimension can possess, + defaulting to 2 plus the size of the largest aligned dimension. + + The strategy will generate a :obj:`python:typing.NamedTuple` containing: + + * ``input_shapes`` as a tuple of the N generated shapes. + * ``result_shape`` as the resulting shape produced by broadcasting the N shapes + with the base shape. + + The following are some examples drawn from this strategy. + + .. code-block:: pycon + + >>> # Draw three shapes where each shape is broadcast-compatible with (2, 3) + ... strat = mutually_broadcastable_shapes(num_shapes=3, base_shape=(2, 3)) + >>> for _ in range(5): + ... print(strat.example()) + BroadcastableShapes(input_shapes=((4, 1, 3), (4, 2, 3), ()), result_shape=(4, 2, 3)) + BroadcastableShapes(input_shapes=((3,), (1, 3), (2, 3)), result_shape=(2, 3)) + BroadcastableShapes(input_shapes=((), (), ()), result_shape=()) + BroadcastableShapes(input_shapes=((3,), (), (3,)), result_shape=(3,)) + BroadcastableShapes(input_shapes=((1, 2, 3), (3,), ()), result_shape=(1, 2, 3)) + + """ + arg_msg = "Pass either the `num_shapes` or the `signature` argument, but not both." + if num_shapes is not not_set: + check_argument(signature is not_set, arg_msg) + check_type(int, num_shapes, "num_shapes") + assert isinstance(num_shapes, int) # for mypy + parsed_signature = None + sig_dims = 0 + else: + check_argument(signature is not not_set, arg_msg) + if signature is None: + raise InvalidArgument( + "Expected a string, but got invalid signature=None. " + "(maybe .signature attribute of an element-wise ufunc?)" + ) + check_type(str, signature, "signature") + parsed_signature = _hypothesis_parse_gufunc_signature(signature) + all_shapes = parsed_signature.input_shapes + (parsed_signature.result_shape,) + sig_dims = min(len(s) for s in all_shapes) + num_shapes = len(parsed_signature.input_shapes) + + if num_shapes < 1: + raise InvalidArgument(f"num_shapes={num_shapes} must be at least 1") + + check_type(tuple, base_shape, "base_shape") + check_type(int, min_side, "min_side") + check_type(int, min_dims, "min_dims") + check_valid_dims(min_dims, "min_dims") + + strict_check = max_dims is not None + + if max_dims is None: + max_dims = min(max(len(base_shape), min_dims) + 2, NDIM_MAX - sig_dims) + check_type(int, max_dims, "max_dims") + check_valid_dims(max_dims, "max_dims") + + if max_side is None: + max_side = max(base_shape[-max_dims:] + (min_side,)) + 2 + check_type(int, max_side, "max_side") + + order_check("dims", 0, min_dims, max_dims) + order_check("side", 0, min_side, max_side) + + if signature is not None and max_dims > NDIM_MAX - sig_dims: + raise InvalidArgument( + f"max_dims={signature!r} would exceed the {NDIM_MAX}-dimension" + "limit Hypothesis imposes on array shapes, " + f"given signature={parsed_signature!r}" + ) + + if strict_check: + dims = max_dims + bound_name = "max_dims" + else: + dims = min_dims + bound_name = "min_dims" + + # check for unsatisfiable min_side + if not all(min_side <= s for s in base_shape[::-1][:dims] if s != 1): + raise InvalidArgument( + f"Given base_shape={base_shape}, there are no broadcast-compatible " + f"shapes that satisfy: {bound_name}={dims} and min_side={min_side}" + ) + + # check for unsatisfiable [min_side, max_side] + if not ( + min_side <= 1 <= max_side or all(s <= max_side for s in base_shape[::-1][:dims]) + ): + raise InvalidArgument( + f"Given base_shape={base_shape}, there are no broadcast-compatible " + f"shapes that satisfy all of {bound_name}={dims}, " + f"min_side={min_side}, and max_side={max_side}" + ) + + if not strict_check: + # reduce max_dims to exclude unsatisfiable dimensions + for n, s in zip(range(max_dims), base_shape[::-1]): + if s < min_side and s != 1: + max_dims = n + break + elif not (min_side <= 1 <= max_side or s <= max_side): + max_dims = n + break + + return MutuallyBroadcastableShapesStrategy( + num_shapes=num_shapes, + signature=parsed_signature, + base_shape=base_shape, + min_dims=min_dims, + max_dims=max_dims, + min_side=min_side, + max_side=max_side, + ) + + +class MutuallyBroadcastableShapesStrategy(st.SearchStrategy): + def __init__( + self, + num_shapes, + signature=None, + base_shape=(), + min_dims=0, + max_dims=None, + min_side=1, + max_side=None, + ): + super().__init__() + self.base_shape = base_shape + self.side_strat = st.integers(min_side, max_side) + self.num_shapes = num_shapes + self.signature = signature + self.min_dims = min_dims + self.max_dims = max_dims + self.min_side = min_side + self.max_side = max_side + + self.size_one_allowed = self.min_side <= 1 <= self.max_side + + def do_draw(self, data): + # We don't usually have a gufunc signature; do the common case first & fast. + if self.signature is None: + return self._draw_loop_dimensions(data) + + # When we *do*, draw the core dims, then draw loop dims, and finally combine. + core_in, core_res = self._draw_core_dimensions(data) + + # If some core shape has omitted optional dimensions, it's an error to add + # loop dimensions to it. We never omit core dims if min_dims >= 1. + # This ensures that we respect Numpy's gufunc broadcasting semantics and user + # constraints without needing to check whether the loop dims will be + # interpreted as an invalid substitute for the omitted core dims. + # We may implement this check later! + use = [None not in shp for shp in core_in] + loop_in, loop_res = self._draw_loop_dimensions(data, use=use) + + def add_shape(loop, core): + return tuple(x for x in (loop + core)[-NDIM_MAX:] if x is not None) + + return BroadcastableShapes( + input_shapes=tuple(add_shape(l_in, c) for l_in, c in zip(loop_in, core_in)), + result_shape=add_shape(loop_res, core_res), + ) + + def _draw_core_dimensions(self, data): + # Draw gufunc core dimensions, with None standing for optional dimensions + # that will not be present in the final shape. We track omitted dims so + # that we can do an accurate per-shape length cap. + dims = {} + shapes = [] + for shape in self.signature.input_shapes + (self.signature.result_shape,): + shapes.append([]) + for name in shape: + if name.isdigit(): + shapes[-1].append(int(name)) + continue + if name not in dims: + dim = name.strip("?") + dims[dim] = data.draw(self.side_strat) + if self.min_dims == 0 and not data.draw_bits(3): + dims[dim + "?"] = None + else: + dims[dim + "?"] = dims[dim] + shapes[-1].append(dims[name]) + return tuple(tuple(s) for s in shapes[:-1]), tuple(shapes[-1]) + + def _draw_loop_dimensions(self, data, use=None): + # All shapes are handled in column-major order; i.e. they are reversed + base_shape = self.base_shape[::-1] + result_shape = list(base_shape) + shapes = [[] for _ in range(self.num_shapes)] + if use is None: + use = [True for _ in range(self.num_shapes)] + else: + assert len(use) == self.num_shapes + assert all(isinstance(x, bool) for x in use) + + for dim_count in range(1, self.max_dims + 1): + dim = dim_count - 1 + + # We begin by drawing a valid dimension-size for the given + # dimension. This restricts the variability across the shapes + # at this dimension such that they can only choose between + # this size and a singleton dimension. + if len(base_shape) < dim_count or base_shape[dim] == 1: + # dim is unrestricted by the base-shape: shrink to min_side + dim_side = data.draw(self.side_strat) + elif base_shape[dim] <= self.max_side: + # dim is aligned with non-singleton base-dim + dim_side = base_shape[dim] + else: + # only a singleton is valid in alignment with the base-dim + dim_side = 1 + + allowed_sides = sorted([1, dim_side]) # shrink to 0 when available + for shape_id, shape in enumerate(shapes): + # Populating this dimension-size for each shape, either + # the drawn size is used or, if permitted, a singleton + # dimension. + if dim <= len(result_shape) and self.size_one_allowed: + # aligned: shrink towards size 1 + side = data.draw(st.sampled_from(allowed_sides)) + else: + side = dim_side + + # Use a trick where where a biased coin is queried to see + # if the given shape-tuple will continue to be grown. All + # of the relevant draws will still be made for the given + # shape-tuple even if it is no longer being added to. + # This helps to ensure more stable shrinking behavior. + if self.min_dims < dim_count: + use[shape_id] &= cu.biased_coin( + data, 1 - 1 / (1 + self.max_dims - dim) + ) + + if use[shape_id]: + shape.append(side) + if len(result_shape) < len(shape): + result_shape.append(shape[-1]) + elif shape[-1] != 1 and result_shape[dim] == 1: + result_shape[dim] = shape[-1] + if not any(use): + break + + result_shape = result_shape[: max(map(len, [self.base_shape] + shapes))] + + assert len(shapes) == self.num_shapes + assert all(self.min_dims <= len(s) <= self.max_dims for s in shapes) + assert all(self.min_side <= s <= self.max_side for side in shapes for s in side) + + return BroadcastableShapes( + input_shapes=tuple(tuple(reversed(shape)) for shape in shapes), + result_shape=tuple(reversed(result_shape)), + ) + + +class BasicIndexStrategy(st.SearchStrategy): + def __init__( + self, + shape, + min_dims, + max_dims, + allow_ellipsis, + allow_newaxis, + allow_fewer_indices_than_dims, + ): + self.shape = shape + self.min_dims = min_dims + self.max_dims = max_dims + self.allow_ellipsis = allow_ellipsis + self.allow_newaxis = allow_newaxis + # allow_fewer_indices_than_dims=False will disable generating indices + # that don't cover all axes, i.e. indices that will flat index arrays. + # This is necessary for the Array API as such indices are not supported. + self.allow_fewer_indices_than_dims = allow_fewer_indices_than_dims + + def do_draw(self, data): + # General plan: determine the actual selection up front with a straightforward + # approach that shrinks well, then complicate it by inserting other things. + result = [] + for dim_size in self.shape: + if dim_size == 0: + result.append(slice(None)) + continue + strategy = st.integers(-dim_size, dim_size - 1) | st.slices(dim_size) + result.append(data.draw(strategy)) + # Insert some number of new size-one dimensions if allowed + result_dims = sum(isinstance(idx, slice) for idx in result) + while ( + self.allow_newaxis + and result_dims < self.max_dims + and (result_dims < self.min_dims or data.draw(st.booleans())) + ): + i = data.draw(st.integers(0, len(result))) + result.insert(i, None) # Note that `np.newaxis is None` + result_dims += 1 + # Check that we'll have the right number of dimensions; reject if not. + # It's easy to do this by construction if you don't care about shrinking, + # which is really important for array shapes. So we filter instead. + assume(self.min_dims <= result_dims <= self.max_dims) + # This is a quick-and-dirty way to insert ..., xor shorten the indexer, + # but it means we don't have to do any structural analysis. + if self.allow_ellipsis and data.draw(st.booleans()): + # Choose an index; then replace all adjacent whole-dimension slices. + i = j = data.draw(st.integers(0, len(result))) + while i > 0 and result[i - 1] == slice(None): + i -= 1 + while j < len(result) and result[j] == slice(None): + j += 1 + result[i:j] = [Ellipsis] + elif self.allow_fewer_indices_than_dims: # pragma: no cover + while result[-1:] == [slice(None, None)] and data.draw(st.integers(0, 7)): + result.pop() + if len(result) == 1 and data.draw(st.booleans()): + # Sometimes generate bare element equivalent to a length-one tuple + return result[0] + return tuple(result) diff --git a/hypothesis-python/src/hypothesis/extra/array_api.py b/hypothesis-python/src/hypothesis/extra/array_api.py new file mode 100644 index 0000000000..1309593455 --- /dev/null +++ b/hypothesis-python/src/hypothesis/extra/array_api.py @@ -0,0 +1,1178 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import sys + +if sys.version_info[:2] < (3, 8): + raise RuntimeError("The Array API standard requires Python 3.8 or later") + +import math +from numbers import Real +from types import SimpleNamespace +from typing import ( + Any, + Iterable, + Iterator, + List, + Literal, + Mapping, + NamedTuple, + Optional, + Sequence, + Tuple, + Type, + TypeVar, + Union, + get_args, +) +from warnings import warn +from weakref import WeakValueDictionary + +from hypothesis import strategies as st +from hypothesis.errors import HypothesisWarning, InvalidArgument +from hypothesis.extra._array_helpers import ( + NDIM_MAX, + BasicIndex, + BasicIndexStrategy, + BroadcastableShapes, + Shape, + array_shapes, + broadcastable_shapes, + check_argument, + check_valid_dims, + mutually_broadcastable_shapes as _mutually_broadcastable_shapes, + order_check, + valid_tuple_axes as _valid_tuple_axes, +) +from hypothesis.internal.conjecture import utils as cu +from hypothesis.internal.coverage import check_function +from hypothesis.internal.floats import next_down +from hypothesis.internal.reflection import proxies +from hypothesis.internal.validation import ( + check_type, + check_valid_bound, + check_valid_integer, + check_valid_interval, +) +from hypothesis.strategies._internal.strategies import check_strategy +from hypothesis.strategies._internal.utils import defines_strategy + +__all__ = [ + "make_strategies_namespace", +] + + +RELEASED_VERSIONS = ("2021.12",) +NOMINAL_VERSIONS = RELEASED_VERSIONS + ("draft",) +assert sorted(NOMINAL_VERSIONS) == list(NOMINAL_VERSIONS) # sanity check +NominalVersion = Literal["2021.12", "draft"] +assert get_args(NominalVersion) == NOMINAL_VERSIONS # sanity check + + +INT_NAMES = ("int8", "int16", "int32", "int64") +UINT_NAMES = ("uint8", "uint16", "uint32", "uint64") +ALL_INT_NAMES = INT_NAMES + UINT_NAMES +FLOAT_NAMES = ("float32", "float64") +REAL_NAMES = ALL_INT_NAMES + FLOAT_NAMES +COMPLEX_NAMES = ("complex64", "complex128") +NUMERIC_NAMES = REAL_NAMES + COMPLEX_NAMES +DTYPE_NAMES = ("bool",) + NUMERIC_NAMES + +DataType = TypeVar("DataType") + + +@check_function +def check_xp_attributes(xp: Any, attributes: List[str]) -> None: + missing_attrs = [attr for attr in attributes if not hasattr(xp, attr)] + if len(missing_attrs) > 0: + f_attrs = ", ".join(missing_attrs) + raise InvalidArgument( + f"Array module {xp.__name__} does not have required attributes: {f_attrs}" + ) + + +def partition_attributes_and_stubs( + xp: Any, attributes: Iterable[str] +) -> Tuple[List[Any], List[str]]: + non_stubs = [] + stubs = [] + for attr in attributes: + try: + non_stubs.append(getattr(xp, attr)) + except AttributeError: + stubs.append(attr) + + return non_stubs, stubs + + +def warn_on_missing_dtypes(xp: Any, stubs: List[str]) -> None: + f_stubs = ", ".join(stubs) + warn( + f"Array module {xp.__name__} does not have the following " + f"dtypes in its namespace: {f_stubs}", + HypothesisWarning, + ) + + +def find_castable_builtin_for_dtype( + xp: Any, api_version: NominalVersion, dtype: DataType +) -> Type[Union[bool, int, float, complex]]: + """Returns builtin type which can have values that are castable to the given + dtype, according to :xp-ref:`type promotion rules `. + + For floating dtypes we always return ``float``, even though ``int`` is also castable. + """ + stubs = [] + + try: + bool_dtype = xp.bool + if dtype == bool_dtype: + return bool + except AttributeError: + stubs.append("bool") + + int_dtypes, int_stubs = partition_attributes_and_stubs(xp, ALL_INT_NAMES) + if dtype in int_dtypes: + return int + + float_dtypes, float_stubs = partition_attributes_and_stubs(xp, FLOAT_NAMES) + # None equals NumPy's xp.float64 object, so we specifically skip it here to + # ensure that InvalidArgument is still raised. xp.float64 is in fact an + # alias of np.dtype('float64'), and its equality with None is meant to be + # deprecated at some point. See https://github.com/numpy/numpy/issues/18434 + if dtype is not None and dtype in float_dtypes: + return float + + stubs.extend(int_stubs) + stubs.extend(float_stubs) + + if api_version > "2021.12": + complex_dtypes, complex_stubs = partition_attributes_and_stubs( + xp, COMPLEX_NAMES + ) + if dtype in complex_dtypes: + return complex + stubs.extend(complex_stubs) + + if len(stubs) > 0: + warn_on_missing_dtypes(xp, stubs) + raise InvalidArgument(f"dtype={dtype} not recognised in {xp.__name__}") + + +@check_function +def dtype_from_name(xp: Any, name: str) -> Any: + if name in DTYPE_NAMES: + try: + return getattr(xp, name) + except AttributeError as e: + raise InvalidArgument( + f"Array module {xp.__name__} does not have dtype {name} in its namespace" + ) from e + else: + f_valid_dtypes = ", ".join(DTYPE_NAMES) + raise InvalidArgument( + f"{name} is not a valid Array API data type (pick from: {f_valid_dtypes})" + ) + + +def _from_dtype( + xp: Any, + api_version: NominalVersion, + dtype: Union[DataType, str], + *, + min_value: Optional[Union[int, float]] = None, + max_value: Optional[Union[int, float]] = None, + allow_nan: Optional[bool] = None, + allow_infinity: Optional[bool] = None, + allow_subnormal: Optional[bool] = None, + exclude_min: Optional[bool] = None, + exclude_max: Optional[bool] = None, +) -> st.SearchStrategy[Union[bool, int, float]]: + """Return a strategy for any value of the given dtype. + + Values generated are of the Python scalar which is + :xp-ref:`promotable ` to ``dtype``, where the values do + not exceed its bounds. + + * ``dtype`` may be a dtype object or the string name of a + :xp-ref:`valid dtype `. + + Compatible ``**kwargs`` are passed to the inferred strategy function for + integers and floats. This allows you to customise the min and max values, + and exclude non-finite numbers. This is particularly useful when kwargs are + passed through from :func:`arrays()`, as it seamlessly handles the ``width`` + or other representable bounds for you. + """ + # TODO: for next released xp version, add note for complex dtype support + check_xp_attributes(xp, ["iinfo", "finfo"]) + + if isinstance(dtype, str): + dtype = dtype_from_name(xp, dtype) + builtin = find_castable_builtin_for_dtype(xp, api_version, dtype) + + def check_valid_minmax(prefix, val, info_obj): + name = f"{prefix}_value" + check_valid_bound(val, name) + check_argument( + val >= info_obj.min, + f"dtype={dtype} requires {name}={val} to be at least {info_obj.min}", + ) + check_argument( + val <= info_obj.max, + f"dtype={dtype} requires {name}={val} to be at most {info_obj.max}", + ) + + if builtin is bool: + return st.booleans() + elif builtin is int: + iinfo = xp.iinfo(dtype) + if min_value is None: + min_value = iinfo.min + if max_value is None: + max_value = iinfo.max + check_valid_integer(min_value, "min_value") + check_valid_integer(max_value, "max_value") + assert isinstance(min_value, int) + assert isinstance(max_value, int) + check_valid_minmax("min", min_value, iinfo) + check_valid_minmax("max", max_value, iinfo) + check_valid_interval(min_value, max_value, "min_value", "max_value") + return st.integers(min_value=min_value, max_value=max_value) + elif builtin is float: + finfo = xp.finfo(dtype) + kw = {} + + # Whilst we know the boundary values of float dtypes from finfo, we do + # not assign them to the floats() strategy by default - passing min/max + # values will modify test case reduction behaviour so that simple bugs + # may become harder for users to identify. We plan to improve floats() + # behaviour in https://github.com/HypothesisWorks/hypothesis/issues/2907. + # Setting width should manage boundary values for us anyway. + if min_value is not None: + check_valid_bound(min_value, "min_value") + assert isinstance(min_value, Real) + check_valid_minmax("min", min_value, finfo) + kw["min_value"] = min_value + if max_value is not None: + check_valid_bound(max_value, "max_value") + assert isinstance(max_value, Real) + check_valid_minmax("max", max_value, finfo) + if min_value is not None: + check_valid_interval(min_value, max_value, "min_value", "max_value") + kw["max_value"] = max_value + + # We infer whether an array module will flush subnormals to zero, as may + # be the case when libraries are built with compiler options that + # violate IEEE-754 (e.g. -ffast-math and -ftz=true). Note we do this for + # the specific dtype, as compilers may end up flushing subnormals for + # one float but supporting subnormals for the other. + # + # By default, floats() will generate subnormals if they are in the + # inferred values range. If we have detected that xp flushes to zero for + # the passed dtype, we ensure from_dtype() will not generate subnormals + # by default. + if allow_subnormal is not None: + kw["allow_subnormal"] = allow_subnormal + else: + subnormal = next_down(finfo.smallest_normal, width=finfo.bits) + ftz = bool(xp.asarray(subnormal, dtype=dtype) == 0) + if ftz: + kw["allow_subnormal"] = False + + if allow_nan is not None: + kw["allow_nan"] = allow_nan + if allow_infinity is not None: + kw["allow_infinity"] = allow_infinity + if exclude_min is not None: + kw["exclude_min"] = exclude_min + if exclude_max is not None: + kw["exclude_max"] = exclude_max + + return st.floats(width=finfo.bits, **kw) + else: + # A less-inelegant solution to support complex dtypes exists, but as + # this is currently a draft feature, we might as well wait for + # discussion of complex inspection to resolve first - a better method + # might become available soon enough. + # See https://github.com/data-apis/array-api/issues/433 + for attr in ["float32", "float64", "complex64"]: + if not hasattr(xp, attr): + raise NotImplementedError( + f"Array module {xp.__name__} has no dtype {attr}, which is " + "currently required for xps.from_dtype() to work with " + "any complex dtype." + ) + component_dtype = xp.float32 if dtype == xp.complex64 else xp.float64 + + floats = _from_dtype( + xp, + api_version, + component_dtype, + allow_nan=allow_nan, + allow_infinity=allow_infinity, + allow_subnormal=allow_subnormal, + ) + + return st.builds(complex, floats, floats) # type: ignore[arg-type] + + +class ArrayStrategy(st.SearchStrategy): + def __init__( + self, *, xp, api_version, elements_strategy, dtype, shape, fill, unique + ): + self.xp = xp + self.elements_strategy = elements_strategy + self.dtype = dtype + self.shape = shape + self.fill = fill + self.unique = unique + self.array_size = math.prod(shape) + self.builtin = find_castable_builtin_for_dtype(xp, api_version, dtype) + self.finfo = None if self.builtin is not float else xp.finfo(self.dtype) + + def check_set_value(self, val, val_0d, strategy): + if val == val and self.builtin(val_0d) != val: + if self.builtin is float: + assert self.finfo is not None # for mypy + try: + is_subnormal = 0 < abs(val) < self.finfo.smallest_normal + except Exception: + # val may be a non-float that does not support the + # operations __lt__ and __abs__ + is_subnormal = False + if is_subnormal: + raise InvalidArgument( + f"Generated subnormal float {val} from strategy " + f"{strategy} resulted in {val_0d!r}, probably " + f"as a result of array module {self.xp.__name__} " + "being built with flush-to-zero compiler options. " + "Consider passing allow_subnormal=False." + ) + raise InvalidArgument( + f"Generated array element {val!r} from strategy {strategy} " + f"cannot be represented with dtype {self.dtype}. " + f"Array module {self.xp.__name__} instead " + f"represents the element as {val_0d}. " + "Consider using a more precise elements strategy, " + "for example passing the width argument to floats()." + ) + + def do_draw(self, data): + if 0 in self.shape: + return self.xp.zeros(self.shape, dtype=self.dtype) + + if self.fill.is_empty: + # We have no fill value (either because the user explicitly + # disabled it or because the default behaviour was used and our + # elements strategy does not produce reusable values), so we must + # generate a fully dense array with a freshly drawn value for each + # entry. + elems = data.draw( + st.lists( + self.elements_strategy, + min_size=self.array_size, + max_size=self.array_size, + unique=self.unique, + ) + ) + try: + result = self.xp.asarray(elems, dtype=self.dtype) + except Exception as e: + if len(elems) <= 6: + f_elems = str(elems) + else: + f_elems = f"[{elems[0]}, {elems[1]}, ..., {elems[-2]}, {elems[-1]}]" + types = tuple( + sorted({type(e) for e in elems}, key=lambda t: t.__name__) + ) + f_types = f"type {types[0]}" if len(types) == 1 else f"types {types}" + raise InvalidArgument( + f"Generated elements {f_elems} from strategy " + f"{self.elements_strategy} could not be converted " + f"to array of dtype {self.dtype}. " + f"Consider if elements of {f_types} " + f"are compatible with {self.dtype}." + ) from e + for i in range(self.array_size): + self.check_set_value(elems[i], result[i], self.elements_strategy) + else: + # We draw arrays as "sparse with an offset". We assume not every + # element will be assigned and so first draw a single value from our + # fill strategy to create a full array. We then draw a collection of + # index assignments within the array and assign fresh values from + # our elements strategy to those indices. + + fill_val = data.draw(self.fill) + try: + result = self.xp.full(self.array_size, fill_val, dtype=self.dtype) + except Exception as e: + raise InvalidArgument( + f"Could not create full array of dtype={self.dtype} " + f"with fill value {fill_val!r}" + ) from e + sample = result[0] + self.check_set_value(fill_val, sample, self.fill) + if self.unique and not self.xp.all(self.xp.isnan(result)): + raise InvalidArgument( + f"Array module {self.xp.__name__} did not recognise fill " + f"value {fill_val!r} as NaN - instead got {sample!r}. " + "Cannot fill unique array with non-NaN values." + ) + + elements = cu.many( + data, + min_size=0, + max_size=self.array_size, + # sqrt isn't chosen for any particularly principled reason. It + # just grows reasonably quickly but sublinearly, and for small + # arrays it represents a decent fraction of the array size. + average_size=min( + 0.9 * self.array_size, # ensure small arrays sometimes use fill + max(10, math.sqrt(self.array_size)), # ...but *only* sometimes + ), + ) + + assigned = set() + seen = set() + + while elements.more(): + i = cu.integer_range(data, 0, self.array_size - 1) + if i in assigned: + elements.reject() + continue + val = data.draw(self.elements_strategy) + if self.unique: + if val in seen: + elements.reject() + continue + else: + seen.add(val) + try: + result[i] = val + except Exception as e: + raise InvalidArgument( + f"Could not add generated array element {val!r} " + f"of type {type(val)} to array of dtype {result.dtype}." + ) from e + self.check_set_value(val, result[i], self.elements_strategy) + assigned.add(i) + + result = self.xp.reshape(result, self.shape) + + return result + + +def _arrays( + xp: Any, + api_version: NominalVersion, + dtype: Union[DataType, str, st.SearchStrategy[DataType], st.SearchStrategy[str]], + shape: Union[int, Shape, st.SearchStrategy[Shape]], + *, + elements: Optional[Union[Mapping[str, Any], st.SearchStrategy]] = None, + fill: Optional[st.SearchStrategy[Any]] = None, + unique: bool = False, +) -> st.SearchStrategy: + """Returns a strategy for :xp-ref:`arrays `. + + * ``dtype`` may be a :xp-ref:`valid dtype ` object or name, + or a strategy that generates such values. + * ``shape`` may be an integer >= 0, a tuple of such integers, or a strategy + that generates such values. + * ``elements`` is a strategy for values to put in the array. If ``None`` + then a suitable value will be inferred based on the dtype, which may give + any legal value (including e.g. NaN for floats). If a mapping, it will be + passed as ``**kwargs`` to :func:`from_dtype()` when inferring based on the dtype. + * ``fill`` is a strategy that may be used to generate a single background + value for the array. If ``None``, a suitable default will be inferred + based on the other arguments. If set to + :func:`~hypothesis.strategies.nothing` then filling behaviour will be + disabled entirely and every element will be generated independently. + * ``unique`` specifies if the elements of the array should all be distinct + from one another; if fill is also set, the only valid values for fill to + return are NaN values. + + Arrays of specified ``dtype`` and ``shape`` are generated for example + like this: + + .. code-block:: pycon + + >>> from numpy import array_api as xp + >>> xps.arrays(xp, xp.int8, (2, 3)).example() + Array([[-8, 6, 3], + [-6, 4, 6]], dtype=int8) + + Specifying element boundaries by a :obj:`python:dict` of the kwargs to pass + to :func:`from_dtype` will ensure ``dtype`` bounds will be respected. + + .. code-block:: pycon + + >>> xps.arrays(xp, xp.int8, 3, elements={"min_value": 10}).example() + Array([125, 13, 79], dtype=int8) + + Refer to :doc:`What you can generate and how ` for passing + your own elements strategy. + + .. code-block:: pycon + + >>> xps.arrays(xp, xp.float32, 3, elements=floats(0, 1, width=32)).example() + Array([ 0.88974794, 0.77387938, 0.1977879 ], dtype=float32) + + Array values are generated in two parts: + + 1. A single value is drawn from the fill strategy and is used to create a + filled array. + 2. Some subset of the coordinates of the array are populated with a value + drawn from the elements strategy (or its inferred form). + + You can set ``fill`` to :func:`~hypothesis.strategies.nothing` if you want + to disable this behaviour and draw a value for every element. + + By default ``arrays`` will attempt to infer the correct fill behaviour: if + ``unique`` is also ``True``, no filling will occur. Otherwise, if it looks + safe to reuse the values of elements across multiple coordinates (this will + be the case for any inferred strategy, and for most of the builtins, but is + not the case for mutable values or strategies built with flatmap, map, + composite, etc.) then it will use the elements strategy as the fill, else it + will default to having no fill. + + Having a fill helps Hypothesis craft high quality examples, but its + main importance is when the array generated is large: Hypothesis is + primarily designed around testing small examples. If you have arrays with + hundreds or more elements, having a fill value is essential if you want + your tests to run in reasonable time. + """ + check_xp_attributes( + xp, ["finfo", "asarray", "zeros", "full", "all", "isnan", "isfinite", "reshape"] + ) + + if isinstance(dtype, st.SearchStrategy): + return dtype.flatmap( + lambda d: _arrays( + xp, api_version, d, shape, elements=elements, fill=fill, unique=unique + ) + ) + elif isinstance(dtype, str): + dtype = dtype_from_name(xp, dtype) + + if isinstance(shape, st.SearchStrategy): + return shape.flatmap( + lambda s: _arrays( + xp, api_version, dtype, s, elements=elements, fill=fill, unique=unique + ) + ) + elif isinstance(shape, int): + shape = (shape,) + elif not isinstance(shape, tuple): + raise InvalidArgument(f"shape={shape} is not a valid shape or strategy") + check_argument( + all(isinstance(x, int) and x >= 0 for x in shape), + f"shape={shape!r}, but all dimensions must be non-negative integers.", + ) + + if elements is None: + elements = _from_dtype(xp, api_version, dtype) + elif isinstance(elements, Mapping): + elements = _from_dtype(xp, api_version, dtype, **elements) + check_strategy(elements, "elements") + + if fill is None: + assert isinstance(elements, st.SearchStrategy) # for mypy + if unique or not elements.has_reusable_values: + fill = st.nothing() + else: + fill = elements + check_strategy(fill, "fill") + + return ArrayStrategy( + xp=xp, + api_version=api_version, + elements_strategy=elements, + dtype=dtype, + shape=shape, + fill=fill, + unique=unique, + ) + + +@check_function +def check_dtypes(xp: Any, dtypes: List[DataType], stubs: List[str]) -> None: + if len(dtypes) == 0: + assert len(stubs) > 0, "No dtypes passed but stubs is empty" + f_stubs = ", ".join(stubs) + raise InvalidArgument( + f"Array module {xp.__name__} does not have the following " + f"required dtypes in its namespace: {f_stubs}" + ) + elif len(stubs) > 0: + warn_on_missing_dtypes(xp, stubs) + + +def _scalar_dtypes(xp: Any, api_version: NominalVersion) -> st.SearchStrategy[DataType]: + """Return a strategy for all :xp-ref:`valid dtype ` objects.""" + return st.one_of(_boolean_dtypes(xp), _numeric_dtypes(xp, api_version)) + + +def _boolean_dtypes(xp: Any) -> st.SearchStrategy[DataType]: + """Return a strategy for just the boolean dtype object.""" + try: + return st.just(xp.bool) + except AttributeError: + raise InvalidArgument( + f"Array module {xp.__name__} does not have a bool dtype in its namespace" + ) from None + + +def _real_dtypes(xp: Any) -> st.SearchStrategy[DataType]: + """Return a strategy for all real-valued dtype objects.""" + return st.one_of( + _integer_dtypes(xp), + _unsigned_integer_dtypes(xp), + _floating_dtypes(xp), + ) + + +def _numeric_dtypes( + xp: Any, api_version: NominalVersion +) -> st.SearchStrategy[DataType]: + """Return a strategy for all numeric dtype objects.""" + strat: st.SearchStrategy[DataType] = _real_dtypes(xp) + if api_version > "2021.12": + strat |= _complex_dtypes(xp) + return strat + + +@check_function +def check_valid_sizes( + category: str, sizes: Sequence[int], valid_sizes: Sequence[int] +) -> None: + check_argument(len(sizes) > 0, "No sizes passed") + + invalid_sizes = [s for s in sizes if s not in valid_sizes] + f_valid_sizes = ", ".join(str(s) for s in valid_sizes) + f_invalid_sizes = ", ".join(str(s) for s in invalid_sizes) + check_argument( + len(invalid_sizes) == 0, + f"The following sizes are not valid for {category} dtypes: " + f"{f_invalid_sizes} (valid sizes: {f_valid_sizes})", + ) + + +def numeric_dtype_names(base_name: str, sizes: Sequence[int]) -> Iterator[str]: + for size in sizes: + yield f"{base_name}{size}" + + +def _integer_dtypes( + xp: Any, *, sizes: Union[int, Sequence[int]] = (8, 16, 32, 64) +) -> st.SearchStrategy[DataType]: + """Return a strategy for signed integer dtype objects. + + ``sizes`` contains the signed integer sizes in bits, defaulting to + ``(8, 16, 32, 64)`` which covers all valid sizes. + """ + if isinstance(sizes, int): + sizes = (sizes,) + check_valid_sizes("int", sizes, (8, 16, 32, 64)) + dtypes, stubs = partition_attributes_and_stubs( + xp, numeric_dtype_names("int", sizes) + ) + check_dtypes(xp, dtypes, stubs) + return st.sampled_from(dtypes) + + +def _unsigned_integer_dtypes( + xp: Any, *, sizes: Union[int, Sequence[int]] = (8, 16, 32, 64) +) -> st.SearchStrategy[DataType]: + """Return a strategy for unsigned integer dtype objects. + + ``sizes`` contains the unsigned integer sizes in bits, defaulting to + ``(8, 16, 32, 64)`` which covers all valid sizes. + """ + if isinstance(sizes, int): + sizes = (sizes,) + check_valid_sizes("int", sizes, (8, 16, 32, 64)) + + dtypes, stubs = partition_attributes_and_stubs( + xp, numeric_dtype_names("uint", sizes) + ) + check_dtypes(xp, dtypes, stubs) + + return st.sampled_from(dtypes) + + +def _floating_dtypes( + xp: Any, *, sizes: Union[int, Sequence[int]] = (32, 64) +) -> st.SearchStrategy[DataType]: + """Return a strategy for real-valued floating-point dtype objects. + + ``sizes`` contains the floating-point sizes in bits, defaulting to + ``(32, 64)`` which covers all valid sizes. + """ + if isinstance(sizes, int): + sizes = (sizes,) + check_valid_sizes("int", sizes, (32, 64)) + dtypes, stubs = partition_attributes_and_stubs( + xp, numeric_dtype_names("float", sizes) + ) + check_dtypes(xp, dtypes, stubs) + return st.sampled_from(dtypes) + + +def _complex_dtypes( + xp: Any, *, sizes: Union[int, Sequence[int]] = (64, 128) +) -> st.SearchStrategy[DataType]: + """Return a strategy for complex dtype objects. + + ``sizes`` contains the complex sizes in bits, defaulting to ``(64, 128)`` + which covers all valid sizes. + """ + if isinstance(sizes, int): + sizes = (sizes,) + check_valid_sizes("complex", sizes, (64, 128)) + dtypes, stubs = partition_attributes_and_stubs( + xp, numeric_dtype_names("complex", sizes) + ) + check_dtypes(xp, dtypes, stubs) + return st.sampled_from(dtypes) + + +@proxies(_valid_tuple_axes) +def valid_tuple_axes(*args, **kwargs): + return _valid_tuple_axes(*args, **kwargs) + + +valid_tuple_axes.__doc__ = f""" + Return a strategy for permissible tuple-values for the ``axis`` + argument in Array API sequential methods e.g. ``sum``, given the specified + dimensionality. + + {_valid_tuple_axes.__doc__} + """ + + +@defines_strategy() +def mutually_broadcastable_shapes( + num_shapes: int, + *, + base_shape: Shape = (), + min_dims: int = 0, + max_dims: Optional[int] = None, + min_side: int = 1, + max_side: Optional[int] = None, +) -> st.SearchStrategy[BroadcastableShapes]: + return _mutually_broadcastable_shapes( + num_shapes=num_shapes, + base_shape=base_shape, + min_dims=min_dims, + max_dims=max_dims, + min_side=min_side, + max_side=max_side, + ) + + +mutually_broadcastable_shapes.__doc__ = _mutually_broadcastable_shapes.__doc__ + + +@defines_strategy() +def indices( + shape: Shape, + *, + min_dims: int = 0, + max_dims: Optional[int] = None, + allow_newaxis: bool = False, + allow_ellipsis: bool = True, +) -> st.SearchStrategy[BasicIndex]: + """Return a strategy for :xp-ref:`valid indices ` of + arrays with the specified shape, which may include dimensions of size zero. + + It generates tuples containing some mix of integers, :obj:`python:slice` + objects, ``...`` (an ``Ellipsis``), and ``None``. When a length-one tuple + would be generated, this strategy may instead return the element which will + index the first axis, e.g. ``5`` instead of ``(5,)``. + + * ``shape`` is the shape of the array that will be indexed, as a tuple of + integers >= 0. This must be at least two-dimensional for a tuple to be a + valid index; for one-dimensional arrays use + :func:`~hypothesis.strategies.slices` instead. + * ``min_dims`` is the minimum dimensionality of the resulting array from use + of the generated index. + * ``max_dims`` is the the maximum dimensionality of the resulting array, + defaulting to ``len(shape) if not allow_newaxis else + max(len(shape), min_dims) + 2``. + * ``allow_ellipsis`` specifies whether ``None`` is allowed in the index. + * ``allow_ellipsis`` specifies whether ``...`` is allowed in the index. + """ + check_type(tuple, shape, "shape") + check_argument( + all(isinstance(x, int) and x >= 0 for x in shape), + f"shape={shape!r}, but all dimensions must be non-negative integers.", + ) + check_type(bool, allow_newaxis, "allow_newaxis") + check_type(bool, allow_ellipsis, "allow_ellipsis") + check_type(int, min_dims, "min_dims") + if not allow_newaxis: + check_argument( + min_dims <= len(shape), + f"min_dims={min_dims} is larger than len(shape)={len(shape)}, " + "but it is impossible for an indexing operation to add dimensions ", + "when allow_newaxis=False.", + ) + check_valid_dims(min_dims, "min_dims") + + if max_dims is None: + if allow_newaxis: + max_dims = min(max(len(shape), min_dims) + 2, NDIM_MAX) + else: + max_dims = min(len(shape), NDIM_MAX) + check_type(int, max_dims, "max_dims") + assert isinstance(max_dims, int) + if not allow_newaxis: + check_argument( + max_dims <= len(shape), + f"max_dims={max_dims} is larger than len(shape)={len(shape)}, " + "but it is impossible for an indexing operation to add dimensions ", + "when allow_newaxis=False.", + ) + check_valid_dims(max_dims, "max_dims") + + order_check("dims", 0, min_dims, max_dims) + + return BasicIndexStrategy( + shape, + min_dims=min_dims, + max_dims=max_dims, + allow_ellipsis=allow_ellipsis, + allow_newaxis=allow_newaxis, + allow_fewer_indices_than_dims=False, + ) + + +# Cache for make_strategies_namespace() +_args_to_xps: WeakValueDictionary = WeakValueDictionary() + + +def make_strategies_namespace( + xp: Any, *, api_version: Optional[NominalVersion] = None +) -> SimpleNamespace: + """Creates a strategies namespace for the given array module. + + * ``xp`` is the Array API library to automatically pass to the namespaced methods. + * ``api_version`` is the version of the Array API which the returned + strategies namespace should conform to. If ``None``, the latest API + version which ``xp`` supports will be inferred from ``xp.__array_api_version__``. + If a version string in the ``YYYY.MM`` format, the strategies namespace + will conform to that version if supported. + + A :obj:`python:types.SimpleNamespace` is returned which contains all the + strategy methods in this module but without requiring the ``xp`` argument. + Creating and using a strategies namespace for NumPy's Array API + implementation would go like this: + + .. code-block:: pycon + + >>> xp.__array_api_version__ # xp is your desired array library + '2021.12' + >>> xps = make_strategies_namespace(xp) + >>> xps.api_version + '2021.12' + >>> x = xps.arrays(xp.int8, (2, 3)).example() + >>> x + Array([[-8, 6, 3], + [-6, 4, 6]], dtype=int8) + >>> x.__array_namespace__() is xp + True + + """ + not_available_msg = ( + "If the standard version you want is not available, please ensure " + "you're using the latest version of Hypothesis, then open an issue if " + "one doesn't already exist." + ) + if api_version is None: + check_argument( + hasattr(xp, "__array_api_version__"), + f"Array module {xp.__name__} has no attribute __array_api_version__, " + "which is required when inferring api_version. If you believe " + f"{xp.__name__} is indeed an Array API module, try explicitly " + "passing an api_version.", + ) + check_argument( + isinstance(xp.__array_api_version__, str) + and xp.__array_api_version__ in RELEASED_VERSIONS, + f"xp.__array_api_version__={xp.__array_api_version__!r}, but it must " + f"be a valid version string {RELEASED_VERSIONS}. {not_available_msg}", + ) + api_version = xp.__array_api_version__ + inferred_version = True + else: + check_argument( + isinstance(api_version, str) and api_version in NOMINAL_VERSIONS, + f"api_version={api_version!r}, but it must be None, or a valid version " + f"string in {RELEASED_VERSIONS}. {not_available_msg}", + ) + inferred_version = False + try: + array = xp.zeros(1) + array.__array_namespace__() + except Exception: + warn( + f"Could not determine whether module {xp.__name__} is an Array API library", + HypothesisWarning, + ) + + try: + namespace = _args_to_xps[(xp, api_version)] + except (KeyError, TypeError): + pass + else: + return namespace + + @defines_strategy(force_reusable_values=True) + def from_dtype( + dtype: Union[DataType, str], + *, + min_value: Optional[Union[int, float]] = None, + max_value: Optional[Union[int, float]] = None, + allow_nan: Optional[bool] = None, + allow_infinity: Optional[bool] = None, + allow_subnormal: Optional[bool] = None, + exclude_min: Optional[bool] = None, + exclude_max: Optional[bool] = None, + ) -> st.SearchStrategy[Union[bool, int, float]]: + return _from_dtype( + xp, + api_version, # type: ignore[arg-type] + dtype, + min_value=min_value, + max_value=max_value, + allow_nan=allow_nan, + allow_infinity=allow_infinity, + allow_subnormal=allow_subnormal, + exclude_min=exclude_min, + exclude_max=exclude_max, + ) + + @defines_strategy(force_reusable_values=True) + def arrays( + dtype: Union[ + DataType, str, st.SearchStrategy[DataType], st.SearchStrategy[str] + ], + shape: Union[int, Shape, st.SearchStrategy[Shape]], + *, + elements: Optional[Union[Mapping[str, Any], st.SearchStrategy]] = None, + fill: Optional[st.SearchStrategy[Any]] = None, + unique: bool = False, + ) -> st.SearchStrategy: + return _arrays( + xp, + api_version, # type: ignore[arg-type] + dtype, + shape, + elements=elements, + fill=fill, + unique=unique, + ) + + @defines_strategy() + def scalar_dtypes() -> st.SearchStrategy[DataType]: + return _scalar_dtypes(xp, api_version) # type: ignore[arg-type] + + @defines_strategy() + def boolean_dtypes() -> st.SearchStrategy[DataType]: + return _boolean_dtypes(xp) + + @defines_strategy() + def real_dtypes() -> st.SearchStrategy[DataType]: + return _real_dtypes(xp) + + @defines_strategy() + def numeric_dtypes() -> st.SearchStrategy[DataType]: + return _numeric_dtypes(xp, api_version) # type: ignore[arg-type] + + @defines_strategy() + def integer_dtypes( + *, sizes: Union[int, Sequence[int]] = (8, 16, 32, 64) + ) -> st.SearchStrategy[DataType]: + return _integer_dtypes(xp, sizes=sizes) + + @defines_strategy() + def unsigned_integer_dtypes( + *, sizes: Union[int, Sequence[int]] = (8, 16, 32, 64) + ) -> st.SearchStrategy[DataType]: + return _unsigned_integer_dtypes(xp, sizes=sizes) + + @defines_strategy() + def floating_dtypes( + *, sizes: Union[int, Sequence[int]] = (32, 64) + ) -> st.SearchStrategy[DataType]: + return _floating_dtypes(xp, sizes=sizes) + + from_dtype.__doc__ = _from_dtype.__doc__ + arrays.__doc__ = _arrays.__doc__ + scalar_dtypes.__doc__ = _scalar_dtypes.__doc__ + boolean_dtypes.__doc__ = _boolean_dtypes.__doc__ + real_dtypes.__doc__ = _real_dtypes.__doc__ + numeric_dtypes.__doc__ = _numeric_dtypes.__doc__ + integer_dtypes.__doc__ = _integer_dtypes.__doc__ + unsigned_integer_dtypes.__doc__ = _unsigned_integer_dtypes.__doc__ + floating_dtypes.__doc__ = _floating_dtypes.__doc__ + + class StrategiesNamespace(SimpleNamespace): + def __init__(self, **kwargs): + for attr in ["name", "api_version"]: + if attr not in kwargs.keys(): + raise ValueError(f"'{attr}' kwarg required") + super().__init__(**kwargs) + + @property + def complex_dtypes(self): + try: + return self.__dict__["complex_dtypes"] + except KeyError as e: + raise AttributeError( + "You attempted to access 'complex_dtypes', but it is not " + f"available for api_version='{self.api_version}' of " + f"xp={self.name}." + ) from e + + def __repr__(self): + f_args = self.name + if not inferred_version: + f_args += f", api_version='{self.api_version}'" + return f"make_strategies_namespace({f_args})" + + kwargs = dict( + name=xp.__name__, + api_version=api_version, + from_dtype=from_dtype, + arrays=arrays, + array_shapes=array_shapes, + scalar_dtypes=scalar_dtypes, + boolean_dtypes=boolean_dtypes, + real_dtypes=real_dtypes, + numeric_dtypes=numeric_dtypes, + integer_dtypes=integer_dtypes, + unsigned_integer_dtypes=unsigned_integer_dtypes, + floating_dtypes=floating_dtypes, + valid_tuple_axes=valid_tuple_axes, + broadcastable_shapes=broadcastable_shapes, + mutually_broadcastable_shapes=mutually_broadcastable_shapes, + indices=indices, + ) + + if api_version > "2021.12": + + @defines_strategy() + def complex_dtypes( + *, sizes: Union[int, Sequence[int]] = (64, 128) + ) -> st.SearchStrategy[DataType]: + return _complex_dtypes(xp, sizes=sizes) + + complex_dtypes.__doc__ = _complex_dtypes.__doc__ + kwargs["complex_dtypes"] = complex_dtypes + + namespace = StrategiesNamespace(**kwargs) + try: + _args_to_xps[(xp, api_version)] = namespace + except TypeError: + pass + + return namespace + + +try: + import numpy as np +except ImportError: + if "sphinx" in sys.modules: + # This is pretty awkward, but also the best way available + from unittest.mock import Mock + + np = Mock() + else: + np = None +if np is not None: + + class FloatInfo(NamedTuple): + bits: int + eps: float + max: float + min: float + smallest_normal: float + + def mock_finfo(dtype: DataType) -> FloatInfo: + """Returns a finfo object compliant with the Array API + + Ensures all attributes are Python scalars and not NumPy scalars. This + lets us ignore corner cases with how NumPy scalars operate, such as + NumPy floats breaking our next_down() util. + + Also ensures the finfo obj has the smallest_normal attribute. NumPy only + introduced it in v1.21.1, so we just use the equivalent tiny attribute + to keep mocking with older versions working. + """ + _finfo = np.finfo(dtype) + return FloatInfo( + int(_finfo.bits), + float(_finfo.eps), + float(_finfo.max), + float(_finfo.min), + float(_finfo.tiny), + ) + + mock_xp = SimpleNamespace( + __name__="mock", + __array_api_version__="2021.12", + # Data types + int8=np.int8, + int16=np.int16, + int32=np.int32, + int64=np.int64, + uint8=np.uint8, + uint16=np.uint16, + uint32=np.uint32, + uint64=np.uint64, + float32=np.float32, + float64=np.float64, + complex64=np.complex64, + complex128=np.complex128, + bool=np.bool_, + # Constants + nan=np.nan, + # Data type functions + astype=lambda x, d: x.astype(d), + iinfo=np.iinfo, + finfo=mock_finfo, + broadcast_arrays=np.broadcast_arrays, + # Creation functions + arange=np.arange, + asarray=np.asarray, + empty=np.empty, + full=np.full, + zeros=np.zeros, + ones=np.ones, + linspace=np.linspace, + # Manipulation functions + reshape=np.reshape, + # Element-wise functions + isnan=np.isnan, + isfinite=np.isfinite, + logical_or=np.logical_or, + # Statistical functions + sum=np.sum, + # Searching functions + nonzero=np.nonzero, + # Sorting functions + sort=np.sort, + # Set functions + unique_values=np.unique, + # Utility functions + any=np.any, + all=np.all, + ) diff --git a/hypothesis-python/src/hypothesis/extra/cli.py b/hypothesis-python/src/hypothesis/extra/cli.py index f9cbd91564..ac530bce1e 100644 --- a/hypothesis-python/src/hypothesis/extra/cli.py +++ b/hypothesis-python/src/hypothesis/extra/cli.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """ .. _hypothesis-cli: @@ -42,11 +37,18 @@ import builtins import importlib +import inspect import sys +import types from difflib import get_close_matches from functools import partial from multiprocessing import Pool +try: + import pytest +except ImportError: + pytest = None # type: ignore + MESSAGE = """ The Hypothesis command-line interface requires the `{}` package, which you do not have installed. Run: @@ -65,7 +67,6 @@ def main(): sys.stderr.write(MESSAGE.format("click")) sys.exit(1) - else: # Ensure that Python scripts in the current working directory are importable, # on the principle that Ghostwriter should 'just work' for novice users. Note @@ -81,31 +82,82 @@ def main(): def obj_name(s: str) -> object: """This "type" imports whatever object is named by a dotted string.""" s = s.strip() + if "/" in s or "\\" in s: + raise click.UsageError( + "Remember that the ghostwriter should be passed the name of a module, not a path." + ) from None try: return importlib.import_module(s) except ImportError: pass + classname = None if "." not in s: modulename, module, funcname = "builtins", builtins, s else: modulename, funcname = s.rsplit(".", 1) try: module = importlib.import_module(modulename) - except ImportError: + except ImportError as err: + try: + modulename, classname = modulename.rsplit(".", 1) + module = importlib.import_module(modulename) + except (ImportError, ValueError): + if s.endswith(".py"): + raise click.UsageError( + "Remember that the ghostwriter should be passed the name of a module, not a file." + ) from None + raise click.UsageError( + f"Failed to import the {modulename} module for introspection. " + "Check spelling and your Python import path, or use the Python API?" + ) from err + + def describe_close_matches( + module_or_class: types.ModuleType, objname: str + ) -> str: + public_names = [ + name for name in vars(module_or_class) if not name.startswith("_") + ] + matches = get_close_matches(objname, public_names) + if matches: + return f" Closest matches: {matches!r}" + else: + return "" + + if classname is None: + try: + return getattr(module, funcname) + except AttributeError as err: + if funcname == "py": + # Likely attempted to pass a local file (Eg., "myscript.py") instead of a module name + raise click.UsageError( + "Remember that the ghostwriter should be passed the name of a module, not a file." + + f"\n\tTry: hypothesis write {s[:-3]}" + ) from None raise click.UsageError( - f"Failed to import the {modulename} module for introspection. " - "Check spelling and your Python import path, or use the Python API?" - ) - try: - return getattr(module, funcname) - except AttributeError: - public_names = [name for name in vars(module) if not name.startswith("_")] - matches = get_close_matches(funcname, public_names) - raise click.UsageError( - f"Found the {modulename!r} module, but it doesn't have a " - f"{funcname!r} attribute." - + (f" Closest matches: {matches!r}" if matches else "") - ) + f"Found the {modulename!r} module, but it doesn't have a " + f"{funcname!r} attribute." + + describe_close_matches(module, funcname) + ) from err + else: + try: + func_class = getattr(module, classname) + except AttributeError as err: + raise click.UsageError( + f"Found the {modulename!r} module, but it doesn't have a " + f"{classname!r} class." + describe_close_matches(module, classname) + ) from err + try: + return getattr(func_class, funcname) + except AttributeError as err: + if inspect.isclass(func_class): + func_class_is = "class" + else: + func_class_is = "attribute" + raise click.UsageError( + f"Found the {modulename!r} module and {classname!r} {func_class_is}, " + f"but it doesn't have a {funcname!r} attribute." + + describe_close_matches(func_class, funcname) + ) from err def _refactor(func, fname): try: @@ -183,14 +235,30 @@ def codemod(path): flag_value="equivalent", help="very useful when optimising or refactoring code", ) - @click.option("--idempotent", "writer", flag_value="idempotent") - @click.option("--binary-op", "writer", flag_value="binary_operation") + @click.option( + "--errors-equivalent", + "writer", + flag_value="errors-equivalent", + help="--equivalent, but also allows consistent errors", + ) + @click.option( + "--idempotent", + "writer", + flag_value="idempotent", + help="check that f(x) == f(f(x))", + ) + @click.option( + "--binary-op", + "writer", + flag_value="binary_operation", + help="associativity, commutativity, identity element", + ) # Note: we deliberately omit a --ufunc flag, because the magic() # detection of ufuncs is both precise and complete. @click.option( "--style", type=click.Choice(["pytest", "unittest"]), - default="pytest", + default="pytest" if pytest else "unittest", help="pytest-style function, or unittest-style method?", ) @click.option( @@ -219,14 +287,18 @@ def write(func, writer, except_, style): # noqa: D301 # \b disables autowrap # NOTE: if you want to call this function from Python, look instead at the # ``hypothesis.extra.ghostwriter`` module. Click-decorated functions have # a different calling convention, and raise SystemExit instead of returning. + kwargs = {"except_": except_ or (), "style": style} if writer is None: writer = "magic" elif writer == "idempotent" and len(func) > 1: raise click.UsageError("Test functions for idempotence one at a time.") elif writer == "roundtrip" and len(func) == 1: writer = "idempotent" - elif writer == "equivalent" and len(func) == 1: + elif "equivalent" in writer and len(func) == 1: writer = "fuzz" + if writer == "errors-equivalent": + writer = "equivalent" + kwargs["allow_same_errors"] = True try: from hypothesis.extra import ghostwriter @@ -234,4 +306,19 @@ def write(func, writer, except_, style): # noqa: D301 # \b disables autowrap sys.stderr.write(MESSAGE.format("black")) sys.exit(1) - print(getattr(ghostwriter, writer)(*func, except_=except_ or (), style=style)) + code = getattr(ghostwriter, writer)(*func, **kwargs) + try: + from rich.console import Console + from rich.syntax import Syntax + + from hypothesis.utils.terminal import guess_background_color + except ImportError: + print(code) + else: + try: + theme = "default" if guess_background_color() == "light" else "monokai" + code = Syntax(code, "python", background_color="default", theme=theme) + Console().print(code, soft_wrap=True) + except Exception: + print("# Error while syntax-highlighting code", file=sys.stderr) + print(code) diff --git a/hypothesis-python/src/hypothesis/extra/codemods.py b/hypothesis-python/src/hypothesis/extra/codemods.py index d7085d424d..6bced42c8d 100644 --- a/hypothesis-python/src/hypothesis/extra/codemods.py +++ b/hypothesis-python/src/hypothesis/extra/codemods.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """ .. _codemods: @@ -50,6 +45,7 @@ .. autofunction:: refactor """ +import functools import importlib from inspect import Parameter, signature from typing import List @@ -113,6 +109,12 @@ def leave_Arg(self, original_node, updated_node): return updated_node +@functools.lru_cache() +def get_fn(import_path): + mod, fn = import_path.rsplit(".", 1) + return getattr(importlib.import_module(mod), fn) + + class HypothesisFixPositionalKeywonlyArgs(VisitorBasedCodemodCommand): """Fix positional arguments for newly keyword-only parameters, e.g.:: @@ -195,10 +197,17 @@ def leave_Call(self, original_node, updated_node): # Get the actual function object so that we can inspect the signature. # This does e.g. incur a dependency on Numpy to fix Numpy-dependent code, # but having a single source of truth about the signatures is worth it. - mod, fn = list(qualnames.intersection(self.kwonly_functions))[0].rsplit(".", 1) try: - func = getattr(importlib.import_module(mod), fn) - except ImportError: + params = signature(get_fn(*qualnames)).parameters.values() + except ModuleNotFoundError: + return updated_node + + # st.floats() has a new allow_subnormal kwonly argument not at the end, + # so we do a bit more of a dance here. + if qualnames == {"hypothesis.strategies.floats"}: + params = [p for p in params if p.name != "allow_subnormal"] + + if len(updated_node.args) > len(params): return updated_node # Create new arg nodes with the newly required keywords @@ -210,6 +219,6 @@ def leave_Call(self, original_node, updated_node): arg if arg.keyword or arg.star or p.kind is not Parameter.KEYWORD_ONLY else arg.with_changes(keyword=cst.Name(p.name), equal=assign_nospace) - for p, arg in zip(signature(func).parameters.values(), updated_node.args) + for p, arg in zip(params, updated_node.args) ] return updated_node.with_changes(args=newargs) diff --git a/hypothesis-python/src/hypothesis/extra/dateutil.py b/hypothesis-python/src/hypothesis/extra/dateutil.py index 4b5b0cee1a..810d0477a2 100644 --- a/hypothesis-python/src/hypothesis/extra/dateutil.py +++ b/hypothesis-python/src/hypothesis/extra/dateutil.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """ -------------------- @@ -28,7 +23,8 @@ from dateutil import tz, zoneinfo # type: ignore -from hypothesis.strategies._internal import core as st +from hypothesis import strategies as st +from hypothesis.strategies._internal.utils import cacheable, defines_strategy __all__ = ["timezones"] @@ -43,8 +39,8 @@ def __zone_sort_key(zone): return (abs(offset), -offset, str(zone)) -@st.cacheable -@st.defines_strategy() +@cacheable +@defines_strategy() def timezones() -> st.SearchStrategy[dt.tzinfo]: """Any timezone from :pypi:`dateutil `. diff --git a/hypothesis-python/src/hypothesis/extra/django/__init__.py b/hypothesis-python/src/hypothesis/extra/django/__init__.py index e821309891..39e6e0182e 100644 --- a/hypothesis-python/src/hypothesis/extra/django/__init__.py +++ b/hypothesis-python/src/hypothesis/extra/django/__init__.py @@ -1,20 +1,17 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis.extra.django._fields import from_field, register_field_strategy from hypothesis.extra.django._impl import ( + LiveServerTestCase, + StaticLiveServerTestCase, TestCase, TransactionTestCase, from_form, @@ -22,6 +19,8 @@ ) __all__ = [ + "LiveServerTestCase", + "StaticLiveServerTestCase", "TestCase", "TransactionTestCase", "from_field", diff --git a/hypothesis-python/src/hypothesis/extra/django/_fields.py b/hypothesis-python/src/hypothesis/extra/django/_fields.py index 9dbf57cc9f..2118cd406a 100644 --- a/hypothesis-python/src/hypothesis/extra/django/_fields.py +++ b/hypothesis-python/src/hypothesis/extra/django/_fields.py @@ -1,26 +1,23 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import re import string from datetime import timedelta from decimal import Decimal +from functools import lru_cache from typing import Any, Callable, Dict, Type, TypeVar, Union import django from django import forms as df +from django.contrib.auth.forms import UsernameField from django.core.validators import ( validate_ipv4_address, validate_ipv6_address, @@ -30,7 +27,6 @@ from hypothesis import strategies as st from hypothesis.errors import InvalidArgument, ResolutionFailed -from hypothesis.extra.pytz import timezones from hypothesis.internal.validation import check_type from hypothesis.provisional import urls from hypothesis.strategies import emails @@ -57,8 +53,24 @@ def inner(field): return inner +@lru_cache() +def timezones(): + # From Django 4.0, the default is to use zoneinfo instead of pytz. + assert getattr(django.conf.settings, "USE_TZ", False) + if getattr(django.conf.settings, "USE_DEPRECATED_PYTZ", True): + from hypothesis.extra.pytz import timezones + else: + from hypothesis.strategies import timezones + + return timezones() + + # Mapping of field types, to strategy objects or functions of (type) -> strategy -_global_field_lookup = { +_FieldLookUpType = Dict[ + Type[AnyField], + Union[st.SearchStrategy, Callable[[Any], st.SearchStrategy]], +] +_global_field_lookup: _FieldLookUpType = { dm.SmallIntegerField: integers_for_field(-32768, 32767), dm.IntegerField: integers_for_field(-2147483648, 2147483647), dm.BigIntegerField: integers_for_field(-9223372036854775808, 9223372036854775807), @@ -81,7 +93,7 @@ def inner(field): df.NullBooleanField: st.one_of(st.none(), st.booleans()), df.URLField: urls(), df.UUIDField: st.uuids(), -} # type: Dict[Type[AnyField], Union[st.SearchStrategy, Callable[[Any], st.SearchStrategy]]] +} _ipv6_strings = st.one_of( st.ip_addresses(v=6).map(str), @@ -136,7 +148,7 @@ def _for_form_time(field): def _for_duration(field): # SQLite stores timedeltas as six bytes of microseconds if using_sqlite(): - delta = timedelta(microseconds=2 ** 47 - 1) + delta = timedelta(microseconds=2**47 - 1) return st.timedeltas(-delta, delta) return st.timedeltas() @@ -181,7 +193,7 @@ def _for_form_ip(field): @register_for(df.DecimalField) def _for_decimal(field): min_value, max_value = numeric_bounds_from_validators(field) - bound = Decimal(10 ** field.max_digits - 1) / (10 ** field.decimal_places) + bound = Decimal(10**field.max_digits - 1) / (10**field.decimal_places) return st.decimals( min_value=max(min_value, -bound), max_value=min(max_value, bound), @@ -212,6 +224,7 @@ def _for_binary(field): @register_for(dm.TextField) @register_for(df.CharField) @register_for(df.RegexField) +@register_for(UsernameField) def _for_text(field): # We can infer a vastly more precise strategy by considering the # validators as well as the field type. This is a minimal proof of @@ -229,7 +242,7 @@ def _for_text(field): # Not maximally efficient, but it makes pathological cases rarer. # If you want a challenge: extend https://qntm.org/greenery to # compute intersections of the full Python regex language. - return st.one_of(*[st.from_regex(r) for r in regexes]) + return st.one_of(*(st.from_regex(r) for r in regexes)) # If there are no (usable) regexes, we use a standard text strategy. min_size, max_size = length_bounds_from_validators(field) strategy = st.text( @@ -279,7 +292,8 @@ def from_field(field: F) -> st.SearchStrategy[Union[F, None]]: This function is used by :func:`~hypothesis.extra.django.from_form` and :func:`~hypothesis.extra.django.from_model` for any fields that require - a value, or for which you passed :obj:`hypothesis.infer`. + a value, or for which you passed ``...`` (:obj:`python:Ellipsis`) to infer + a strategy from an annotation. It's pretty similar to the core :func:`~hypothesis.strategies.from_type` function, with a subtle but important difference: ``from_field`` takes a @@ -288,7 +302,7 @@ def from_field(field: F) -> st.SearchStrategy[Union[F, None]]: """ check_type((dm.Field, df.Field), field, "field") if getattr(field, "choices", False): - choices = [] # type: list + choices: list = [] for value, name_or_optgroup in field.choices: if isinstance(name_or_optgroup, (list, tuple)): choices.extend(key for key, _ in name_or_optgroup) diff --git a/hypothesis-python/src/hypothesis/extra/django/_impl.py b/hypothesis-python/src/hypothesis/extra/django/_impl.py index fc60eb4963..397898cdd7 100644 --- a/hypothesis-python/src/hypothesis/extra/django/_impl.py +++ b/hypothesis-python/src/hypothesis/extra/django/_impl.py @@ -1,33 +1,36 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import sys import unittest from functools import partial from inspect import Parameter, signature -from typing import Optional, Type, Union +from typing import TYPE_CHECKING, Optional, Type, Union from django import forms as df, test as dt +from django.contrib.staticfiles import testing as dst from django.core.exceptions import ValidationError from django.db import IntegrityError, models as dm -from hypothesis import reject +from hypothesis import reject, strategies as st from hypothesis.errors import InvalidArgument from hypothesis.extra.django._fields import from_field -from hypothesis.strategies._internal import core as st -from hypothesis.utils.conventions import InferType, infer +from hypothesis.internal.reflection import define_function_signature +from hypothesis.strategies._internal.utils import defines_strategy + +if sys.version_info >= (3, 10): # pragma: no cover + from types import EllipsisType as EllipsisType +elif TYPE_CHECKING: + from builtins import ellipsis as EllipsisType +else: + EllipsisType = type(Ellipsis) class HypothesisTestCase: @@ -53,9 +56,17 @@ class TransactionTestCase(HypothesisTestCase, dt.TransactionTestCase): pass -@st.defines_strategy() +class LiveServerTestCase(HypothesisTestCase, dt.LiveServerTestCase): + pass + + +class StaticLiveServerTestCase(HypothesisTestCase, dst.StaticLiveServerTestCase): + pass + + +@defines_strategy() def from_model( - *model: Type[dm.Model], **field_strategies: Union[st.SearchStrategy, InferType] + *model: Type[dm.Model], **field_strategies: Union[st.SearchStrategy, EllipsisType] ) -> st.SearchStrategy: """Return a strategy for examples of ``model``. @@ -79,7 +90,7 @@ def from_model( shop_strategy = from_model(Shop, company=from_model(Company)) Like for :func:`~hypothesis.strategies.builds`, you can pass - :obj:`~hypothesis.infer` as a keyword argument to infer a strategy for + ``...`` (:obj:`python:Ellipsis`) as a keyword argument to infer a strategy for a field which has a default value instead of using the default. """ if len(model) == 1: @@ -94,7 +105,7 @@ def from_model( fields_by_name = {f.name: f for f in m_type._meta.concrete_fields} for name, value in sorted(field_strategies.items()): - if value is infer: + if value is ...: field_strategies[name] = from_field(fields_by_name[name]) for name, field in sorted(fields_by_name.items()): if ( @@ -119,13 +130,17 @@ def from_model( return _models_impl(st.builds(m_type.objects.get_or_create, **field_strategies)) -if sys.version_info[:2] >= (3, 8): # pragma: no cover +if sys.version_info[:2] >= (3, 8): # pragma: no branch # See notes above definition of st.builds() - this signature is compatible # and better matches the semantics of the function. Great for documentation! sig = signature(from_model) params = list(sig.parameters.values()) params[0] = params[0].replace(kind=Parameter.POSITIONAL_ONLY) - from_model.__signature__ = sig.replace(parameters=params) + from_model = define_function_signature( + name=from_model.__name__, + docstring=from_model.__doc__, + signature=sig.replace(parameters=params), + )(from_model) @st.composite @@ -137,11 +152,11 @@ def _models_impl(draw, strat): reject() -@st.defines_strategy() +@defines_strategy() def from_form( form: Type[df.Form], form_kwargs: Optional[dict] = None, - **field_strategies: Union[st.SearchStrategy, InferType], + **field_strategies: Union[st.SearchStrategy, EllipsisType], ) -> st.SearchStrategy[df.Form]: """Return a strategy for examples of ``form``. @@ -162,7 +177,7 @@ def from_form( shop_strategy = from_form(Shop, form_kwargs={"company_id": 5}) Like for :func:`~hypothesis.strategies.builds`, you can pass - :obj:`~hypothesis.infer` as a keyword argument to infer a strategy for + ``...`` (:obj:`python:Ellipsis`) as a keyword argument to infer a strategy for a field which has a default value instead of using the default. """ # currently unsupported: @@ -198,7 +213,7 @@ def from_form( else: fields_by_name[name] = field for name, value in sorted(field_strategies.items()): - if value is infer: + if value is ...: field_strategies[name] = from_field(fields_by_name[name]) for name, field in sorted(fields_by_name.items()): diff --git a/hypothesis-python/src/hypothesis/extra/dpcontracts.py b/hypothesis-python/src/hypothesis/extra/dpcontracts.py index d8593f47c9..bf2a267240 100644 --- a/hypothesis-python/src/hypothesis/extra/dpcontracts.py +++ b/hypothesis-python/src/hypothesis/extra/dpcontracts.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """ ----------------------- diff --git a/hypothesis-python/src/hypothesis/extra/ghostwriter.py b/hypothesis-python/src/hypothesis/extra/ghostwriter.py index d56498f64c..6b24549a44 100644 --- a/hypothesis-python/src/hypothesis/extra/ghostwriter.py +++ b/hypothesis-python/src/hypothesis/extra/ghostwriter.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """ Writing tests with Hypothesis frees you from the tedium of deciding on and @@ -47,12 +42,18 @@ Options: --roundtrip start by testing write/read or encode/decode! --equivalent very useful when optimising or refactoring code - --idempotent - --binary-op + --errors-equivalent --equivalent, but also allows consistent errors + --idempotent check that f(x) == f(f(x)) + --binary-op associativity, commutativity, identity element --style [pytest|unittest] pytest-style function, or unittest-style method? -e, --except OBJ_NAME dotted name of exception(s) to ignore -h, --help Show this message and exit. +.. tip:: + + Using a light theme? Hypothesis respects `NO_COLOR `__ + and :envvar:`DJANGO_COLORS=light `. + .. note:: The ghostwriter requires :pypi:`black`, but the generated code only @@ -67,21 +68,26 @@ public domain dedication, so you can use it without any restrictions. """ +import ast import builtins import contextlib import enum import inspect +import os import re import sys import types from collections import OrderedDict, defaultdict from itertools import permutations, zip_longest +from keyword import iskeyword from string import ascii_lowercase from textwrap import dedent, indent from typing import ( + TYPE_CHECKING, Any, Callable, Dict, + List, Mapping, Optional, Set, @@ -93,11 +99,13 @@ import black -from hypothesis import find, strategies as st +from hypothesis import Verbosity, find, settings, strategies as st from hypothesis.errors import InvalidArgument from hypothesis.internal.compat import get_type_hints -from hypothesis.internal.reflection import is_mock +from hypothesis.internal.reflection import get_signature, is_mock from hypothesis.internal.validation import check_type +from hypothesis.provisional import domains +from hypothesis.strategies._internal.collections import ListStrategy from hypothesis.strategies._internal.core import BuildsStrategy from hypothesis.strategies._internal.flatmapped import FlatMapStrategy from hypothesis.strategies._internal.lazy import LazyStrategy, unwrap_strategies @@ -105,9 +113,17 @@ FilteredStrategy, MappedSearchStrategy, OneOfStrategy, + SampledFromStrategy, ) -from hypothesis.strategies._internal.types import _global_type_lookup -from hypothesis.utils.conventions import InferType, infer +from hypothesis.strategies._internal.types import _global_type_lookup, is_generic_type + +if sys.version_info >= (3, 10): # pragma: no cover + from types import EllipsisType as EllipsisType +elif TYPE_CHECKING: + from builtins import ellipsis as EllipsisType +else: + EllipsisType = type(Ellipsis) + IMPORT_SECTION = """ # This test code was written by the `hypothesis.extra.ghostwriter` module @@ -122,14 +138,33 @@ def test_{test_kind}_{func_name}({arg_names}): {test_body} """ -SUPPRESS_BLOCK = """\ +SUPPRESS_BLOCK = """ try: {test_body} except {exceptions}: reject() -""" +""".strip() Except = Union[Type[Exception], Tuple[Type[Exception], ...]] +ImportSet = Set[Union[str, Tuple[str, str]]] +RE_TYPES = (type(re.compile(".")), type(re.match(".", "abc"))) +_quietly_settings = settings( + database=None, + deadline=None, + derandomize=True, + verbosity=Verbosity.quiet, +) + + +def _dedupe_exceptions(exc: Tuple[Type[Exception], ...]) -> Tuple[Type[Exception], ...]: + # This is reminiscent of de-duplication logic I wrote for flake8-bugbear, + # but with access to the actual objects we can just check for subclasses. + # This lets us print e.g. `Exception` instead of `(Exception, OSError)`. + uniques = list(exc) + for a, b in permutations(exc, 2): + if a in uniques and issubclass(a, b): + uniques.remove(a) + return tuple(sorted(uniques, key=lambda e: e.__name__)) def _check_except(except_: Except) -> Tuple[Type[Exception], ...]: @@ -149,28 +184,124 @@ def _check_except(except_: Except) -> Tuple[Type[Exception], ...]: return (except_,) +def _exception_string(except_: Tuple[Type[Exception], ...]) -> Tuple[ImportSet, str]: + if not except_: + return set(), "" + exceptions = [] + imports: ImportSet = set() + for ex in _dedupe_exceptions(except_): + if ex.__qualname__ in dir(builtins): + exceptions.append(ex.__qualname__) + else: + imports.add(ex.__module__) + exceptions.append(_get_qualname(ex, include_module=True)) + return imports, ( + "(" + ", ".join(exceptions) + ")" if len(exceptions) > 1 else exceptions[0] + ) + + def _check_style(style: str) -> None: if style not in ("pytest", "unittest"): raise InvalidArgument(f"Valid styles are 'pytest' or 'unittest', got {style!r}") -# Simple strategies to guess for common argument names - we wouldn't do this in -# builds() where strict correctness is required, but we only use these guesses -# when the alternative is nothing() to force user edits anyway. -# -# This table was constructed manually after skimming through the documentation -# for the builtins and a few stdlib modules. Future enhancements could be based -# on analysis of type-annotated code to detect arguments which almost always -# take values of a particular type. -_GUESS_STRATEGIES_BY_NAME = ( - (st.text(), ["name", "filename", "fname"]), - (st.floats(), ["real", "imag"]), - (st.functions(), ["function", "func", "f"]), - (st.iterables(st.integers()) | st.iterables(st.text()), ["iterable"]), -) +def _exceptions_from_docstring(doc: str) -> Tuple[Type[Exception], ...]: + """Return a tuple of exceptions that the docstring says may be raised. + Note that we ignore non-builtin exception types for simplicity, as this is + used directly in _write_call() and passing import sets around would be really + really annoying. + """ + # TODO: it would be great to handle Google- and Numpy-style docstrings + # (e.g. by using the Napoleon Sphinx extension) + assert isinstance(doc, str), doc + raises = [] + for excname in re.compile(r"\:raises\s+(\w+)\:", re.MULTILINE).findall(doc): + exc_type = getattr(builtins, excname, None) + if isinstance(exc_type, type) and issubclass(exc_type, Exception): + raises.append(exc_type) + return tuple(_dedupe_exceptions(tuple(raises))) + + +def _type_from_doc_fragment(token: str) -> Optional[type]: + # Special cases for "integer" and for numpy array-like and dtype + if token == "integer": + return int + if "numpy" in sys.modules: + if re.fullmatch(r"[Aa]rray[-_ ]?like", token): + return sys.modules["numpy"].ndarray + elif token == "dtype": + return sys.modules["numpy"].dtype + # Natural-language syntax, e.g. "sequence of integers" + coll_match = re.fullmatch(r"(\w+) of (\w+)", token) + if coll_match is not None: + coll_token, elem_token = coll_match.groups() + elems = _type_from_doc_fragment(elem_token) + if elems is None and elem_token.endswith("s"): + elems = _type_from_doc_fragment(elem_token[:-1]) + if elems is not None and coll_token in ("list", "sequence", "collection"): + return List[elems] # type: ignore + # This might be e.g. "array-like of float"; arrays is better than nothing + # even if we can't conveniently pass a generic type around. + return _type_from_doc_fragment(coll_token) + # Check either builtins, or the module for a dotted name + if "." not in token: + return getattr(builtins, token, None) + mod, name = token.rsplit(".", maxsplit=1) + return getattr(sys.modules.get(mod, None), name, None) + + +def _strategy_for(param: inspect.Parameter, docstring: str) -> st.SearchStrategy: + # Example types in docstrings: + # - `:type a: sequence of integers` + # - `b (list, tuple, or None): ...` + # - `c : {"foo", "bar", or None}` + for pattern in ( + rf"^\s*\:type\s+{param.name}\:\s+(.+)", # RST-style + rf"^\s*{param.name} \((.+)\):", # Google-style + rf"^\s*{param.name} \: (.+)", # Numpy-style + ): + match = re.search(pattern, docstring, flags=re.MULTILINE) + if match is None: + continue + doc_type = match.group(1) + if doc_type.endswith(", optional"): + # Convention to describe "argument may be omitted" + doc_type = doc_type[: -len(", optional")] + doc_type = doc_type.strip("}{") + elements = [] + types = [] + for token in re.split(r",? +or +| *, *", doc_type): + for prefix in ("default ", "python "): + # `str or None, default "auto"`; `python int or numpy.int64` + if token.startswith(prefix): + token = token[len(prefix) :] + if not token: + continue + try: + # Elements of `{"inner", "outer"}` etc. + elements.append(ast.literal_eval(token)) + continue + except (ValueError, SyntaxError): + t = _type_from_doc_fragment(token) + if isinstance(t, type) or is_generic_type(t): + assert t is not None + types.append(t) + if ( + param.default is not inspect.Parameter.empty + and param.default not in elements + and not isinstance( + param.default, tuple(t for t in types if isinstance(t, type)) + ) + ): + with contextlib.suppress(SyntaxError): + compile(repr(st.just(param.default)), "", "eval") + elements.insert(0, param.default) + if elements or types: + return (st.sampled_from(elements) if elements else st.nothing()) | ( + st.one_of(*map(st.from_type, types)) if types else st.nothing() + ) -def _strategy_for(param: inspect.Parameter) -> Union[st.SearchStrategy, InferType]: # If our default value is an Enum or a boolean, we assume that any value # of that type is acceptable. Otherwise, we only generate the default. if isinstance(param.default, bool): @@ -182,14 +313,124 @@ def _strategy_for(param: inspect.Parameter) -> Union[st.SearchStrategy, InferTyp # failures in cases like the `flags` argument to regex functions. # Better in to keep it simple, and let the user elaborate if desired. return st.just(param.default) - # If there's no annotation and no default value, we check against a table - # of guesses of simple strategies for common argument names. - if "string" in param.name and "as" not in param.name: + return _guess_strategy_by_argname(name=param.name.lower()) + + +# fmt: off +BOOL_NAMES = ( + "keepdims", "verbose", "debug", "force", "train", "training", "trainable", "bias", + "shuffle", "show", "load", "pretrained", "save", "overwrite", "normalize", + "reverse", "success", "enabled", "strict", "copy", "quiet", "required", "inplace", + "recursive", "enable", "active", "create", "validate", "refresh", "use_bias", +) +POSITIVE_INTEGER_NAMES = ( + "width", "size", "length", "limit", "idx", "stride", "epoch", "epochs", "depth", + "pid", "steps", "iteration", "iterations", "vocab_size", "ttl", "count", +) +FLOAT_NAMES = ( + "real", "imag", "alpha", "theta", "beta", "sigma", "gamma", "angle", "reward", + "tau", "temperature", +) +STRING_NAMES = ( + "text", "txt", "password", "label", "prefix", "suffix", "desc", "description", + "str", "pattern", "subject", "reason", "comment", "prompt", "sentence", "sep", +) +# fmt: on + + +def _guess_strategy_by_argname(name: str) -> st.SearchStrategy: + """ + If all else fails, we try guessing a strategy based on common argument names. + + We wouldn't do this in builds() where strict correctness is required, but for + the ghostwriter we accept "good guesses" since the user would otherwise have + to change the strategy anyway - from `nothing()` - if we refused to guess. + + A "good guess" is _usually correct_, and _a reasonable mistake_ if not. + The logic below is therefore based on a manual reading of the builtins and + some standard-library docs, plus the analysis of about three hundred million + arguments in https://github.com/HypothesisWorks/hypothesis/issues/3311 + """ + # Special-cased names + if name in ("function", "func", "f"): + return st.functions() + if name in ("pred", "predicate"): + return st.functions(returns=st.booleans(), pure=True) + if name in ("iterable",): + return st.iterables(st.integers()) | st.iterables(st.text()) + if name in ("list", "lst", "ls"): + return st.lists(st.nothing()) + if name in ("object",): + return st.builds(object) + if "uuid" in name: + return st.uuids().map(str) + + # Names which imply the value is a boolean + if name.startswith("is_") or name in BOOL_NAMES: + return st.booleans() + + # Names which imply that the value is a number, perhaps in a particular range + if name in ("amount", "threshold", "number", "num"): + return st.integers() | st.floats() + + if name in ("port",): + return st.integers(0, 2**16 - 1) + if ( + name.endswith("_size") + or (name.endswith("size") and "_" not in name) + or re.fullmatch(r"n(um)?_[a-z_]*s", name) + or name in POSITIVE_INTEGER_NAMES + ): + return st.integers(min_value=0) + if name in ("offset", "seed", "dim", "total", "priority"): + return st.integers() + + if name in ("learning_rate", "dropout", "dropout_rate", "epsilon", "eps", "prob"): + return st.floats(0, 1) + if name in ("lat", "latitude"): + return st.floats(-90, 90) + if name in ("lon", "longitude"): + return st.floats(-180, 180) + if name in ("radius", "tol", "tolerance", "rate"): + return st.floats(min_value=0) + if name in FLOAT_NAMES: + return st.floats() + + # Names which imply that the value is a string + if name in ("host", "hostname"): + return domains() + if name in ("email",): + return st.emails() + if name in ("word", "slug", "api_key"): + return st.from_regex(r"\w+", fullmatch=True) + if name in ("char", "character"): + return st.characters() + + if ( + "file" in name + or "path" in name + or name.endswith("_dir") + or name in ("fname", "dir", "dirname", "directory", "folder") + ): + # Common names for filesystem paths: these are usually strings, but we + # don't want to make strings more convenient than pathlib.Path. + return st.nothing() + + if ( + name.endswith("_name") + or (name.endswith("name") and "_" not in name) + or ("string" in name and "as" not in name) + or name.endswith("label") + or name in STRING_NAMES + ): return st.text() - for strategy, names in _GUESS_STRATEGIES_BY_NAME: - if param.name in names: - assert isinstance(strategy, st.SearchStrategy) - return strategy + + # Last clever idea: maybe we're looking a plural, and know the singular: + if re.fullmatch(r"\w*[^s]s", name): + elems = _guess_strategy_by_argname(name[:-1]) + if not elems.is_empty: + return st.lists(elems) + # And if all that failed, we'll return nothing() - the user will have to # fill this in by hand, and we'll leave a comment to that effect later. return st.nothing() @@ -199,7 +440,7 @@ def _get_params(func: Callable) -> Dict[str, inspect.Parameter]: """Get non-vararg parameters of `func` as an ordered dict.""" var_param_kinds = (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD) try: - params = list(inspect.signature(func).parameters.values()) + params = list(get_signature(func).parameters.values()) except Exception: if ( isinstance(func, (types.BuiltinFunctionType, types.BuiltinMethodType)) @@ -207,18 +448,29 @@ def _get_params(func: Callable) -> Dict[str, inspect.Parameter]: and isinstance(func.__doc__, str) ): # inspect.signature doesn't work on all builtin functions or methods. - # In such cases, including the operator module on Python 3.6, we can try - # to reconstruct simple signatures from the docstring. - pattern = rf"^{func.__name__}\(([a-z]+(, [a-z]+)*)(, \\)?\)" - args = re.match(pattern, func.__doc__) - if args is None: + # In such cases, we can try to reconstruct simple signatures from the docstring. + match = re.match(rf"^{func.__name__}\((.+?)\)", func.__doc__) + if match is None: raise - params = [ - # Note that we assume that the args are positional-only regardless of - # whether the signature shows a `/`, because this is often the case. - inspect.Parameter(name=name, kind=inspect.Parameter.POSITIONAL_ONLY) - for name in args.group(1).split(", ") - ] + args = match.group(1).replace("[", "").replace("]", "") + params = [] + # Even if the signature doesn't contain a /, we assume that arguments + # are positional-only until shown otherwise - the / is often omitted. + kind: inspect._ParameterKind = inspect.Parameter.POSITIONAL_ONLY + for arg in args.split(", "): + arg, *_ = arg.partition("=") + arg = arg.strip() + if arg == "/": + kind = inspect.Parameter.POSITIONAL_OR_KEYWORD + continue + if arg.startswith("*") or arg == "...": + kind = inspect.Parameter.KEYWORD_ONLY + continue # we omit *varargs, if there are any + if iskeyword(arg.lstrip("*")) or not arg.lstrip("*").isidentifier(): + print(repr(args)) + break # skip all subsequent params if this name is invalid + params.append(inspect.Parameter(name=arg, kind=kind)) + elif _is_probably_ufunc(func): # `inspect.signature` doesn't work on ufunc objects, but we can work out # what the required parameters would look like if it did. @@ -267,14 +519,17 @@ def _get_strategies( This dict is used to construct our call to the `@given(...)` decorator. """ - given_strategies = {} # type: Dict[str, st.SearchStrategy] + assert funcs, "Must pass at least one function" + given_strategies: Dict[str, st.SearchStrategy] = {} for i, f in enumerate(funcs): params = _get_params(f) if pass_result_to_next_func and i >= 1: del params[next(iter(params))] hints = get_type_hints(f) + docstring = getattr(f, "__doc__", None) or "" builder_args = { - k: infer if k in hints else _strategy_for(v) for k, v in params.items() + k: ... if k in hints else _strategy_for(v, docstring) + for k, v in params.items() } with _with_any_registered(): strat = st.builds(f, **builder_args).wrapped_strategy # type: ignore @@ -298,7 +553,7 @@ def _get_strategies( return dict(sorted(given_strategies.items())) -def _assert_eq(style, a, b): +def _assert_eq(style: str, a: str, b: str) -> str: if style == "unittest": return f"self.assertEqual({a}, {b})" assert style == "pytest" @@ -309,7 +564,14 @@ def _assert_eq(style, a, b): def _imports_for_object(obj): """Return the imports for `obj`, which may be empty for e.g. lambdas""" + if isinstance(obj, RE_TYPES): + return {"re"} try: + if is_generic_type(obj): + if isinstance(obj, TypeVar): + return {(obj.__module__, obj.__name__)} + with contextlib.suppress(Exception): + return set().union(*map(_imports_for_object, obj.__args__)) if (not callable(obj)) or obj.__name__ == "": return set() name = _get_qualname(obj).split(".")[0] @@ -321,8 +583,13 @@ def _imports_for_object(obj): def _imports_for_strategy(strategy): # If we have a lazy from_type strategy, because unwrapping it gives us an # error or invalid syntax, import that type and we're done. - if isinstance(strategy, LazyStrategy) and strategy.function is st.from_type: - return _imports_for_object(strategy._LazyStrategy__args[0]) + if isinstance(strategy, LazyStrategy): + if strategy.function is st.from_type: + return _imports_for_object(strategy._LazyStrategy__args[0]) + elif _get_module(strategy.function).startswith("hypothesis.extra."): + return {(_get_module(strategy.function), strategy.function.__name__)} + module = _get_module(strategy.function).replace("._array_helpers", ".numpy") + return {(module, strategy.function.__name__)} imports = set() strategy = unwrap_strategies(strategy) @@ -352,6 +619,13 @@ def _imports_for_strategy(strategy): for s in strategy.kwargs.values(): imports |= _imports_for_strategy(s) + if isinstance(strategy, SampledFromStrategy): + for obj in strategy.elements: + imports |= _imports_for_object(obj) + + if isinstance(strategy, ListStrategy): + imports |= _imports_for_strategy(strategy.element_strategy) + return imports @@ -366,6 +640,8 @@ def _valid_syntax_repr(strategy): seen = set() elems = [] for s in strategy.element_strategies: + if isinstance(s, SampledFromStrategy) and s.elements == (os.environ,): + continue if repr(s) not in seen: elems.append(s) seen.add(repr(s)) @@ -384,29 +660,40 @@ def _valid_syntax_repr(strategy): # .filter(), and .flatmap() to work without NameError imports = {i for i in _imports_for_strategy(strategy) if i[1] in r} return imports, r - except (SyntaxError, InvalidArgument): + except (SyntaxError, RecursionError, InvalidArgument): return set(), "nothing()" # When we ghostwrite for a module, we want to treat that as the __module__ for # each function, rather than whichever internal file it was actually defined in. KNOWN_FUNCTION_LOCATIONS: Dict[object, str] = {} -# (g)ufuncs do not have a __module__ attribute, so we simply look for them in -# any of the following modules which are found in sys.modules. The order of -# these entries doesn't matter, because we check identity of the found object. -LOOK_FOR_UFUNCS_IN_MODULES = ("numpy", "astropy", "erfa", "dask", "numba") + + +def _get_module_helper(obj): + # Get the __module__ attribute of the object, and return the first ancestor module + # which contains the object; falling back to the literal __module__ if none do. + # The goal is to show location from which obj should usually be accessed, rather + # than what we assume is an internal submodule which defined it. + module_name = obj.__module__ + dots = [i for i, c in enumerate(module_name) if c == "."] + [None] + for idx in dots: + if getattr(sys.modules.get(module_name[:idx]), obj.__name__, None) is obj: + KNOWN_FUNCTION_LOCATIONS[obj] = module_name[:idx] + return module_name[:idx] + return module_name def _get_module(obj): if obj in KNOWN_FUNCTION_LOCATIONS: return KNOWN_FUNCTION_LOCATIONS[obj] try: - return obj.__module__ + return _get_module_helper(obj) except AttributeError: if not _is_probably_ufunc(obj): raise - for module_name in LOOK_FOR_UFUNCS_IN_MODULES: - if obj is getattr(sys.modules.get(module_name), obj.__name__, None): + for module_name in sorted(sys.modules, key=lambda n: tuple(n.split("."))): + if obj is getattr(sys.modules[module_name], obj.__name__, None): + KNOWN_FUNCTION_LOCATIONS[obj] = module_name return module_name raise RuntimeError(f"Could not find module for ufunc {obj.__name__} ({obj!r}") @@ -420,11 +707,20 @@ def _get_qualname(obj, include_module=False): return qname -def _write_call(func: Callable, *pass_variables: str) -> str: +def _write_call( + func: Callable, *pass_variables: str, except_: Except, assign: str = "" +) -> str: """Write a call to `func` with explicit and implicit arguments. >>> _write_call(sorted, "my_seq", "func") "builtins.sorted(my_seq, key=func, reverse=reverse)" + + >>> write_call(f, assign="var1") + "var1 = f()" + + The fancy part is that we'll check the docstring for any known exceptions + which `func` might raise, and catch-and-reject on them... *unless* they're + subtypes of `except_`, which will be handled in an outer try-except block. """ args = ", ".join( (v or p.name) @@ -432,7 +728,27 @@ def _write_call(func: Callable, *pass_variables: str) -> str: else f"{p.name}={v or p.name}" for v, p in zip_longest(pass_variables, _get_params(func).values()) ) - return f"{_get_qualname(func, include_module=True)}({args})" + call = f"{_get_qualname(func, include_module=True)}({args})" + if assign: + call = f"{assign} = {call}" + raises = _exceptions_from_docstring(getattr(func, "__doc__", "") or "") + exnames = [ex.__name__ for ex in raises if not issubclass(ex, except_)] + if not exnames: + return call + return SUPPRESS_BLOCK.format( + test_body=indent(call, prefix=" "), + exceptions="(" + ", ".join(exnames) + ")" if len(exnames) > 1 else exnames[0], + ) + + +def _st_strategy_names(s: str) -> str: + """Replace strategy name() with st.name(). + + Uses a tricky re.sub() to avoid problems with frozensets() matching + sets() too. + """ + names = "|".join(sorted(st.__all__, key=len, reverse=True)) + return re.sub(pattern=rf"\b(?:{names})\(", repl=r"st.\g<0>", string=s) def _make_test_body( @@ -440,13 +756,15 @@ def _make_test_body( ghost: str, test_body: str, except_: Tuple[Type[Exception], ...], + assertions: str = "", style: str, given_strategies: Optional[Mapping[str, Union[str, st.SearchStrategy]]] = None, -) -> Tuple[Set[Union[str, Tuple[str, str]]], str]: + imports: Optional[ImportSet] = None, +) -> Tuple[ImportSet, str]: # A set of modules to import - we might add to this later. The import code # is written later, so we can have one import section for multiple magic() # test functions. - imports = {_get_module(f) for f in funcs} + imports = (imports or set()) | {_get_module(f) for f in funcs} # Get strategies for all the arguments to each function we're testing. with _with_any_registered(): @@ -456,34 +774,22 @@ def _make_test_body( reprs = [((k,) + _valid_syntax_repr(v)) for k, v in given_strategies.items()] imports = imports.union(*(imp for _, imp, _ in reprs)) given_args = ", ".join(f"{k}={v}" for k, _, v in reprs) - for name in st.__all__: - given_args = given_args.replace(f"{name}(", f"st.{name}(") + given_args = _st_strategy_names(given_args) if except_: - # This is reminiscent of de-duplication logic I wrote for flake8-bugbear, - # but with access to the actual objects we can just check for subclasses. - # This lets us print e.g. `Exception` instead of `(Exception, OSError)`. - uniques = list(except_) - for a, b in permutations(except_, 2): - if a in uniques and issubclass(a, b): - uniques.remove(a) - # Then convert to strings, either builtin names or qualified names. - exceptions = [] - for ex in uniques: - if ex.__qualname__ in dir(builtins): - exceptions.append(ex.__qualname__) - else: - imports.add(ex.__module__) - exceptions.append(_get_qualname(ex, include_module=True)) + # Convert to strings, either builtin names or qualified names. + imp, exc_string = _exception_string(except_) + imports.update(imp) # And finally indent the existing test body into a try-except block # which catches these exceptions and calls `hypothesis.reject()`. test_body = SUPPRESS_BLOCK.format( test_body=indent(test_body, prefix=" "), - exceptions="(" + ", ".join(exceptions) + ")" - if len(exceptions) > 1 - else exceptions[0], + exceptions=exc_string, ) + if assertions: + test_body = f"{test_body}\n{assertions}" + # Indent our test code to form the body of a function or method. argnames = (["self"] if style == "unittest" else []) + list(given_strategies) body = TEMPLATE.format( @@ -506,7 +812,7 @@ def _make_test_body( return imports, body -def _make_test(imports: Set[Union[str, Tuple[str, str]]], body: str) -> str: +def _make_test(imports: ImportSet, body: str) -> str: # Discarding "builtins." and "__main__" probably isn't particularly useful # for user code, but important for making a good impression in demos. body = body.replace("builtins.", "").replace("__main__.", "") @@ -520,11 +826,12 @@ def _make_test(imports: Set[Union[str, Tuple[str, str]]], body: str) -> str: direct = {f"import {i}" for i in imports - do_not_import if isinstance(i, str)} from_imports = defaultdict(set) for module, name in {i for i in imports if isinstance(i, tuple)}: - from_imports[module].add(name) + if not (module.startswith("hypothesis.strategies") and name in st.__all__): + from_imports[module].add(name) from_ = { "from {} import {}".format(module, ", ".join(sorted(names))) for module, names in from_imports.items() - if module not in do_not_import + if isinstance(module, str) and module not in do_not_import } header = IMPORT_SECTION.format(imports="\n".join(sorted(direct) + sorted(from_))) nothings = body.count("st.nothing()") @@ -555,12 +862,17 @@ def _is_probably_ufunc(obj): (r"(.*)en(.+)", "{}de{}"), # Shared postfix, prefix only on "inverse" function (r"(.+)", "de{}"), - (r"(.+)", "un{}"), + (r"(?!safe)(.+)", "un{}"), # safe_load / unsafe_load isn't a roundtrip # a2b_postfix and b2a_postfix. Not a fan of this pattern, but it's pretty # common in code imitating an C API - see e.g. the stdlib binascii module. (r"(.+)2(.+?)(_.+)?", "{1}2{0}{2}"), # Common in e.g. the colorsys module (r"(.+)_to_(.+)", "{1}_to_{0}"), + # Sockets patterns + (r"(inet|if)_(.+)to(.+)", "{0}_{2}to{1}"), + (r"(\w)to(\w)(.+)", "{1}to{0}{2}"), + (r"send(.+)", "recv{}"), + (r"send(.+)", "receive{}"), ) @@ -591,39 +903,72 @@ def magic( for thing in modules_or_functions: if callable(thing): functions.add(thing) + # class need to be added for exploration + if inspect.isclass(thing): + funcs: List[Optional[Any]] = [thing] + else: + funcs = [] elif isinstance(thing, types.ModuleType): if hasattr(thing, "__all__"): - funcs = [getattr(thing, name, None) for name in thing.__all__] # type: ignore - else: + funcs = [getattr(thing, name, None) for name in thing.__all__] + elif hasattr(thing, "__package__"): + pkg = thing.__package__ funcs = [ v for k, v in vars(thing).items() - if callable(v) and not k.startswith("_") + if callable(v) + and not is_mock(v) + and ((not pkg) or getattr(v, "__module__", pkg).startswith(pkg)) + and not k.startswith("_") ] - for f in funcs: - try: - if ( - (not is_mock(f)) - and callable(f) - and _get_params(f) - and not isinstance(f, enum.EnumMeta) - ): - functions.add(f) - if getattr(thing, "__name__", None): - KNOWN_FUNCTION_LOCATIONS[f] = thing.__name__ - except (TypeError, ValueError): - pass + if pkg and any(getattr(f, "__module__", pkg) == pkg for f in funcs): + funcs = [f for f in funcs if getattr(f, "__module__", pkg) == pkg] else: raise InvalidArgument(f"Can't test non-module non-callable {thing!r}") + for f in list(funcs): + if inspect.isclass(f): + funcs += [ + v.__get__(f) + for k, v in vars(f).items() + if hasattr(v, "__func__") + and not is_mock(v) + and not k.startswith("_") + ] + for f in funcs: + try: + if ( + (not is_mock(f)) + and callable(f) + and _get_params(f) + and not isinstance(f, enum.EnumMeta) + ): + functions.add(f) + if getattr(thing, "__name__", None): + if inspect.isclass(thing): + KNOWN_FUNCTION_LOCATIONS[f] = _get_module_helper(thing) + else: + KNOWN_FUNCTION_LOCATIONS[f] = thing.__name__ + except (TypeError, ValueError): + pass + imports = set() parts = [] + + def make_(how, *args, **kwargs): + imp, body = how(*args, **kwargs, except_=except_, style=style) + imports.update(imp) + parts.append(body) + by_name = {} for f in functions: try: + _get_params(f) by_name[_get_qualname(f, include_module=True)] = f except Exception: - pass # e.g. Pandas 'CallableDynamicDoc' object has no attribute '__name__' + # usually inspect.signature on C code such as socket.inet_aton, sometimes + # e.g. Pandas 'CallableDynamicDoc' object has no attribute '__name__' + pass if not by_name: return ( f"# Found no testable functions in\n" @@ -638,14 +983,20 @@ def magic( inverse_name = readname.format(*match.groups()) for other in sorted( n for n in by_name if n.split(".")[-1] == inverse_name - )[:1]: - imp, body = _make_roundtrip_body( - (by_name.pop(name), by_name.pop(other)), - except_=except_, - style=style, - ) - imports |= imp - parts.append(body) + ): + make_(_make_roundtrip_body, (by_name.pop(name), by_name.pop(other))) + break + else: + try: + other_func = getattr( + sys.modules[_get_module(by_name[name])], + inverse_name, + ) + _get_params(other_func) # we want to skip if this fails + except Exception: + pass + else: + make_(_make_roundtrip_body, (by_name.pop(name), other_func)) # Look for equivalent functions: same name, all required arguments of any can # be found in all signatures, and if all have return-type annotations they match. @@ -657,9 +1008,7 @@ def magic( sentinel = object() returns = {get_type_hints(f).get("return", sentinel) for f in group} if len(returns - {sentinel}) <= 1: - imp, body = _make_equiv_body(group, except_=except_, style=style) - imports |= imp - parts.append(body) + make_(_make_equiv_body, group) for f in group: by_name.pop(_get_qualname(f, include_module=True)) @@ -668,32 +1017,28 @@ def magic( for name, func in sorted(by_name.items()): hints = get_type_hints(func) hints.pop("return", None) - if len(hints) == len(_get_params(func)) == 2: + params = _get_params(func) + if len(hints) == len(params) == 2: a, b = hints.values() - if a == b: - imp, body = _make_binop_body(func, except_=except_, style=style) - imports |= imp - parts.append(body) + arg1, arg2 = params + if a == b and len(arg1) == len(arg2) <= 3: + make_(_make_binop_body, func) del by_name[name] # Look for Numpy ufuncs or gufuncs, and write array-oriented tests for them. if "numpy" in sys.modules: for name, func in sorted(by_name.items()): if _is_probably_ufunc(func): - imp, body = _make_ufunc_body(func, except_=except_, style=style) - imports |= imp - parts.append(body) + make_(_make_ufunc_body, func) del by_name[name] # For all remaining callables, just write a fuzz-test. In principle we could # guess at equivalence or idempotence; but it doesn't seem accurate enough to # be worth the trouble when it's so easy for the user to specify themselves. for _, f in sorted(by_name.items()): - imp, body = _make_test_body( - f, test_body=_write_call(f), except_=except_, ghost="fuzz", style=style + make_( + _make_test_body, f, test_body=_write_call(f, except_=except_), ghost="fuzz" ) - imports |= imp - parts.append(body) return _make_test(imports, "\n".join(parts)) @@ -742,7 +1087,11 @@ def test_fuzz_compile(pattern, flags): _check_style(style) imports, body = _make_test_body( - func, test_body=_write_call(func), except_=except_, ghost="fuzz", style=style + func, + test_body=_write_call(func, except_=except_), + except_=except_, + ghost="fuzz", + style=style, ) return _make_test(imports, body) @@ -787,14 +1136,16 @@ def test_idempotent_timsort(seq): except_ = _check_except(except_) _check_style(style) - test_body = "result = {}\nrepeat = {}\n{}".format( - _write_call(func), - _write_call(func, "result"), - _assert_eq(style, "result", "repeat"), - ) - imports, body = _make_test_body( - func, test_body=test_body, except_=except_, ghost="idempotent", style=style + func, + test_body="result = {}\nrepeat = {}".format( + _write_call(func, except_=except_), + _write_call(func, "result", except_=except_), + ), + except_=except_, + assertions=_assert_eq(style, "result", "repeat"), + ghost="idempotent", + style=style, ) return _make_test(imports, body) @@ -802,17 +1153,17 @@ def test_idempotent_timsort(seq): def _make_roundtrip_body(funcs, except_, style): first_param = next(iter(_get_params(funcs[0]))) test_lines = [ - "value0 = " + _write_call(funcs[0]), + _write_call(funcs[0], assign="value0", except_=except_), *( - f"value{i + 1} = " + _write_call(f, f"value{i}") + _write_call(f, f"value{i}", assign=f"value{i + 1}", except_=except_) for i, f in enumerate(funcs[1:]) ), - _assert_eq(style, first_param, f"value{len(funcs) - 1}"), ] return _make_test_body( *funcs, test_body="\n".join(test_lines), except_=except_, + assertions=_assert_eq(style, first_param, f"value{len(funcs) - 1}"), ghost="roundtrip", style=style, ) @@ -847,23 +1198,89 @@ def _make_equiv_body(funcs, except_, style): if len(set(var_names)) < len(var_names): var_names = [f"result_{i}_{ f.__name__}" for i, f in enumerate(funcs)] test_lines = [ - vname + " = " + _write_call(f) for vname, f in zip(var_names, funcs) - ] + [_assert_eq(style, var_names[0], vname) for vname in var_names[1:]] + _write_call(f, assign=vname, except_=except_) + for vname, f in zip(var_names, funcs) + ] + assertions = "\n".join( + _assert_eq(style, var_names[0], vname) for vname in var_names[1:] + ) return _make_test_body( *funcs, test_body="\n".join(test_lines), except_=except_, + assertions=assertions, + ghost="equivalent", + style=style, + ) + + +EQUIV_FIRST_BLOCK = """ +try: +{} + exc_type = None + target(1, label="input was valid") +{}except Exception as exc: + exc_type = type(exc) +""".strip() + +EQUIV_CHECK_BLOCK = """ +if exc_type: + with {ctx}(exc_type): +{check_raises} +else: +{call} +{compare} +""".rstrip() + + +def _make_equiv_errors_body(funcs, except_, style): + var_names = [f"result_{f.__name__}" for f in funcs] + if len(set(var_names)) < len(var_names): + var_names = [f"result_{i}_{ f.__name__}" for i, f in enumerate(funcs)] + + first, *rest = funcs + first_call = _write_call(first, assign=var_names[0], except_=except_) + extra_imports, suppress = _exception_string(except_) + extra_imports.add(("hypothesis", "target")) + catch = f"except {suppress}:\n reject()\n" if suppress else "" + test_lines = [EQUIV_FIRST_BLOCK.format(indent(first_call, prefix=" "), catch)] + + for vname, f in zip(var_names[1:], rest): + if style == "pytest": + ctx = "pytest.raises" + extra_imports.add("pytest") + else: + assert style == "unittest" + ctx = "self.assertRaises" + block = EQUIV_CHECK_BLOCK.format( + ctx=ctx, + check_raises=indent(_write_call(f, except_=()), " "), + call=indent(_write_call(f, assign=vname, except_=()), " "), + compare=indent(_assert_eq(style, var_names[0], vname), " "), + ) + test_lines.append(block) + + imports, source_code = _make_test_body( + *funcs, + test_body="\n".join(test_lines), + except_=(), ghost="equivalent", style=style, ) + return imports | extra_imports, source_code -def equivalent(*funcs: Callable, except_: Except = (), style: str = "pytest") -> str: +def equivalent( + *funcs: Callable, + allow_same_errors: bool = False, + except_: Except = (), + style: str = "pytest", +) -> str: """Write source code for a property-based test of ``funcs``. - The resulting test checks that calling each of the functions gives - the same result. This can be used as a classic 'oracle', such as testing + The resulting test checks that calling each of the functions returns + an equal value. This can be used as a classic 'oracle', such as testing a fast sorting algorithm against the :func:`python:sorted` builtin, or for differential testing where none of the compared functions are fully trusted but any difference indicates a bug (e.g. running a function on @@ -872,15 +1289,25 @@ def equivalent(*funcs: Callable, except_: Except = (), style: str = "pytest") -> The functions should have reasonably similar signatures, as only the common parameters will be passed the same arguments - any other parameters will be allowed to vary. + + If allow_same_errors is True, then the test will pass if calling each of + the functions returns an equal value, *or* if the first function raises an + exception and each of the others raises an exception of the same type. + This relaxed mode can be useful for code synthesis projects. """ if len(funcs) < 2: raise InvalidArgument("Need at least two functions to compare.") for i, f in enumerate(funcs): if not callable(f): raise InvalidArgument(f"Got non-callable funcs[{i}]={f!r}") + check_type(bool, allow_same_errors, "allow_same_errors") except_ = _check_except(except_) _check_style(style) - return _make_test(*_make_equiv_body(funcs, except_, style)) + if allow_same_errors and not any(issubclass(Exception, ex) for ex in except_): + imports, source_code = _make_equiv_errors_body(funcs, except_, style) + else: + imports, source_code = _make_equiv_body(funcs, except_, style) + return _make_test(imports, source_code) X = TypeVar("X") @@ -892,7 +1319,7 @@ def binary_operation( *, associative: bool = True, commutative: bool = True, - identity: Union[X, InferType, None] = infer, + identity: Union[X, EllipsisType, None] = ..., distributes_over: Optional[Callable[[X, X], X]] = None, except_: Except = (), style: str = "pytest", @@ -953,15 +1380,13 @@ def _make_binop_body( *, associative: bool = True, commutative: bool = True, - identity: Union[X, InferType, None] = infer, + identity: Union[X, EllipsisType, None] = ..., distributes_over: Optional[Callable[[X, X], X]] = None, except_: Tuple[Type[Exception], ...], style: str, -) -> Tuple[Set[Union[str, Tuple[str, str]]], str]: - # TODO: collapse togther first two strategies, keep any others (for flags etc.) - # assign this as a global variable, which will be prepended to the test bodies +) -> Tuple[ImportSet, str]: strategies = _get_strategies(func) - operands, b = [strategies.pop(p) for p in list(_get_params(func))[:2]] + operands, b = (strategies.pop(p) for p in list(_get_params(func))[:2]) if repr(operands) != repr(b): operands |= b operands_name = func.__name__ + "_operands" @@ -975,13 +1400,17 @@ def maker( body: str, right: Optional[str] = None, ) -> None: - if right is not None: - body = f"left={body}\nright={right}\n" + _assert_eq(style, "left", "right") + if right is None: + assertions = "" + else: + body = f"{body}\n{right}" + assertions = _assert_eq(style, "left", "right") imports, body = _make_test_body( func, test_body=body, ghost=sub_property + "_binary_operation", except_=except_, + assertions=assertions, style=style, given_strategies={**strategies, **{n: operands_name for n in args}}, ) @@ -995,31 +1424,43 @@ def maker( maker( "associative", "abc", - _write_call(func, "a", _write_call(func, "b", "c")), - _write_call(func, _write_call(func, "a", "b"), "c"), + _write_call( + func, + "a", + _write_call(func, "b", "c", except_=Exception), + except_=Exception, + assign="left", + ), + _write_call( + func, + _write_call(func, "a", "b", except_=Exception), + "c", + except_=Exception, + assign="right", + ), ) if commutative: maker( "commutative", "ab", - _write_call(func, "a", "b"), - _write_call(func, "b", "a"), + _write_call(func, "a", "b", except_=Exception, assign="left"), + _write_call(func, "b", "a", except_=Exception, assign="right"), ) if identity is not None: # Guess that the identity element is the minimal example from our operands # strategy. This is correct often enough to be worthwhile, and close enough # that it's a good starting point to edit much of the rest. - if identity is infer: + if identity is ...: try: - identity = find(operands, lambda x: True) + identity = find(operands, lambda x: True, settings=_quietly_settings) except Exception: identity = "identity element here" # type: ignore # If the repr of this element is invalid Python, stringify it - this # can't be executed as-is, but at least makes it clear what should - # happpen. E.g. type(None) -> -> quoted. + # happen. E.g. type(None) -> -> quoted. try: # We don't actually execute this code object; we're just compiling - # to check that the repr is syntatically valid. HOWEVER, we're + # to check that the repr is syntactically valid. HOWEVER, we're # going to output that code string into test code which will be # executed; so you still shouldn't ghostwrite for hostile code. compile(repr(identity), "", "exec") @@ -1028,7 +1469,11 @@ def maker( maker( "identity", "a", - _assert_eq(style, "a", _write_call(func, "a", repr(identity))), + _assert_eq( + style, + "a", + _write_call(func, "a", repr(identity), except_=Exception), + ), ) if distributes_over: maker( @@ -1036,15 +1481,22 @@ def maker( "abc", _write_call( distributes_over, - _write_call(func, "a", "b"), - _write_call(func, "a", "c"), + _write_call(func, "a", "b", except_=Exception), + _write_call(func, "a", "c", except_=Exception), + except_=Exception, + assign="left", + ), + _write_call( + func, + "a", + _write_call(distributes_over, "b", "c", except_=Exception), + except_=Exception, + assign="right", ), - _write_call(func, "a", _write_call(distributes_over, "b", "c")), ) _, operands_repr = _valid_syntax_repr(operands) - for name in st.__all__: - operands_repr = operands_repr.replace(f"{name}(", f"st.{name}(") + operands_repr = _st_strategy_names(operands_repr) classdef = "" if style == "unittest": classdef = f"class TestBinaryOperation{func.__name__}(unittest.TestCase):\n " @@ -1055,10 +1507,10 @@ def maker( def ufunc(func: Callable, *, except_: Except = (), style: str = "pytest") -> str: - """Write a property-based test for the :np-ref:`array unfunc ` ``func``. + """Write a property-based test for the :doc:`array ufunc ` ``func``. - The resulting test checks that your ufunc or :np-ref:`gufunc - ` has the expected broadcasting and dtype casting + The resulting test checks that your ufunc or :doc:`gufunc + ` has the expected broadcasting and dtype casting behaviour. You will probably want to add extra assertions, but as with the other ghostwriters this gives you a great place to start. @@ -1081,37 +1533,41 @@ def _make_ufunc_body(func, *, except_, style): shapes = npst.mutually_broadcastable_shapes(num_shapes=func.nin) else: shapes = npst.mutually_broadcastable_shapes(signature=func.signature) + shapes.function.__module__ = npst.__name__ body = """ input_shapes, expected_shape = shapes input_dtypes, expected_dtype = types.split("->") - array_st = [npst.arrays(d, s) for d, s in zip(input_dtypes, input_shapes)] + array_strats = [ + arrays(dtype=dtp, shape=shp, elements={{"allow_nan": True}}) + for dtp, shp in zip(input_dtypes, input_shapes) + ] - {array_names} = data.draw(st.tuples(*array_st)) + {array_names} = data.draw(st.tuples(*array_strats)) result = {call} - - {shape_assert} - {type_assert} """.format( array_names=", ".join(ascii_lowercase[: func.nin]), - call=_write_call(func, *ascii_lowercase[: func.nin]), + call=_write_call(func, *ascii_lowercase[: func.nin], except_=except_), + ) + assertions = "\n{shape_assert}\n{type_assert}".format( shape_assert=_assert_eq(style, "result.shape", "expected_shape"), type_assert=_assert_eq(style, "result.dtype.char", "expected_dtype"), ) - imports, body = _make_test_body( + qname = _get_qualname(func, include_module=True) + obj_sigs = ["O" in sig for sig in func.types] + if all(obj_sigs) or not any(obj_sigs): + types = f"sampled_from({qname}.types)" + else: + types = f"sampled_from([sig for sig in {qname}.types if 'O' not in sig])" + + return _make_test_body( func, test_body=dedent(body).strip(), except_=except_, + assertions=assertions, ghost="ufunc" if func.signature is None else "gufunc", style=style, - given_strategies={ - "data": st.data(), - "shapes": shapes, - "types": f"sampled_from({_get_qualname(func, include_module=True)}.types)" - ".filter(lambda sig: 'O' not in sig)", - }, + given_strategies={"data": st.data(), "shapes": shapes, "types": types}, + imports={("hypothesis.extra.numpy", "arrays")}, ) - imports.add("hypothesis.extra.numpy as npst") - body = body.replace("mutually_broadcastable", "npst.mutually_broadcastable") - return imports, body diff --git a/hypothesis-python/src/hypothesis/extra/lark.py b/hypothesis-python/src/hypothesis/extra/lark.py index ff332cfa38..127604b8e4 100644 --- a/hypothesis-python/src/hypothesis/extra/lark.py +++ b/hypothesis-python/src/hypothesis/extra/lark.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """ ---------------- @@ -31,20 +26,21 @@ Note that as Lark is at version 0.x, this module *may* break API compatibility in minor releases if supporting the latest version of Lark would otherwise be infeasible. We may also be quite aggressive in bumping the minimum version of -Lark, unless someone volunteers to either fund or do the maintainence. +Lark, unless someone volunteers to either fund or do the maintenance. """ -from inspect import getfullargspec +from inspect import signature from typing import Dict, Optional import attr import lark -from lark.grammar import NonTerminal, Terminal +from lark.grammar import NonTerminal, Terminal # type: ignore +from hypothesis import strategies as st from hypothesis.errors import InvalidArgument from hypothesis.internal.conjecture.utils import calc_label_from_name from hypothesis.internal.validation import check_type -from hypothesis.strategies._internal import SearchStrategy, core as st +from hypothesis.strategies._internal.utils import cacheable, defines_strategy __all__ = ["from_lark"] @@ -77,7 +73,7 @@ def get_terminal_names(terminals, rules, ignore_names): return names -class LarkStrategy(SearchStrategy): +class LarkStrategy(st.SearchStrategy): """Low-level strategy implementation wrapping a Lark grammar. See ``from_lark`` for details. @@ -94,7 +90,7 @@ def __init__(self, grammar, start, explicit): # This is a total hack, but working around the changes is a nicer user # experience than breaking for anyone who doesn't instantly update their # installation of Lark alongside Hypothesis. - compile_args = getfullargspec(grammar.grammar.compile).args + compile_args = signature(grammar.grammar.compile).parameters if "terminals_to_keep" in compile_args: terminals, rules, ignore_names = grammar.grammar.compile(start, ()) elif "start" in compile_args: # pragma: no cover @@ -170,7 +166,7 @@ def draw_symbol(self, data, symbol, draw_state): "use of %%declare unless you pass `explicit`, a dict of " 'names-to-strategies, such as `{%r: st.just("")}`' % (symbol.name, symbol.name) - ) + ) from None draw_state.result.append(data.draw(strategy)) else: assert isinstance(symbol, NonTerminal) @@ -198,8 +194,8 @@ def inner(value): return inner -@st.cacheable -@st.defines_strategy(force_reusable_values=True) +@cacheable +@defines_strategy(force_reusable_values=True) def from_lark( grammar: lark.lark.Lark, *, diff --git a/hypothesis-python/src/hypothesis/extra/numpy.py b/hypothesis-python/src/hypothesis/extra/numpy.py index e195f85945..06376ae42a 100644 --- a/hypothesis-python/src/hypothesis/extra/numpy.py +++ b/hypothesis-python/src/hypothesis/extra/numpy.py @@ -1,46 +1,73 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import math -import re -from typing import Any, Mapping, NamedTuple, Optional, Sequence, Tuple, Union +from typing import Any, Mapping, Optional, Sequence, Tuple, Union import numpy as np -from hypothesis import assume -from hypothesis.errors import InvalidArgument +from hypothesis import strategies as st +from hypothesis._settings import note_deprecation +from hypothesis.errors import HypothesisException, InvalidArgument +from hypothesis.extra._array_helpers import ( + NDIM_MAX, + BasicIndex, + BasicIndexStrategy, + BroadcastableShapes, + Shape, + array_shapes, + broadcastable_shapes, + check_argument, + check_valid_dims, + mutually_broadcastable_shapes as _mutually_broadcastable_shapes, + order_check, + valid_tuple_axes as _valid_tuple_axes, +) from hypothesis.internal.conjecture import utils as cu from hypothesis.internal.coverage import check_function from hypothesis.internal.reflection import proxies -from hypothesis.internal.validation import check_type, check_valid_interval -from hypothesis.strategies._internal import SearchStrategy, check_strategy, core as st -from hypothesis.strategies._internal.strategies import T -from hypothesis.utils.conventions import UniqueIdentifier, not_set - -Shape = Tuple[int, ...] -# flake8 and mypy disagree about `ellipsis` (the type of `...`), and hence: -BasicIndex = Tuple[Union[int, slice, "ellipsis", np.newaxis], ...] # noqa: F821 -TIME_RESOLUTIONS = tuple("Y M D h m s ms us ns ps fs as".split()) +from hypothesis.internal.validation import check_type +from hypothesis.strategies._internal.strategies import T, check_strategy +from hypothesis.strategies._internal.utils import defines_strategy + +__all__ = [ + "BroadcastableShapes", + "from_dtype", + "arrays", + "array_shapes", + "scalar_dtypes", + "boolean_dtypes", + "unsigned_integer_dtypes", + "integer_dtypes", + "floating_dtypes", + "complex_number_dtypes", + "datetime64_dtypes", + "timedelta64_dtypes", + "byte_string_dtypes", + "unicode_string_dtypes", + "array_dtypes", + "nested_dtypes", + "valid_tuple_axes", + "broadcastable_shapes", + "mutually_broadcastable_shapes", + "basic_indices", + "integer_array_indices", +] +TIME_RESOLUTIONS = tuple("Y M D h m s ms us ns ps fs as".split()) -class BroadcastableShapes(NamedTuple): - input_shapes: Tuple[Shape, ...] - result_shape: Shape +# See https://github.com/HypothesisWorks/hypothesis/pull/3394 and linked discussion. +NP_FIXED_UNICODE = tuple(int(x) for x in np.__version__.split(".")[:2]) >= (1, 19) -@st.defines_strategy(force_reusable_values=True) +@defines_strategy(force_reusable_values=True) def from_dtype( dtype: np.dtype, *, @@ -51,6 +78,7 @@ def from_dtype( max_value: Union[int, float, None] = None, allow_nan: Optional[bool] = None, allow_infinity: Optional[bool] = None, + allow_subnormal: Optional[bool] = None, exclude_min: Optional[bool] = None, exclude_max: Optional[bool] = None, ) -> st.SearchStrategy[Any]: @@ -93,15 +121,16 @@ def compat_kw(*args, **kw): # Scalar datatypes if dtype.kind == "b": - result = st.booleans() # type: SearchStrategy[Any] + result: st.SearchStrategy[Any] = st.booleans() elif dtype.kind == "f": result = st.floats( - width=8 * dtype.itemsize, + width=min(8 * dtype.itemsize, 64), **compat_kw( "min_value", "max_value", "allow_nan", "allow_infinity", + "allow_subnormal", "exclude_min", "exclude_max", ), @@ -131,6 +160,8 @@ def compat_kw(*args, **kw): elif dtype.kind == "U": # Encoded in UTF-32 (four bytes/codepoint) and null-terminated max_size = (dtype.itemsize or 0) // 4 or None + if NP_FIXED_UNICODE and "alphabet" not in kwargs: + kwargs["alphabet"] = st.characters() result = st.text(**compat_kw("alphabet", "min_size", max_size=max_size)).filter( lambda b: b[-1:] != "\0" ) @@ -142,37 +173,13 @@ def compat_kw(*args, **kw): # it here because we'd have to guard against equivalents in arrays() # regardless and drawing scalars is a valid use-case. res = st.sampled_from(TIME_RESOLUTIONS) - result = st.builds(dtype.type, st.integers(-(2 ** 63), 2 ** 63 - 1), res) + result = st.builds(dtype.type, st.integers(-(2**63), 2**63 - 1), res) else: raise InvalidArgument(f"No strategy inference for {dtype}") return result.map(dtype.type) -@check_function -def check_argument(condition, fail_message, *f_args, **f_kwargs): - if not condition: - raise InvalidArgument(fail_message.format(*f_args, **f_kwargs)) - - -@check_function -def order_check(name, floor, small, large): - check_argument( - floor <= small, - "min_{name} must be at least {} but was {}", - floor, - small, - name=name, - ) - check_argument( - small <= large, - "min_{name}={} is larger than max_{name}={}", - small, - large, - name=name, - ) - - -class ArrayStrategy(SearchStrategy): +class ArrayStrategy(st.SearchStrategy): def __init__(self, element_strategy, shape, dtype, fill, unique): self.shape = tuple(shape) self.fill = fill @@ -182,9 +189,7 @@ def __init__(self, element_strategy, shape, dtype, fill, unique): self.unique = unique self._check_elements = dtype.kind not in ("O", "V") - def set_element(self, data, result, idx, strategy=None): - strategy = strategy or self.element_strategy - val = data.draw(strategy) + def set_element(self, val, result, idx, *, fill=False): try: result[idx] = val except TypeError as err: @@ -192,7 +197,32 @@ def set_element(self, data, result, idx, strategy=None): f"Could not add element={val!r} of {val.dtype!r} to array of " f"{result.dtype!r} - possible mismatch of time units in dtypes?" ) from err - if self._check_elements and val != result[idx] and val == val: + try: + elem_changed = self._check_elements and val != result[idx] and val == val + except Exception as err: # pragma: no cover + # This branch only exists to help debug weird behaviour in Numpy, + # such as the string problems we had a while back. + raise HypothesisException( + "Internal error when checking element=%r of %r to array of %r" + % (val, val.dtype, result.dtype) + ) from err + if elem_changed: + strategy = self.fill if fill else self.element_strategy + if self.dtype.kind == "f": + try: + is_subnormal = 0 < abs(val) < np.finfo(self.dtype).tiny + except Exception: # pragma: no cover + # val may be a non-float that does not support the + # operations __lt__ and __abs__ + is_subnormal = False # pragma: no cover + if is_subnormal: + raise InvalidArgument( # pragma: no cover + f"Generated subnormal float {val} from strategy " + f"{strategy} resulted in {result[idx]!r}, probably " + "as a result of NumPy being built with flush-to-zero " + "compiler options. Consider passing " + "allow_subnormal=False." + ) raise InvalidArgument( "Generated array element %r from %r cannot be represented as " "dtype %r - instead it becomes %r (type %r). Consider using a more " @@ -225,28 +255,17 @@ def do_draw(self, data): # generate a fully dense array with a freshly drawn value for each # entry. if self.unique: - seen = set() - elements = cu.many( - data, + elems = st.lists( + self.element_strategy, min_size=self.array_size, max_size=self.array_size, - average_size=self.array_size, + unique=True, ) - i = 0 - while elements.more(): - # We assign first because this means we check for - # uniqueness after numpy has converted it to the relevant - # type for us. Because we don't increment the counter on - # a duplicate we will overwrite it on the next draw. - self.set_element(data, result, i) - if result[i] not in seen: - seen.add(result[i]) - i += 1 - else: - elements.reject() + for i, v in enumerate(data.draw(elems)): + self.set_element(v, result, i) else: for i in range(len(result)): - self.set_element(data, result, i) + self.set_element(data.draw(self.element_strategy), result, i) else: # We draw numpy arrays as "sparse with an offset". We draw a # collection of index assignments within the array and assign @@ -262,7 +281,10 @@ def do_draw(self, data): # sqrt isn't chosen for any particularly principled reason. It # just grows reasonably quickly but sublinearly, and for small # arrays it represents a decent fraction of the array size. - average_size=math.sqrt(self.array_size), + average_size=min( + 0.9 * self.array_size, # ensure small arrays sometimes use fill + max(10, math.sqrt(self.array_size)), # ...but *only* sometimes + ), ) needs_fill = np.full(self.array_size, True) @@ -273,7 +295,7 @@ def do_draw(self, data): if not needs_fill[i]: elements.reject() continue - self.set_element(data, result, i) + self.set_element(data.draw(self.element_strategy), result, i) if self.unique: if result[i] in seen: elements.reject() @@ -296,7 +318,7 @@ def do_draw(self, data): one_element = np.zeros( shape=1, dtype=object if unsized_string_dtype else self.dtype ) - self.set_element(data, one_element, 0, self.fill) + self.set_element(data.draw(self.fill), one_element, 0, fill=True) if unsized_string_dtype: one_element = one_element.astype(self.dtype) fill_value = one_element[0] @@ -319,7 +341,7 @@ def do_draw(self, data): if mismatch.any(): raise InvalidArgument( "Array elements %r cannot be represented as dtype %r - instead " - "they becomes %r. Use a more precise strategy, e.g. without " + "they become %r. Use a more precise strategy, e.g. without " "trailing null bytes, as this will be an error future versions." % (result[mismatch], self.dtype, out[mismatch]) ) @@ -340,16 +362,16 @@ def fill_for(elements, unique, fill, name=""): else: fill = elements else: - st.check_strategy(fill, f"{name}.fill" if name else "fill") + check_strategy(fill, f"{name}.fill" if name else "fill") return fill -@st.defines_strategy(force_reusable_values=True) +@defines_strategy(force_reusable_values=True) def arrays( dtype: Any, - shape: Union[int, Shape, st.SearchStrategy[Shape]], + shape: Union[int, st.SearchStrategy[int], Shape, st.SearchStrategy[Shape]], *, - elements: Optional[Union[SearchStrategy, Mapping[str, Any]]] = None, + elements: Optional[Union[st.SearchStrategy, Mapping[str, Any]]] = None, fill: Optional[st.SearchStrategy[Any]] = None, unique: bool = False, ) -> st.SearchStrategy[np.ndarray]: @@ -362,7 +384,7 @@ def arrays( strategy that generates such values. * ``elements`` is a strategy for generating values to put in the array. If it is None a suitable value will be inferred based on the dtype, - which may give any legal value (including eg ``NaN`` for floats). + which may give any legal value (including eg NaN for floats). If a mapping, it will be passed as ``**kwargs`` to ``from_dtype()`` * ``fill`` is a strategy that may be used to generate a single background value for the array. If None, a suitable default will be inferred @@ -374,8 +396,9 @@ def arrays( distinct from one another. Note that in this case multiple NaN values may still be allowed. If fill is also set, the only valid values for it to return are NaN values (anything for which :obj:`numpy:numpy.isnan` - returns True. So e.g. for complex numbers (nan+1j) is also a valid fill). - Note that if unique is set to True the generated values must be hashable. + returns True. So e.g. for complex numbers ``nan+1j`` is also a valid fill). + Note that if ``unique`` is set to ``True`` the generated values must be + hashable. Arrays of specified ``dtype`` and ``shape`` are generated for example like this: @@ -383,17 +406,11 @@ def arrays( .. code-block:: pycon >>> import numpy as np + >>> from hypothesis import strategies as st >>> arrays(np.int8, (2, 3)).example() array([[-8, 6, 3], [-6, 4, 6]], dtype=int8) - - - See :doc:`What you can generate and how `. - - .. code-block:: pycon - - >>> import numpy as np - >>> from hypothesis.strategies import floats - >>> arrays(np.float, 3, elements=floats(0, 1)).example() + >>> arrays(np.float, 3, elements=st.floats(0, 1)).example() array([ 0.88974794, 0.77387938, 0.1977879 ]) Array values are generated in two parts: @@ -401,14 +418,14 @@ def arrays( 1. Some subset of the coordinates of the array are populated with a value drawn from the elements strategy (or its inferred form). 2. If any coordinates were not assigned in the previous step, a single - value is drawn from the fill strategy and is assigned to all remaining + value is drawn from the ``fill`` strategy and is assigned to all remaining places. - You can set fill to :func:`~hypothesis.strategies.nothing` if you want to + You can set :func:`fill=nothing() ` to disable this behaviour and draw a value for every element. - If fill is set to None then it will attempt to infer the correct behaviour - automatically: If unique is True, no filling will occur by default. + If ``fill=None``, then it will attempt to infer the correct behaviour + automatically. If ``unique`` is ``True``, no filling will occur by default. Otherwise, if it looks safe to reuse the values of elements across multiple coordinates (this will be the case for any inferred strategy, and for most of the builtins, but is not the case for mutable values or @@ -426,11 +443,11 @@ def arrays( # strategy (i.e. repeated argument handling and validation) when it's not # needed. So we get the best of both worlds by recursing with flatmap, # but only when it's actually needed. - if isinstance(dtype, SearchStrategy): + if isinstance(dtype, st.SearchStrategy): return dtype.flatmap( lambda d: arrays(d, shape, elements=elements, fill=fill, unique=unique) ) - if isinstance(shape, SearchStrategy): + if isinstance(shape, st.SearchStrategy): return shape.flatmap( lambda s: arrays(dtype, s, elements=elements, fill=fill, unique=unique) ) @@ -461,42 +478,7 @@ def arrays( return ArrayStrategy(elements, shape, dtype, fill, unique) -@st.defines_strategy() -def array_shapes( - *, - min_dims: int = 1, - max_dims: Optional[int] = None, - min_side: int = 1, - max_side: Optional[int] = None, -) -> st.SearchStrategy[Shape]: - """Return a strategy for array shapes (tuples of int >= 1).""" - check_type(int, min_dims, "min_dims") - check_type(int, min_side, "min_side") - if min_dims > 32: - raise InvalidArgument( - "Got min_dims=%r, but numpy does not support arrays greater than 32 dimensions" - % min_dims - ) - if max_dims is None: - max_dims = min(min_dims + 2, 32) - check_type(int, max_dims, "max_dims") - if max_dims > 32: - raise InvalidArgument( - "Got max_dims=%r, but numpy does not support arrays greater than 32 dimensions" - % max_dims - ) - if max_side is None: - max_side = min_side + 5 - check_type(int, max_side, "max_side") - order_check("dims", 0, min_dims, max_dims) - order_check("side", 0, min_side, max_side) - - return st.lists( - st.integers(min_side, max_side), min_size=min_dims, max_size=max_dims - ).map(tuple) - - -@st.defines_strategy() +@defines_strategy() def scalar_dtypes() -> st.SearchStrategy[np.dtype]: """Return a strategy that can return any non-flexible scalar dtype.""" return st.one_of( @@ -511,7 +493,7 @@ def scalar_dtypes() -> st.SearchStrategy[np.dtype]: def defines_dtype_strategy(strat: T) -> T: - @st.defines_strategy() + @defines_strategy() @proxies(strat) def inner(*args, **kwargs): return strat(*args, **kwargs).map(np.dtype) @@ -604,7 +586,7 @@ def complex_number_dtypes( """Return a strategy for complex-number dtypes. sizes is the total size in bits of a complex number, which consists - of two floats. Complex halfs (a 16-bit real part) are not supported + of two floats. Complex halves (a 16-bit real part) are not supported by numpy and will not be generated by this strategy. """ return dtype_factory("c", sizes, (64, 128, 192, 256), endianness) @@ -742,7 +724,7 @@ def array_dtypes( ).filter(_no_title_is_name_of_a_titled_field) -@st.defines_strategy() +@defines_strategy() def nested_dtypes( subtype_strategy: st.SearchStrategy[np.dtype] = scalar_dtypes(), *, @@ -764,434 +746,38 @@ def nested_dtypes( ).filter(lambda d: max_itemsize is None or d.itemsize <= max_itemsize) -@st.defines_strategy() -def valid_tuple_axes( - ndim: int, - *, - min_size: int = 0, - max_size: Optional[int] = None, -) -> st.SearchStrategy[Shape]: - """Return a strategy for generating permissible tuple-values for the +@proxies(_valid_tuple_axes) +def valid_tuple_axes(*args, **kwargs): + return _valid_tuple_axes(*args, **kwargs) + + +valid_tuple_axes.__doc__ = f""" + Return a strategy for generating permissible tuple-values for the ``axis`` argument for a numpy sequential function (e.g. :func:`numpy:numpy.sum`), given an array of the specified dimensionality. - All tuples will have an length >= min_size and <= max_size. The default - value for max_size is ``ndim``. - - Examples from this strategy shrink towards an empty tuple, which render - most sequential functions as no-ops. - - The following are some examples drawn from this strategy. - - .. code-block:: pycon - - >>> [valid_tuple_axes(3).example() for i in range(4)] - [(-3, 1), (0, 1, -1), (0, 2), (0, -2, 2)] - - ``valid_tuple_axes`` can be joined with other strategies to generate - any type of valid axis object, i.e. integers, tuples, and ``None``: - - .. code-block:: pycon - - any_axis_strategy = none() | integers(-ndim, ndim - 1) | valid_tuple_axes(ndim) - - """ - if max_size is None: - max_size = ndim - - check_type(int, ndim, "ndim") - check_type(int, min_size, "min_size") - check_type(int, max_size, "max_size") - order_check("size", 0, min_size, max_size) - check_valid_interval(max_size, ndim, "max_size", "ndim") - - # shrink axis values from negative to positive - axes = st.integers(0, max(0, 2 * ndim - 1)).map( - lambda x: x if x < ndim else x - 2 * ndim - ) - return st.lists( - axes, min_size=min_size, max_size=max_size, unique_by=lambda x: x % ndim - ).map(tuple) - - -@st.defines_strategy() -def broadcastable_shapes( - shape: Shape, - *, - min_dims: int = 0, - max_dims: Optional[int] = None, - min_side: int = 1, - max_side: Optional[int] = None, -) -> st.SearchStrategy[Shape]: - """Return a strategy for generating shapes that are broadcast-compatible - with the provided shape. - - Examples from this strategy shrink towards a shape with length ``min_dims``. - The size of an aligned dimension shrinks towards size ``1``. The - size of an unaligned dimension shrink towards ``min_side``. - - * ``shape`` a tuple of integers - * ``min_dims`` The smallest length that the generated shape can possess. - * ``max_dims`` The largest length that the generated shape can possess. - The default-value for ``max_dims`` is ``min(32, max(len(shape), min_dims) + 2)``. - * ``min_side`` The smallest size that an unaligned dimension can possess. - * ``max_side`` The largest size that an unaligned dimension can possess. - The default value is 2 + 'size-of-largest-aligned-dimension'. - - The following are some examples drawn from this strategy. - - .. code-block:: pycon - - >>> [broadcastable_shapes(shape=(2, 3)).example() for i in range(5)] - [(1, 3), (), (2, 3), (2, 1), (4, 1, 3), (3, )] - + {_valid_tuple_axes.__doc__} """ - check_type(tuple, shape, "shape") - strict_check = max_side is None or max_dims is None - check_type(int, min_side, "min_side") - check_type(int, min_dims, "min_dims") - - if max_dims is None: - max_dims = min(32, max(len(shape), min_dims) + 2) - else: - check_type(int, max_dims, "max_dims") - - if max_side is None: - max_side = max(tuple(shape[-max_dims:]) + (min_side,)) + 2 - else: - check_type(int, max_side, "max_side") - - order_check("dims", 0, min_dims, max_dims) - order_check("side", 0, min_side, max_side) - - if 32 < max_dims: - raise InvalidArgument("max_dims cannot exceed 32") - - dims, bnd_name = (max_dims, "max_dims") if strict_check else (min_dims, "min_dims") - - # check for unsatisfiable min_side - if not all(min_side <= s for s in shape[::-1][:dims] if s != 1): - raise InvalidArgument( - "Given shape=%r, there are no broadcast-compatible " - "shapes that satisfy: %s=%s and min_side=%s" - % (shape, bnd_name, dims, min_side) - ) - - # check for unsatisfiable [min_side, max_side] - if not ( - min_side <= 1 <= max_side or all(s <= max_side for s in shape[::-1][:dims]) - ): - raise InvalidArgument( - "Given shape=%r, there are no broadcast-compatible shapes " - "that satisfy: %s=%s and [min_side=%s, max_side=%s]" - % (shape, bnd_name, dims, min_side, max_side) - ) - - if not strict_check: - # reduce max_dims to exclude unsatisfiable dimensions - for n, s in zip(range(max_dims), reversed(shape)): - if s < min_side and s != 1: - max_dims = n - break - elif not (min_side <= 1 <= max_side or s <= max_side): - max_dims = n - break - - return MutuallyBroadcastableShapesStrategy( - num_shapes=1, - base_shape=shape, - min_dims=min_dims, - max_dims=max_dims, - min_side=min_side, - max_side=max_side, - ).map(lambda x: x.input_shapes[0]) - - -class MutuallyBroadcastableShapesStrategy(SearchStrategy): - def __init__( - self, - num_shapes, - signature=None, - base_shape=(), - min_dims=0, - max_dims=None, - min_side=1, - max_side=None, - ): - assert 0 <= min_side <= max_side - assert 0 <= min_dims <= max_dims <= 32 - SearchStrategy.__init__(self) - self.base_shape = base_shape - self.side_strat = st.integers(min_side, max_side) - self.num_shapes = num_shapes - self.signature = signature - self.min_dims = min_dims - self.max_dims = max_dims - self.min_side = min_side - self.max_side = max_side - - self.size_one_allowed = self.min_side <= 1 <= self.max_side - - def do_draw(self, data): - # We don't usually have a gufunc signature; do the common case first & fast. - if self.signature is None: - return self._draw_loop_dimensions(data) - - # When we *do*, draw the core dims, then draw loop dims, and finally combine. - core_in, core_res = self._draw_core_dimensions(data) - - # If some core shape has omitted optional dimensions, it's an error to add - # loop dimensions to it. We never omit core dims if min_dims >= 1. - # This ensures that we respect Numpy's gufunc broadcasting semantics and user - # constraints without needing to check whether the loop dims will be - # interpreted as an invalid substitute for the omitted core dims. - # We may implement this check later! - use = [None not in shp for shp in core_in] - loop_in, loop_res = self._draw_loop_dimensions(data, use=use) - - def add_shape(loop, core): - return tuple(x for x in (loop + core)[-32:] if x is not None) - - return BroadcastableShapes( - input_shapes=tuple(add_shape(l, c) for l, c in zip(loop_in, core_in)), - result_shape=add_shape(loop_res, core_res), - ) - - def _draw_core_dimensions(self, data): - # Draw gufunc core dimensions, with None standing for optional dimensions - # that will not be present in the final shape. We track omitted dims so - # that we can do an accurate per-shape length cap. - dims = {} - shapes = [] - for shape in self.signature.input_shapes + (self.signature.result_shape,): - shapes.append([]) - for name in shape: - if name.isdigit(): - shapes[-1].append(int(name)) - continue - if name not in dims: - dim = name.strip("?") - dims[dim] = data.draw(self.side_strat) - if self.min_dims == 0 and not data.draw_bits(3): - dims[dim + "?"] = None - else: - dims[dim + "?"] = dims[dim] - shapes[-1].append(dims[name]) - return tuple(tuple(s) for s in shapes[:-1]), tuple(shapes[-1]) - - def _draw_loop_dimensions(self, data, use=None): - # All shapes are handled in column-major order; i.e. they are reversed - base_shape = self.base_shape[::-1] - result_shape = list(base_shape) - shapes = [[] for _ in range(self.num_shapes)] - if use is None: - use = [True for _ in range(self.num_shapes)] - else: - assert len(use) == self.num_shapes - assert all(isinstance(x, bool) for x in use) - - for dim_count in range(1, self.max_dims + 1): - dim = dim_count - 1 - - # We begin by drawing a valid dimension-size for the given - # dimension. This restricts the variability across the shapes - # at this dimension such that they can only choose between - # this size and a singleton dimension. - if len(base_shape) < dim_count or base_shape[dim] == 1: - # dim is unrestricted by the base-shape: shrink to min_side - dim_side = data.draw(self.side_strat) - elif base_shape[dim] <= self.max_side: - # dim is aligned with non-singleton base-dim - dim_side = base_shape[dim] - else: - # only a singleton is valid in alignment with the base-dim - dim_side = 1 - - for shape_id, shape in enumerate(shapes): - # Populating this dimension-size for each shape, either - # the drawn size is used or, if permitted, a singleton - # dimension. - if dim_count <= len(base_shape) and self.size_one_allowed: - # aligned: shrink towards size 1 - side = data.draw(st.sampled_from([1, dim_side])) - else: - side = dim_side - - # Use a trick where where a biased coin is queried to see - # if the given shape-tuple will continue to be grown. All - # of the relevant draws will still be made for the given - # shape-tuple even if it is no longer being added to. - # This helps to ensure more stable shrinking behavior. - if self.min_dims < dim_count: - use[shape_id] &= cu.biased_coin( - data, 1 - 1 / (1 + self.max_dims - dim) - ) - - if use[shape_id]: - shape.append(side) - if len(result_shape) < len(shape): - result_shape.append(shape[-1]) - elif shape[-1] != 1 and result_shape[dim] == 1: - result_shape[dim] = shape[-1] - if not any(use): - break - - result_shape = result_shape[: max(map(len, [self.base_shape] + shapes))] - - assert len(shapes) == self.num_shapes - assert all(self.min_dims <= len(s) <= self.max_dims for s in shapes) - assert all(self.min_side <= s <= self.max_side for side in shapes for s in side) - - return BroadcastableShapes( - input_shapes=tuple(tuple(reversed(shape)) for shape in shapes), - result_shape=tuple(reversed(result_shape)), - ) - - -# See https://docs.scipy.org/doc/numpy/reference/c-api.generalized-ufuncs.html -# Implementation based on numpy.lib.function_base._parse_gufunc_signature -# with minor upgrades to handle numeric and optional dimensions. Examples: -# -# add (),()->() binary ufunc -# sum1d (i)->() reduction -# inner1d (i),(i)->() vector-vector multiplication -# matmat (m,n),(n,p)->(m,p) matrix multiplication -# vecmat (n),(n,p)->(p) vector-matrix multiplication -# matvec (m,n),(n)->(m) matrix-vector multiplication -# matmul (m?,n),(n,p?)->(m?,p?) combination of the four above -# cross1d (3),(3)->(3) cross product with frozen dimensions -# -# Note that while no examples of such usage are given, Numpy does allow -# generalised ufuncs that have *multiple output arrays*. This is not -# currently supported by Hypothesis - please contact us if you would use it! -# -# We are unsure if gufuncs allow frozen dimensions to be optional, but it's -# easy enough to support here - and so we will unless we learn otherwise. -# -_DIMENSION = r"\w+\??" # Note that \w permits digits too! -_SHAPE = r"\((?:{0}(?:,{0})".format(_DIMENSION) + r"{0,31})?\)" -_ARGUMENT_LIST = "{0}(?:,{0})*".format(_SHAPE) -_SIGNATURE = fr"^{_ARGUMENT_LIST}->{_SHAPE}$" -_SIGNATURE_MULTIPLE_OUTPUT = r"^{0}->{0}$".format(_ARGUMENT_LIST) - -class _GUfuncSig(NamedTuple): - input_shapes: Tuple[Shape, ...] - result_shape: Shape +@proxies(_mutually_broadcastable_shapes) +def mutually_broadcastable_shapes(*args, **kwargs): + return _mutually_broadcastable_shapes(*args, **kwargs) -def _hypothesis_parse_gufunc_signature(signature, all_checks=True): - # Disable all_checks to better match the Numpy version, for testing - if not re.match(_SIGNATURE, signature): - if re.match(_SIGNATURE_MULTIPLE_OUTPUT, signature): - raise InvalidArgument( - "Hypothesis does not yet support generalised ufunc signatures " - "with multiple output arrays - mostly because we don't know of " - "anyone who uses them! Please get in touch with us to fix that." - f"\n (signature={signature!r})" - ) - if re.match(np.lib.function_base._SIGNATURE, signature): - raise InvalidArgument( - f"signature={signature!r} matches Numpy's regex for gufunc signatures, " - "but contains shapes with more than 32 dimensions and is thus invalid." - ) - raise InvalidArgument(f"{signature!r} is not a valid gufunc signature") - input_shapes, output_shapes = ( - tuple(tuple(re.findall(_DIMENSION, a)) for a in re.findall(_SHAPE, arg_list)) - for arg_list in signature.split("->") - ) - assert len(output_shapes) == 1 - result_shape = output_shapes[0] - if all_checks: - # Check that there are no names in output shape that do not appear in inputs. - # (kept out of parser function for easier generation of test values) - # We also disallow frozen optional dimensions - this is ambiguous as there is - # no way to share an un-named dimension between shapes. Maybe just padding? - # Anyway, we disallow it pending clarification from upstream. - frozen_optional_err = ( - "Got dimension %r, but handling of frozen optional dimensions " - "is ambiguous. If you known how this should work, please " - "contact us to get this fixed and documented (signature=%r)." - ) - only_out_err = ( - "The %r dimension only appears in the output shape, and is " - "not frozen, so the size is not determined (signature=%r)." - ) - names_in = {n.strip("?") for shp in input_shapes for n in shp} - names_out = {n.strip("?") for n in result_shape} - for shape in input_shapes + (result_shape,): - for name in shape: - try: - int(name.strip("?")) - if "?" in name: - raise InvalidArgument(frozen_optional_err % (name, signature)) - except ValueError: - if name.strip("?") in (names_out - names_in): - raise InvalidArgument( - only_out_err % (name, signature) - ) from None - return _GUfuncSig(input_shapes=input_shapes, result_shape=result_shape) - -@st.defines_strategy() -def mutually_broadcastable_shapes( - *, - num_shapes: Union[UniqueIdentifier, int] = not_set, - signature: Union[UniqueIdentifier, str] = not_set, - base_shape: Shape = (), - min_dims: int = 0, - max_dims: Optional[int] = None, - min_side: int = 1, - max_side: Optional[int] = None, -) -> st.SearchStrategy[BroadcastableShapes]: - """Return a strategy for generating a specified number of shapes, N, that are - mutually-broadcastable with one another and with the provided "base-shape". - - The strategy will generate a named-tuple of: - - * input_shapes: the N generated shapes - * result_shape: the resulting shape, produced by broadcasting the - N shapes with the base-shape - - Each shape produced from this strategy shrinks towards a shape with length - ``min_dims``. The size of an aligned dimension shrinks towards being having - a size of 1. The size of an unaligned dimension shrink towards ``min_side``. - - * ``num_shapes`` The number of mutually broadcast-compatible shapes to generate. - * ``base-shape`` The shape against which all generated shapes can broadcast. - The default shape is empty, which corresponds to a scalar and thus does not - constrain broadcasting at all. - * ``min_dims`` The smallest length that any generated shape can possess. - * ``max_dims`` The largest length that any generated shape can possess. - It cannot exceed 32, which is the greatest supported dimensionality for a - numpy array. The default-value for ``max_dims`` is - ``2 + max(len(shape), min_dims)``, capped at 32. - * ``min_side`` The smallest size that an unaligned dimension can possess. - * ``max_side`` The largest size that an unaligned dimension can possess. - The default value is 2 + 'size-of-largest-aligned-dimension'. - - The following are some examples drawn from this strategy. - - .. code-block:: pycon - - >>> # Draw three shapes, and each shape is broadcast-compatible with `(2, 3)` - >>> for _ in range(5): - ... mutually_broadcastable_shapes(num_shapes=3, base_shape=(2, 3)).example() - BroadcastableShapes(input_shapes=((4, 1, 3), (4, 2, 3), ()), result_shape=(4, 2, 3)) - BroadcastableShapes(input_shapes=((3,), (1,), (2, 1)), result_shape=(2, 3)) - BroadcastableShapes(input_shapes=((3,), (1, 3), (2, 3)), result_shape=(2, 3)) - BroadcastableShapes(input_shapes=((), (), ()), result_shape=(2, 3)) - BroadcastableShapes(input_shapes=((3,), (), (3,)), result_shape=(2, 3)) +mutually_broadcastable_shapes.__doc__ = f""" + {_mutually_broadcastable_shapes.__doc__} **Use with Generalised Universal Function signatures** - A :np-ref:`universal function ` (or ufunc for short) is a function + A :doc:`universal function ` (or ufunc for short) is a function that operates on ndarrays in an element-by-element fashion, supporting array broadcasting, type casting, and several other standard features. - A :np-ref:`generalised ufunc ` operates on + A :doc:`generalised ufunc ` operates on sub-arrays rather than elements, based on the "signature" of the function. - Compare e.g. :obj:`numpy:numpy.add` (ufunc) to :obj:`numpy:numpy.matmul` (gufunc). + Compare e.g. :obj:`numpy.add() ` (ufunc) to + :obj:`numpy.matmul() ` (gufunc). To generate shapes for a gufunc, you can pass the ``signature`` argument instead of ``num_shapes``. This must be a gufunc signature string; which you can write by @@ -1217,150 +803,11 @@ def mutually_broadcastable_shapes( BroadcastableShapes(input_shapes=((2,), (2,)), result_shape=()) BroadcastableShapes(input_shapes=((3, 4, 2), (1, 2)), result_shape=(3, 4)) BroadcastableShapes(input_shapes=((4, 2), (1, 2, 3)), result_shape=(4, 3)) - """ - arg_msg = "Pass either the `num_shapes` or the `signature` argument, but not both." - if num_shapes is not not_set: - check_argument(signature is not_set, arg_msg) - check_type(int, num_shapes, "num_shapes") - assert isinstance(num_shapes, int) # for mypy - check_argument(num_shapes >= 1, "num_shapes={} must be at least 1", num_shapes) - parsed_signature = None - sig_dims = 0 - else: - check_argument(signature is not not_set, arg_msg) - if signature is None: - raise InvalidArgument( - "Expected a string, but got invalid signature=None. " - "(maybe .signature attribute of an element-wise ufunc?)" - ) - check_type(str, signature, "signature") - parsed_signature = _hypothesis_parse_gufunc_signature(signature) - sig_dims = min( - map(len, parsed_signature.input_shapes + (parsed_signature.result_shape,)) - ) - num_shapes = len(parsed_signature.input_shapes) - assert num_shapes >= 1 - - check_type(tuple, base_shape, "base_shape") - strict_check = max_dims is not None - check_type(int, min_side, "min_side") - check_type(int, min_dims, "min_dims") - - if max_dims is None: - max_dims = min(32 - sig_dims, max(len(base_shape), min_dims) + 2) - else: - check_type(int, max_dims, "max_dims") - - if max_side is None: - max_side = max(tuple(base_shape[-max_dims:]) + (min_side,)) + 2 - else: - check_type(int, max_side, "max_side") - - order_check("dims", 0, min_dims, max_dims) - order_check("side", 0, min_side, max_side) - - if 32 - sig_dims < max_dims: - if sig_dims == 0: - raise InvalidArgument("max_dims cannot exceed 32") - raise InvalidArgument( - f"max_dims={signature!r} would exceed the 32-dimension limit given " - f"signature={parsed_signature!r}" - ) - - dims, bnd_name = (max_dims, "max_dims") if strict_check else (min_dims, "min_dims") - - # check for unsatisfiable min_side - if not all(min_side <= s for s in base_shape[::-1][:dims] if s != 1): - raise InvalidArgument( - "Given base_shape=%r, there are no broadcast-compatible " - "shapes that satisfy: %s=%s and min_side=%s" - % (base_shape, bnd_name, dims, min_side) - ) - - # check for unsatisfiable [min_side, max_side] - if not ( - min_side <= 1 <= max_side or all(s <= max_side for s in base_shape[::-1][:dims]) - ): - raise InvalidArgument( - "Given base_shape=%r, there are no broadcast-compatible shapes " - "that satisfy all of %s=%s, min_side=%s, and max_side=%s" - % (base_shape, bnd_name, dims, min_side, max_side) - ) - - if not strict_check: - # reduce max_dims to exclude unsatisfiable dimensions - for n, s in zip(range(max_dims), reversed(base_shape)): - if s < min_side and s != 1: - max_dims = n - break - elif not (min_side <= 1 <= max_side or s <= max_side): - max_dims = n - break - - return MutuallyBroadcastableShapesStrategy( - num_shapes=num_shapes, - signature=parsed_signature, - base_shape=base_shape, - min_dims=min_dims, - max_dims=max_dims, - min_side=min_side, - max_side=max_side, - ) - - -class BasicIndexStrategy(SearchStrategy): - def __init__(self, shape, min_dims, max_dims, allow_ellipsis, allow_newaxis): - assert 0 <= min_dims <= max_dims <= 32 - SearchStrategy.__init__(self) - self.shape = shape - self.min_dims = min_dims - self.max_dims = max_dims - self.allow_ellipsis = allow_ellipsis - self.allow_newaxis = allow_newaxis - def do_draw(self, data): - # General plan: determine the actual selection up front with a straightforward - # approach that shrinks well, then complicate it by inserting other things. - result = [] - for dim_size in self.shape: - if dim_size == 0: - result.append(slice(None)) - continue - strategy = st.integers(-dim_size, dim_size - 1) | st.slices(dim_size) - result.append(data.draw(strategy)) - # Insert some number of new size-one dimensions if allowed - result_dims = sum(isinstance(idx, slice) for idx in result) - while ( - self.allow_newaxis - and result_dims < self.max_dims - and (result_dims < self.min_dims or data.draw(st.booleans())) - ): - result.insert(data.draw(st.integers(0, len(result))), np.newaxis) - result_dims += 1 - # Check that we'll have the right number of dimensions; reject if not. - # It's easy to do this by construction iff you don't care about shrinking, - # which is really important for array shapes. So we filter instead. - assume(self.min_dims <= result_dims <= self.max_dims) - # This is a quick-and-dirty way to insert ..., xor shorten the indexer, - # but it means we don't have to do any structural analysis. - if self.allow_ellipsis and data.draw(st.booleans()): - # Choose an index; then replace all adjacent whole-dimension slices. - i = j = data.draw(st.integers(0, len(result))) - while i > 0 and result[i - 1] == slice(None): - i -= 1 - while j < len(result) and result[j] == slice(None): - j += 1 - result[i:j] = [Ellipsis] - else: - while result[-1:] == [slice(None, None)] and data.draw(st.integers(0, 7)): - result.pop() - if len(result) == 1 and data.draw(st.booleans()): - # Sometimes generate bare element equivalent to a length-one tuple - return result[0] - return tuple(result) + """ -@st.defines_strategy() +@defines_strategy() def basic_indices( shape: Shape, *, @@ -1369,67 +816,83 @@ def basic_indices( allow_newaxis: bool = False, allow_ellipsis: bool = True, ) -> st.SearchStrategy[BasicIndex]: - """ - The ``basic_indices`` strategy generates `basic indexes - `__ for - arrays of the specified shape, which may include dimensions of size zero. - - It generates tuples containing some mix of integers, :obj:`python:slice` objects, - ``...`` (Ellipsis), and :obj:`numpy:numpy.newaxis`; which when used to index a - ``shape``-shaped array will produce either a scalar or a shared-memory view. - When a length-one tuple would be generated, this strategy may instead return - the element which will index the first axis, e.g. ``5`` instead of ``(5,)``. - - * ``shape``: the array shape that will be indexed, as a tuple of integers >= 0. - This must be at least two-dimensional for a tuple to be a valid basic index; - for one-dimensional arrays use :func:`~hypothesis.strategies.slices` instead. - * ``min_dims``: the minimum dimensionality of the resulting view from use of - the generated index. When ``min_dims == 0``, scalars and zero-dimensional + """Return a strategy for :doc:`basic indexes ` of + arrays with the specified shape, which may include dimensions of size zero. + + It generates tuples containing some mix of integers, :obj:`python:slice` + objects, ``...`` (an ``Ellipsis``), and ``None``. When a length-one tuple + would be generated, this strategy may instead return the element which will + index the first axis, e.g. ``5`` instead of ``(5,)``. + + * ``shape`` is the shape of the array that will be indexed, as a tuple of + positive integers. This must be at least two-dimensional for a tuple to be + a valid index; for one-dimensional arrays use + :func:`~hypothesis.strategies.slices` instead. + * ``min_dims`` is the minimum dimensionality of the resulting array from use + of the generated index. When ``min_dims == 0``, scalars and zero-dimensional arrays are both allowed. - * ``max_dims``: the maximum dimensionality of the resulting view. - If not specified, it defaults to ``max(len(shape), min_dims) + 2``. - * ``allow_ellipsis``: whether ``...``` is allowed in the index. - * ``allow_newaxis``: whether :obj:`numpy:numpy.newaxis` is allowed in the index. - - Note that the length of the generated tuple may be anywhere between zero - and ``min_dims``. It may not match the length of ``shape``, or even the - dimensionality of the array view resulting from its use! + * ``max_dims`` is the the maximum dimensionality of the resulting array, + defaulting to ``len(shape) if not allow_newaxis else + max(len(shape), min_dims) + 2``. + * ``allow_newaxis`` specifies whether ``None`` is allowed in the index. + * ``allow_ellipsis`` specifies whether ``...`` is allowed in the index. """ # Arguments to exclude scalars, zero-dim arrays, and dims of size zero were # all considered and rejected. We want users to explicitly consider those # cases if they're dealing in general indexers, and while it's fiddly we can # back-compatibly add them later (hence using kwonlyargs). check_type(tuple, shape, "shape") + check_argument( + all(isinstance(x, int) and x >= 0 for x in shape), + f"shape={shape!r}, but all dimensions must be non-negative integers.", + ) check_type(bool, allow_ellipsis, "allow_ellipsis") check_type(bool, allow_newaxis, "allow_newaxis") check_type(int, min_dims, "min_dims") + if min_dims > len(shape) and not allow_newaxis: + note_deprecation( + f"min_dims={min_dims} is larger than len(shape)={len(shape)}, " + "but allow_newaxis=False makes it impossible for an indexing " + "operation to add dimensions.", + since="2021-09-15", + has_codemod=False, + ) + check_valid_dims(min_dims, "min_dims") + if max_dims is None: - max_dims = min(max(len(shape), min_dims) + 2, 32) + if allow_newaxis: + max_dims = min(max(len(shape), min_dims) + 2, NDIM_MAX) + else: + max_dims = min(len(shape), NDIM_MAX) else: check_type(int, max_dims, "max_dims") + if max_dims > len(shape) and not allow_newaxis: + note_deprecation( + f"max_dims={max_dims} is larger than len(shape)={len(shape)}, " + "but allow_newaxis=False makes it impossible for an indexing " + "operation to add dimensions.", + since="2021-09-15", + has_codemod=False, + ) + check_valid_dims(max_dims, "max_dims") + order_check("dims", 0, min_dims, max_dims) - check_argument( - max_dims <= 32, - f"max_dims={max_dims!r}, but numpy arrays have at most 32 dimensions", - ) - check_argument( - all(isinstance(x, int) and x >= 0 for x in shape), - f"shape={shape!r}, but all dimensions must be of integer size >= 0", - ) + return BasicIndexStrategy( shape, min_dims=min_dims, max_dims=max_dims, allow_ellipsis=allow_ellipsis, allow_newaxis=allow_newaxis, + allow_fewer_indices_than_dims=True, ) -@st.defines_strategy() +@defines_strategy() def integer_array_indices( shape: Shape, *, - result_shape: SearchStrategy[Shape] = array_shapes(), + result_shape: st.SearchStrategy[Shape] = array_shapes(), dtype: np.dtype = "int", ) -> st.SearchStrategy[Tuple[np.ndarray, ...]]: """Return a search strategy for tuples of integer-arrays that, when used @@ -1490,5 +953,5 @@ def array_for(index_shape, size): ) return result_shape.flatmap( - lambda index_shape: st.tuples(*[array_for(index_shape, size) for size in shape]) + lambda index_shape: st.tuples(*(array_for(index_shape, size) for size in shape)) ) diff --git a/hypothesis-python/src/hypothesis/extra/pandas/__init__.py b/hypothesis-python/src/hypothesis/extra/pandas/__init__.py index ea4443692d..2fd9c627a1 100644 --- a/hypothesis-python/src/hypothesis/extra/pandas/__init__.py +++ b/hypothesis-python/src/hypothesis/extra/pandas/__init__.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis.extra.pandas.impl import ( column, diff --git a/hypothesis-python/src/hypothesis/extra/pandas/impl.py b/hypothesis-python/src/hypothesis/extra/pandas/impl.py index 29e6668af0..a1220d6b3f 100644 --- a/hypothesis-python/src/hypothesis/extra/pandas/impl.py +++ b/hypothesis-python/src/hypothesis/extra/pandas/impl.py @@ -1,39 +1,38 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from collections import OrderedDict, abc from copy import copy +from datetime import datetime, timedelta from typing import Any, List, Optional, Sequence, Set, Union import attr import numpy as np import pandas +from hypothesis import strategies as st +from hypothesis._settings import note_deprecation from hypothesis.control import reject from hypothesis.errors import InvalidArgument from hypothesis.extra import numpy as npst from hypothesis.internal.conjecture import utils as cu from hypothesis.internal.coverage import check, check_function +from hypothesis.internal.reflection import get_pretty_function_description from hypothesis.internal.validation import ( check_type, check_valid_interval, check_valid_size, try_convert, ) -from hypothesis.strategies._internal import core as st -from hypothesis.strategies._internal.strategies import Ex +from hypothesis.strategies._internal.strategies import Ex, check_strategy +from hypothesis.strategies._internal.utils import cacheable, defines_strategy try: from pandas.api.types import is_categorical_dtype @@ -67,7 +66,7 @@ def elements_and_dtype(elements, dtype, source=None): prefix = f"{source}." if elements is not None: - st.check_strategy(elements, f"{prefix}elements") + check_strategy(elements, f"{prefix}elements") else: with check("dtype is not None"): if dtype is None: @@ -81,6 +80,19 @@ def elements_and_dtype(elements, dtype, source=None): f"{prefix}dtype is categorical, which is currently unsupported" ) + if isinstance(dtype, type) and np.dtype(dtype).kind == "O" and dtype is not object: + note_deprecation( + f"Passed dtype={dtype!r} is not a valid Pandas dtype. We'll treat it as " + "dtype=object for now, but this will be an error in a future version.", + since="2021-12-31", + has_codemod=False, + ) + + if isinstance(dtype, st.SearchStrategy): + raise InvalidArgument( + f"Passed dtype={dtype!r} is a strategy, but we require a concrete dtype " + "here. See https://stackoverflow.com/q/74355937 for workaround patterns." + ) dtype = try_convert(np.dtype, dtype, "dtype") if elements is None: @@ -95,11 +107,11 @@ def convert_element(value): raise InvalidArgument( "Cannot convert %s=%r of type %s to dtype %s" % (name, value, type(value).__name__, dtype.str) - ) + ) from None except ValueError: raise InvalidArgument( f"Cannot convert {name}={value!r} to type {dtype.str}" - ) + ) from None elements = elements.map(convert_element) assert elements is not None @@ -108,13 +120,14 @@ def convert_element(value): class ValueIndexStrategy(st.SearchStrategy): - def __init__(self, elements, dtype, min_size, max_size, unique): + def __init__(self, elements, dtype, min_size, max_size, unique, name): super().__init__() self.elements = elements self.dtype = dtype self.min_size = min_size self.max_size = max_size self.unique = unique + self.name = name def do_draw(self, data): result = [] @@ -140,14 +153,16 @@ def do_draw(self, data): dtype = infer_dtype_if_necessary( dtype=self.dtype, values=result, elements=self.elements, draw=data.draw ) - return pandas.Index(result, dtype=dtype, tupleize_cols=False) + return pandas.Index( + result, dtype=dtype, tupleize_cols=False, name=data.draw(self.name) + ) DEFAULT_MAX_SIZE = 10 -@st.cacheable -@st.defines_strategy() +@cacheable +@defines_strategy() def range_indexes( min_size: int = 0, max_size: Optional[int] = None, @@ -164,13 +179,13 @@ def range_indexes( check_valid_size(min_size, "min_size") check_valid_size(max_size, "max_size") if max_size is None: - max_size = min([min_size + DEFAULT_MAX_SIZE, 2 ** 63 - 1]) + max_size = min([min_size + DEFAULT_MAX_SIZE, 2**63 - 1]) check_valid_interval(min_size, max_size, "min_size", "max_size") return st.integers(min_size, max_size).map(pandas.RangeIndex) -@st.cacheable -@st.defines_strategy() +@cacheable +@defines_strategy() def indexes( *, elements: Optional[st.SearchStrategy[Ex]] = None, @@ -178,6 +193,7 @@ def indexes( min_size: int = 0, max_size: Optional[int] = None, unique: bool = True, + name: st.SearchStrategy[Optional[str]] = st.none(), ) -> st.SearchStrategy[pandas.Index]: """Provides a strategy for producing a :class:`pandas.Index`. @@ -197,6 +213,8 @@ def indexes( should pass a max_size explicitly. * unique specifies whether all of the elements in the resulting index should be distinct. + * name is a strategy for strings or ``None``, which will be passed to + the :class:`pandas.Index` constructor. """ check_valid_size(min_size, "min_size") check_valid_size(max_size, "max_size") @@ -207,10 +225,10 @@ def indexes( if max_size is None: max_size = min_size + DEFAULT_MAX_SIZE - return ValueIndexStrategy(elements, dtype, min_size, max_size, unique) + return ValueIndexStrategy(elements, dtype, min_size, max_size, unique, name) -@st.defines_strategy() +@defines_strategy() def series( *, elements: Optional[st.SearchStrategy[Ex]] = None, @@ -218,6 +236,7 @@ def series( index: Optional[st.SearchStrategy[Union[Sequence, pandas.Index]]] = None, fill: Optional[st.SearchStrategy[Ex]] = None, unique: bool = False, + name: st.SearchStrategy[Optional[str]] = st.none(), ) -> st.SearchStrategy[pandas.Series]: """Provides a strategy for producing a :class:`pandas.Series`. @@ -244,6 +263,9 @@ def series( :func:`~hypothesis.extra.pandas.range_indexes` function to produce values for this argument. + * name: is a strategy for strings or ``None``, which will be passed to + the :class:`pandas.Series` constructor. + Usage: .. code-block:: pycon @@ -255,7 +277,7 @@ def series( if index is None: index = range_indexes() else: - st.check_strategy(index, "index") + check_strategy(index, "index") elements, dtype = elements_and_dtype(elements, dtype) index_strategy = index @@ -288,7 +310,7 @@ def result(draw): ) ) - return pandas.Series(result_data, index=index, dtype=dtype) + return pandas.Series(result_data, index=index, dtype=dtype, name=draw(name)) else: return pandas.Series( (), @@ -296,6 +318,7 @@ def result(draw): dtype=dtype if dtype is not None else draw(dtype_for_elements_strategy(elements)), + name=draw(name), ) return result() @@ -321,7 +344,7 @@ class column: name = attr.ib(default=None) elements = attr.ib(default=None) - dtype = attr.ib(default=None) + dtype = attr.ib(default=None, repr=get_pretty_function_description) fill = attr.ib(default=None) unique = attr.ib(default=False) @@ -344,7 +367,7 @@ def columns( create the columns. """ if isinstance(names_or_number, (int, float)): - names = [None] * names_or_number # type: list + names: List[Union[int, str, None]] = [None] * names_or_number else: names = list(names_or_number) return [ @@ -353,7 +376,7 @@ def columns( ] -@st.defines_strategy() +@defines_strategy() def data_frames( columns: Optional[Sequence[column]] = None, *, @@ -465,7 +488,7 @@ def data_frames( if index is None: index = range_indexes() else: - st.check_strategy(index, "index") + check_strategy(index, "index") index_strategy = index @@ -496,10 +519,10 @@ def row(): return rows_only() assert columns is not None - cols = try_convert(tuple, columns, "columns") # type: Sequence[column] + cols = try_convert(tuple, columns, "columns") rewritten_columns = [] - column_names = set() # type: Set[str] + column_names: Set[str] = set() for i, c in enumerate(cols): check_type(column, c, f"columns[{i}]") @@ -514,10 +537,9 @@ def row(): hash(c.name) except TypeError: raise InvalidArgument( - "Column names must be hashable, but columns[%d].name was " - "%r of type %s, which cannot be hashed." - % (i, c.name, type(c.name).__name__) - ) + f"Column names must be hashable, but columns[{i}].name was " + f"{c.name!r} of type {type(c.name).__name__}, which cannot be hashed." + ) from None if c.name in column_names: raise InvalidArgument(f"duplicate definition of column name {c.name!r}") @@ -528,8 +550,7 @@ def row(): if c.dtype is None and rows is not None: raise InvalidArgument( - "Must specify a dtype for all columns when combining rows with" - " columns." + "Must specify a dtype for all columns when combining rows with columns." ) c.fill = npst.fill_for( @@ -581,7 +602,21 @@ def just_draw_columns(draw): reject() else: value = draw(c.elements) - data[c.name][i] = value + try: + data[c.name][i] = value + except ValueError as err: # pragma: no cover + # This just works in Pandas 1.4 and later, but gives + # a confusing error on previous versions. + if c.dtype is None and not isinstance( + value, (float, int, str, bool, datetime, timedelta) + ): + raise ValueError( + f"Failed to add value={value!r} to column " + f"{c.name} with dtype=None. Maybe passing " + "dtype=object would help?" + ) from err + # Unclear how this could happen, but users find a way... + raise for c in rewritten_columns: if not c.fill.is_empty: @@ -668,11 +703,8 @@ def assign_rows(draw): if len(row) > len(rewritten_columns): raise InvalidArgument( - ( - "Row %r contains too many entries. Has %d but " - "expected at most %d" - ) - % (original_row, len(row), len(rewritten_columns)) + f"Row {original_row!r} contains too many entries. Has " + f"{len(row)} but expected at most {len(rewritten_columns)}" ) while len(row) < len(rewritten_columns): c = rewritten_columns[len(row)] diff --git a/hypothesis-python/src/hypothesis/extra/pytestplugin.py b/hypothesis-python/src/hypothesis/extra/pytestplugin.py index 01f83ae949..d21b209b41 100644 --- a/hypothesis-python/src/hypothesis/extra/pytestplugin.py +++ b/hypothesis-python/src/hypothesis/extra/pytestplugin.py @@ -1,263 +1,19 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER - -import base64 -from inspect import signature - -import pytest - -from hypothesis import HealthCheck, Verbosity, core, settings -from hypothesis.errors import InvalidArgument -from hypothesis.internal.detection import is_hypothesis_test -from hypothesis.internal.healthcheck import fail_health_check -from hypothesis.reporting import default as default_reporter, with_reporter -from hypothesis.statistics import collector, describe_statistics - -LOAD_PROFILE_OPTION = "--hypothesis-profile" -VERBOSITY_OPTION = "--hypothesis-verbosity" -PRINT_STATISTICS_OPTION = "--hypothesis-show-statistics" -SEED_OPTION = "--hypothesis-seed" - - -class StoringReporter: - def __init__(self, config): - self.config = config - self.results = [] - - def __call__(self, msg): - if self.config.getoption("capture", "fd") == "no": - default_reporter(msg) - if not isinstance(msg, str): - msg = repr(msg) - self.results.append(msg) - - -# Avoiding distutils.version.LooseVersion due to -# https://github.com/HypothesisWorks/hypothesis/issues/2490 -if tuple(map(int, pytest.__version__.split(".")[:2])) < (4, 6): # pragma: no cover - import warnings - - from hypothesis.errors import HypothesisWarning - - PYTEST_TOO_OLD_MESSAGE = """ - You are using pytest version %s. Hypothesis tests work with any test - runner, but our pytest plugin requires pytest 4.6 or newer. - Note that the pytest developers no longer support your version either! - Disabling the Hypothesis pytest plugin... - """ - warnings.warn(PYTEST_TOO_OLD_MESSAGE % (pytest.__version__,), HypothesisWarning) - -else: - - def pytest_addoption(parser): - group = parser.getgroup("hypothesis", "Hypothesis") - group.addoption( - LOAD_PROFILE_OPTION, - action="store", - help="Load in a registered hypothesis.settings profile", - ) - group.addoption( - VERBOSITY_OPTION, - action="store", - choices=[opt.name for opt in Verbosity], - help="Override profile with verbosity setting specified", - ) - group.addoption( - PRINT_STATISTICS_OPTION, - action="store_true", - help="Configure when statistics are printed", - default=False, - ) - group.addoption( - SEED_OPTION, - action="store", - help="Set a seed to use for all Hypothesis tests", - ) - - def pytest_report_header(config): - if config.option.verbose < 1 and settings.default.verbosity < Verbosity.verbose: - return None - profile = config.getoption(LOAD_PROFILE_OPTION) - if not profile: - profile = settings._current_profile - settings_str = settings.get_profile(profile).show_changed() - if settings_str != "": - settings_str = f" -> {settings_str}" - return f"hypothesis profile {profile!r}{settings_str}" - - def pytest_configure(config): - core.running_under_pytest = True - profile = config.getoption(LOAD_PROFILE_OPTION) - if profile: - settings.load_profile(profile) - verbosity_name = config.getoption(VERBOSITY_OPTION) - if verbosity_name: - verbosity_value = Verbosity[verbosity_name] - name = f"{settings._current_profile}-with-{verbosity_name}-verbosity" - # register_profile creates a new profile, exactly like the current one, - # with the extra values given (in this case 'verbosity') - settings.register_profile(name, verbosity=verbosity_value) - settings.load_profile(name) - seed = config.getoption(SEED_OPTION) - if seed is not None: - try: - seed = int(seed) - except ValueError: - pass - core.global_force_seed = seed - config.addinivalue_line("markers", "hypothesis: Tests which use hypothesis.") - - @pytest.hookimpl(hookwrapper=True) - def pytest_runtest_call(item): - if not hasattr(item, "obj"): - yield - elif not is_hypothesis_test(item.obj): - # If @given was not applied, check whether other hypothesis - # decorators were applied, and raise an error if they were. - if getattr(item.obj, "is_hypothesis_strategy_function", False): - raise InvalidArgument( - "%s is a function that returns a Hypothesis strategy, but pytest " - "has collected it as a test function. This is useless as the " - "function body will never be executed. To define a test " - "function, use @given instead of @composite." % (item.nodeid,) - ) - message = "Using `@%s` on a test without `@given` is completely pointless." - for name, attribute in [ - ("example", "hypothesis_explicit_examples"), - ("seed", "_hypothesis_internal_use_seed"), - ("settings", "_hypothesis_internal_settings_applied"), - ("reproduce_example", "_hypothesis_internal_use_reproduce_failure"), - ]: - if hasattr(item.obj, attribute): - raise InvalidArgument(message % (name,)) - yield - else: - # Retrieve the settings for this test from the test object, which - # is normally a Hypothesis wrapped_test wrapper. If this doesn't - # work, the test object is probably something weird - # (e.g a stateful test wrapper), so we skip the function-scoped - # fixture check. - settings = getattr(item.obj, "_hypothesis_internal_use_settings", None) - - # Check for suspicious use of function-scoped fixtures, but only - # if the corresponding health check is not suppressed. - if ( - settings is not None - and HealthCheck.function_scoped_fixture - not in settings.suppress_health_check - ): - # Warn about function-scoped fixtures, excluding autouse fixtures because - # the advice is probably not actionable and the status quo seems OK... - # See https://github.com/HypothesisWorks/hypothesis/issues/377 for detail. - msg = ( - "%s uses the %r fixture, which is reset between function calls but not " - "between test cases generated by `@given(...)`. You can change it to " - "a module- or session-scoped fixture if it is safe to reuse; if not " - "we recommend using a context manager inside your test function. See " - "https://docs.pytest.org/en/latest/fixture.html#sharing-test-data " - "for details on fixture scope." - ) - argnames = None - for fx_defs in item._request._fixturemanager.getfixtureinfo( - node=item, func=item.function, cls=None - ).name2fixturedefs.values(): - if argnames is None: - argnames = frozenset(signature(item.function).parameters) - for fx in fx_defs: - if fx.argname in argnames: - active_fx = item._request._get_active_fixturedef(fx.argname) - if active_fx.scope == "function": - fail_health_check( - settings, - msg % (item.nodeid, fx.argname), - HealthCheck.function_scoped_fixture, - ) - - if item.get_closest_marker("parametrize") is not None: - # Give every parametrized test invocation a unique database key - key = item.nodeid.encode("utf-8") - item.obj.hypothesis.inner_test._hypothesis_internal_add_digest = key - - store = StoringReporter(item.config) - - def note_statistics(stats): - stats["nodeid"] = item.nodeid - item.hypothesis_statistics = base64.b64encode( - describe_statistics(stats).encode() - ).decode() - - with collector.with_value(note_statistics): - with with_reporter(store): - yield - if store.results: - item.hypothesis_report_information = list(store.results) - - @pytest.hookimpl(hookwrapper=True) - def pytest_runtest_makereport(item, call): - report = (yield).get_result() - if hasattr(item, "hypothesis_report_information"): - report.sections.append( - ("Hypothesis", "\n".join(item.hypothesis_report_information)) - ) - if hasattr(item, "hypothesis_statistics") and report.when == "teardown": - name = "hypothesis-statistics-" + item.nodeid - try: - item.config._xml.add_global_property(name, item.hypothesis_statistics) - except AttributeError: - # --junitxml not passed, or Pytest 4.5 (before add_global_property) - # We'll fail xunit2 xml schema checks, upgrade pytest if you care. - report.user_properties.append((name, item.hypothesis_statistics)) - # If there's an HTML report, include our summary stats for each test - stats = base64.b64decode(item.hypothesis_statistics.encode()).decode() - pytest_html = item.config.pluginmanager.getplugin("html") - if pytest_html is not None: # pragma: no cover - report.extra = getattr(report, "extra", []) + [ - pytest_html.extras.text(stats, name="Hypothesis stats") - ] - - def pytest_terminal_summary(terminalreporter): - if not terminalreporter.config.getoption(PRINT_STATISTICS_OPTION): - return - terminalreporter.section("Hypothesis Statistics") - - def report(properties): - for name, value in properties: - if name.startswith("hypothesis-statistics-"): - if hasattr(value, "uniobj"): - # Under old versions of pytest, `value` was a `py.xml.raw` - # rather than a string, so we get the (unicode) string off it. - value = value.uniobj - line = base64.b64decode(value.encode()).decode() + "\n\n" - terminalreporter.write_line(line) - - try: - global_properties = terminalreporter.config._xml.global_properties - except AttributeError: - # terminalreporter.stats is a dict, where the empty string appears to - # always be the key for a list of _pytest.reports.TestReport objects - for test_report in terminalreporter.stats.get("", []): - if test_report.when == "teardown": - report(test_report.user_properties) - else: - report(global_properties) - def pytest_collection_modifyitems(items): - for item in items: - if isinstance(item, pytest.Function) and is_hypothesis_test(item.obj): - item.add_marker("hypothesis") +""" +Stub for users who manually load our pytest plugin. +The plugin implementation is now located in a top-level module outside the main +hypothesis tree, so that Pytest can load the plugin without thereby triggering +the import of Hypothesis itself (and thus loading our own plugins). +""" -def load(): - """Required for `pluggy` to load a plugin from setuptools entrypoints.""" +from _hypothesis_pytestplugin import * # noqa diff --git a/hypothesis-python/src/hypothesis/extra/pytz.py b/hypothesis-python/src/hypothesis/extra/pytz.py index efffdbc0b6..5ac18907f2 100644 --- a/hypothesis-python/src/hypothesis/extra/pytz.py +++ b/hypothesis-python/src/hypothesis/extra/pytz.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """ ---------------- @@ -28,15 +23,16 @@ import datetime as dt import pytz -from pytz.tzfile import StaticTzInfo +from pytz.tzfile import StaticTzInfo # type: ignore # considered private by typeshed -from hypothesis.strategies._internal import core as st +from hypothesis import strategies as st +from hypothesis.strategies._internal.utils import cacheable, defines_strategy __all__ = ["timezones"] -@st.cacheable -@st.defines_strategy() +@cacheable +@defines_strategy() def timezones() -> st.SearchStrategy[dt.tzinfo]: """Any timezone in the Olsen database, as a pytz tzinfo object. @@ -48,7 +44,7 @@ def timezones() -> st.SearchStrategy[dt.tzinfo]: # Some timezones have always had a constant offset from UTC. This makes # them simpler than timezones with daylight savings, and the smaller the # absolute offset the simpler they are. Of course, UTC is even simpler! - static = [pytz.UTC] # type: list + static: list = [pytz.UTC] static += sorted( (t for t in all_timezones if isinstance(t, StaticTzInfo)), key=lambda tz: abs(tz.utcoffset(dt.datetime(2000, 1, 1))), diff --git a/hypothesis-python/src/hypothesis/extra/redis.py b/hypothesis-python/src/hypothesis/extra/redis.py index 29f64e5ade..e0f1c1b851 100644 --- a/hypothesis-python/src/hypothesis/extra/redis.py +++ b/hypothesis-python/src/hypothesis/extra/redis.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from contextlib import contextmanager from datetime import timedelta @@ -24,7 +19,7 @@ class RedisExampleDatabase(ExampleDatabase): - """Store Hypothesis examples as sets in the given :class:`redis.Redis` datastore. + """Store Hypothesis examples as sets in the given :class:`~redis.Redis` datastore. This is particularly useful for shared databases, as per the recipe for a :class:`~hypothesis.database.MultiplexedDatabase`. diff --git a/hypothesis-python/src/hypothesis/internal/__init__.py b/hypothesis-python/src/hypothesis/internal/__init__.py index e61e3c2872..fcb1ac6538 100644 --- a/hypothesis-python/src/hypothesis/internal/__init__.py +++ b/hypothesis-python/src/hypothesis/internal/__init__.py @@ -1,14 +1,9 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER diff --git a/hypothesis-python/src/hypothesis/internal/cache.py b/hypothesis-python/src/hypothesis/internal/cache.py index 71a4cb80b7..891b2111f5 100644 --- a/hypothesis-python/src/hypothesis/internal/cache.py +++ b/hypothesis-python/src/hypothesis/internal/cache.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import attr @@ -94,7 +89,7 @@ def __setitem__(self, key, value): if self.max_size == self.__pinned_entry_count: raise ValueError( "Cannot increase size of cache where all keys have been pinned." - ) + ) from None entry = Entry(key, value, self.new_entry(key, value)) if len(self.data) >= self.max_size: evicted = self.data[0] @@ -267,3 +262,16 @@ def on_access(self, key, value, score): score[0] = 2 score[1] = self.tick() return score + + def pin(self, key): + try: + super().pin(key) + except KeyError: + # The whole point of an LRU cache is that it might drop things for you + assert key not in self.keys_to_indices + + def unpin(self, key): + try: + super().unpin(key) + except KeyError: + assert key not in self.keys_to_indices diff --git a/hypothesis-python/src/hypothesis/internal/cathetus.py b/hypothesis-python/src/hypothesis/internal/cathetus.py index 8e89424e84..30e0d214f1 100644 --- a/hypothesis-python/src/hypothesis/internal/cathetus.py +++ b/hypothesis-python/src/hypothesis/internal/cathetus.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from math import fabs, inf, isinf, isnan, nan, sqrt from sys import float_info diff --git a/hypothesis-python/src/hypothesis/internal/charmap.py b/hypothesis-python/src/hypothesis/internal/charmap.py index cc6f1203f5..148f438223 100644 --- a/hypothesis-python/src/hypothesis/internal/charmap.py +++ b/hypothesis-python/src/hypothesis/internal/charmap.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import gzip import json @@ -318,7 +313,7 @@ def _query_for_key(key): return result -limited_category_index_cache = {} # type: cache_type +limited_category_index_cache: cache_type = {} def query( @@ -330,7 +325,7 @@ def query( exclude_characters="", ): """Return a tuple of intervals covering the codepoints for all characters - that meet the critera (min_codepoint <= codepoint(c) <= max_codepoint and + that meet the criteria (min_codepoint <= codepoint(c) <= max_codepoint and any(cat in include_categories for cat in categories(c)) and all(cat not in exclude_categories for cat in categories(c)) or (c in include_characters) diff --git a/hypothesis-python/src/hypothesis/internal/compat.py b/hypothesis-python/src/hypothesis/internal/compat.py index aca6f0b3aa..e7e3bf2a53 100644 --- a/hypothesis-python/src/hypothesis/internal/compat.py +++ b/hypothesis-python/src/hypothesis/internal/compat.py @@ -1,85 +1,61 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import codecs -import importlib import inspect import platform import sys import typing +try: + BaseExceptionGroup = BaseExceptionGroup + ExceptionGroup = ExceptionGroup # pragma: no cover +except NameError: + from exceptiongroup import ( + BaseExceptionGroup as BaseExceptionGroup, + ExceptionGroup as ExceptionGroup, + ) +if typing.TYPE_CHECKING: # pragma: no cover + from typing_extensions import Concatenate as Concatenate, ParamSpec as ParamSpec +else: + try: + from typing import Concatenate as Concatenate, ParamSpec as ParamSpec + except ImportError: + try: + from typing_extensions import ( + Concatenate as Concatenate, + ParamSpec as ParamSpec, + ) + except ImportError: + Concatenate, ParamSpec = None, None + PYPY = platform.python_implementation() == "PyPy" WINDOWS = platform.system() == "Windows" -def bit_length(n): - return n.bit_length() - - -def str_to_bytes(s): - return s.encode(a_good_encoding()) - - -def escape_unicode_characters(s): +def escape_unicode_characters(s: str) -> str: return codecs.encode(s, "unicode_escape").decode("ascii") -def int_from_bytes(data): +def int_from_bytes(data: typing.Union[bytes, bytearray]) -> int: return int.from_bytes(data, "big") -def int_to_bytes(i, size): +def int_to_bytes(i: int, size: int) -> bytes: return i.to_bytes(size, "big") -def int_to_byte(i): +def int_to_byte(i: int) -> bytes: return bytes([i]) -def a_good_encoding(): - return "utf-8" - - -def to_unicode(x): - if isinstance(x, str): - return x - else: - return x.decode(a_good_encoding()) - - -def qualname(f): - try: - return f.__qualname__ - except AttributeError: - return f.__name__ - - -try: - # These types are new in Python 3.7, but also (partially) backported to the - # typing backport on PyPI. Use if possible; or fall back to older names. - typing_root_type = (typing._Final, typing._GenericAlias) # type: ignore - ForwardRef = typing.ForwardRef # type: ignore -except AttributeError: - typing_root_type = (typing.TypingMeta, typing.TypeVar) # type: ignore - try: - typing_root_type += (typing._Union,) # type: ignore - except AttributeError: - pass - ForwardRef = typing._ForwardRef # type: ignore - - def is_typed_named_tuple(cls): """Return True if cls is probably a subtype of `typing.NamedTuple`. @@ -108,46 +84,43 @@ def get_type_hints(thing): Never errors: instead of raising TypeError for uninspectable objects, or NameError for unresolvable forward references, just return an empty dict. """ + kwargs = {} if sys.version_info[:2] < (3, 9) else {"include_extras": True} + try: - hints = typing.get_type_hints(thing) + hints = typing.get_type_hints(thing, **kwargs) except (AttributeError, TypeError, NameError): hints = {} - if not inspect.isclass(thing): - return hints - - try: - hints.update(typing.get_type_hints(thing.__init__)) - except (TypeError, NameError, AttributeError): - pass + if inspect.isclass(thing): + try: + hints.update(typing.get_type_hints(thing.__init__, **kwargs)) + except (TypeError, NameError, AttributeError): + pass try: if hasattr(thing, "__signature__"): # It is possible for the signature and annotations attributes to # differ on an object due to renamed arguments. - # To prevent missing arguments we use the signature to provide any type - # hints it has and then override any common names with the more - # comprehensive type information from get_type_hints - # See https://github.com/HypothesisWorks/hypothesis/pull/2580 - # for more details. + from hypothesis.internal.reflection import get_signature from hypothesis.strategies._internal.types import is_a_type vkinds = (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD) - for p in inspect.signature(thing).parameters.values(): - if p.kind not in vkinds and is_a_type(p.annotation): + for p in get_signature(thing).parameters.values(): + if ( + p.kind not in vkinds + and is_a_type(p.annotation) + and p.annotation is not p.empty + ): if p.default is None: hints[p.name] = typing.Optional[p.annotation] else: hints[p.name] = p.annotation - except (AttributeError, TypeError, NameError): + except (AttributeError, TypeError, NameError): # pragma: no cover pass return hints -importlib_invalidate_caches = getattr(importlib, "invalidate_caches", lambda: ()) - - def update_code_location(code, newfile, newlineno): """Take a code object and lie shamelessly about where it comes from. @@ -164,40 +137,31 @@ def update_code_location(code, newfile, newlineno): # added to facilitate future-proof code. See BPO-37032 for details. return code.replace(co_filename=newfile, co_firstlineno=newlineno) - # This field order is accurate for 3.5 - 3.7, but not 3.8 when a new field - # was added for positional-only arguments. However it also added a .replace() - # method that we use instead of field indices, so they're fine as-is. - CODE_FIELD_ORDER = [ - "co_argcount", - "co_kwonlyargcount", - "co_nlocals", - "co_stacksize", - "co_flags", - "co_code", - "co_consts", - "co_names", - "co_varnames", - "co_filename", - "co_name", - "co_firstlineno", - "co_lnotab", - "co_freevars", - "co_cellvars", - ] - unpacked = [getattr(code, name) for name in CODE_FIELD_ORDER] - unpacked[CODE_FIELD_ORDER.index("co_filename")] = newfile - unpacked[CODE_FIELD_ORDER.index("co_firstlineno")] = newlineno - return type(code)(*unpacked) - - -def cast_unicode(s, encoding=None): - if isinstance(s, bytes): - return s.decode(encoding or a_good_encoding(), "replace") - return s - - -def get_stream_enc(stream, default=None): - return getattr(stream, "encoding", None) or default + else: # pragma: no cover + # This field order is accurate for 3.5 - 3.7, but not 3.8 when a new field + # was added for positional-only arguments. However it also added a .replace() + # method that we use instead of field indices, so they're fine as-is. + CODE_FIELD_ORDER = [ + "co_argcount", + "co_kwonlyargcount", + "co_nlocals", + "co_stacksize", + "co_flags", + "co_code", + "co_consts", + "co_names", + "co_varnames", + "co_filename", + "co_name", + "co_firstlineno", + "co_lnotab", + "co_freevars", + "co_cellvars", + ] + unpacked = [getattr(code, name) for name in CODE_FIELD_ORDER] + unpacked[CODE_FIELD_ORDER.index("co_filename")] = newfile + unpacked[CODE_FIELD_ORDER.index("co_firstlineno")] = newlineno + return type(code)(*unpacked) # Under Python 2, math.floor and math.ceil returned floats, which cannot @@ -223,9 +187,10 @@ def ceil(x): def bad_django_TestCase(runner): if runner is None or "django.test" not in sys.modules: return False - if not isinstance(runner, sys.modules["django.test"].TransactionTestCase): - return False + else: # pragma: no cover + if not isinstance(runner, sys.modules["django.test"].TransactionTestCase): + return False - from hypothesis.extra.django._impl import HypothesisTestCase + from hypothesis.extra.django._impl import HypothesisTestCase - return not isinstance(runner, HypothesisTestCase) + return not isinstance(runner, HypothesisTestCase) diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/__init__.py b/hypothesis-python/src/hypothesis/internal/conjecture/__init__.py index e61e3c2872..fcb1ac6538 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/__init__.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/__init__.py @@ -1,14 +1,9 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/choicetree.py b/hypothesis-python/src/hypothesis/internal/conjecture/choicetree.py index 549bd3f5ca..38bcc3a571 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/choicetree.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/choicetree.py @@ -1,29 +1,28 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from collections import defaultdict +from random import Random +from typing import Callable, Dict, Iterable, List, Optional, Sequence from hypothesis.internal.conjecture.junkdrawer import LazySequenceCopy, pop_random -def prefix_selection_order(prefix): +def prefix_selection_order( + prefix: Sequence[int], +) -> Callable[[int, int], Iterable[int]]: """Select choices starting from ``prefix```, preferring to move left then wrapping around to the right.""" - def selection_order(depth, n): + def selection_order(depth: int, n: int) -> Iterable[int]: if depth < len(prefix): i = prefix[depth] if i >= n: @@ -36,10 +35,10 @@ def selection_order(depth, n): return selection_order -def random_selection_order(random): +def random_selection_order(random: Random) -> Callable[[int, int], Iterable[int]]: """Select choices uniformly at random.""" - def selection_order(depth, n): + def selection_order(depth: int, n: int) -> Iterable[int]: pending = LazySequenceCopy(range(n)) while pending: yield pop_random(random, pending) @@ -50,14 +49,21 @@ def selection_order(depth, n): class Chooser: """A source of nondeterminism for use in shrink passes.""" - def __init__(self, tree, selection_order): + def __init__( + self, + tree: "ChoiceTree", + selection_order: Callable[[int, int], Iterable[int]], + ): self.__selection_order = selection_order - self.__tree = tree self.__node_trail = [tree.root] - self.__choices = [] + self.__choices: "List[int]" = [] self.__finished = False - def choose(self, values, condition=lambda x: True): + def choose( + self, + values: Sequence[int], + condition: Callable[[int], bool] = lambda x: True, + ) -> int: """Return some element of values satisfying the condition that will not lead to an exhausted branch, or raise DeadBranch if no such element exist". @@ -85,7 +91,7 @@ def choose(self, values, condition=lambda x: True): assert node.live_child_count == 0 raise DeadBranch() - def finish(self): + def finish(self) -> Sequence[int]: """Record the decisions made in the underlying tree and return a prefix that can be used for the next Chooser to be used.""" self.__finished = True @@ -100,6 +106,7 @@ def finish(self): i = self.__choices.pop() target = self.__node_trail[-1] target.children[i] = DeadNode + assert target.live_child_count is not None target.live_child_count -= 1 return result @@ -112,14 +119,18 @@ class ChoiceTree: decisions about what to do. """ - def __init__(self): + def __init__(self) -> None: self.root = TreeNode() @property - def exhausted(self): + def exhausted(self) -> bool: return self.root.exhausted - def step(self, selection_order, f): + def step( + self, + selection_order: Callable[[int, int], Iterable[int]], + f: Callable[[Chooser], None], + ) -> Sequence[int]: assert not self.exhausted chooser = Chooser(self, selection_order) @@ -131,13 +142,13 @@ def step(self, selection_order, f): class TreeNode: - def __init__(self): - self.children = defaultdict(TreeNode) - self.live_child_count = None - self.n = None + def __init__(self) -> None: + self.children: Dict[int, TreeNode] = defaultdict(TreeNode) + self.live_child_count: "Optional[int]" = None + self.n: "Optional[int]" = None @property - def exhausted(self): + def exhausted(self) -> bool: return self.live_child_count == 0 diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/data.py b/hypothesis-python/src/hypothesis/internal/conjecture/data.py index 2b030210c2..4cb0b463f7 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/data.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/data.py @@ -1,43 +1,75 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import time from collections import defaultdict from enum import IntEnum +from random import Random +from typing import ( + TYPE_CHECKING, + Any, + Dict, + FrozenSet, + Hashable, + Iterable, + Iterator, + List, + Optional, + Sequence, + Set, + Tuple, + Type, + Union, +) import attr from hypothesis.errors import Frozen, InvalidArgument, StopTest -from hypothesis.internal.compat import bit_length, int_from_bytes, int_to_bytes +from hypothesis.internal.compat import int_from_bytes, int_to_bytes from hypothesis.internal.conjecture.junkdrawer import IntList, uniform from hypothesis.internal.conjecture.utils import calc_label_from_name +if TYPE_CHECKING: + from typing_extensions import dataclass_transform + + from hypothesis.strategies import SearchStrategy + from hypothesis.strategies._internal.strategies import Ex +else: + + def dataclass_transform(): + def wrapper(tp): + return tp + + return wrapper + + TOP_LABEL = calc_label_from_name("top") DRAW_BYTES_LABEL = calc_label_from_name("draw_bytes() in ConjectureData") +InterestingOrigin = Tuple[ + Type[BaseException], str, int, Tuple[Any, ...], Tuple[Tuple[Any, ...], ...] +] +TargetObservations = Dict[Optional[str], Union[int, float]] + + class ExtraInformation: """A class for holding shared state on a ``ConjectureData`` that should be added to the final ``ConjectureResult``.""" - def __repr__(self): + def __repr__(self) -> str: return "ExtraInformation({})".format( - ", ".join([f"{k}={v!r}" for k, v in self.__dict__.items()]), + ", ".join(f"{k}={v!r}" for k, v in self.__dict__.items()), ) - def has_information(self): + def has_information(self) -> bool: return bool(self.__dict__) @@ -47,19 +79,20 @@ class Status(IntEnum): VALID = 2 INTERESTING = 3 - def __repr__(self): + def __repr__(self) -> str: return f"Status.{self.name}" -@attr.s(frozen=True, slots=True) +@dataclass_transform() +@attr.s(frozen=True, slots=True, auto_attribs=True) class StructuralCoverageTag: - label = attr.ib() + label: int -STRUCTURAL_COVERAGE_CACHE = {} +STRUCTURAL_COVERAGE_CACHE: Dict[int, StructuralCoverageTag] = {} -def structural_coverage(label): +def structural_coverage(label: int) -> StructuralCoverageTag: try: return STRUCTURAL_COVERAGE_CACHE[label] except KeyError: @@ -98,29 +131,29 @@ class Example: __slots__ = ("owner", "index") - def __init__(self, owner, index): + def __init__(self, owner: "Examples", index: int) -> None: self.owner = owner self.index = index - def __eq__(self, other): + def __eq__(self, other: object) -> bool: if self is other: return True if not isinstance(other, Example): return NotImplemented return (self.owner is other.owner) and (self.index == other.index) - def __ne__(self, other): + def __ne__(self, other: object) -> bool: if self is other: return False if not isinstance(other, Example): return NotImplemented return (self.owner is not other.owner) or (self.index != other.index) - def __repr__(self): + def __repr__(self) -> str: return f"examples[{self.index}]" @property - def label(self): + def label(self) -> int: """A label is an opaque value that associates each example with its approximate origin, such as a particular strategy class or a particular kind of draw.""" @@ -134,12 +167,12 @@ def parent(self): return self.owner.parentage[self.index] @property - def start(self): + def start(self) -> int: """The position of the start of this example in the byte stream.""" return self.owner.starts[self.index] @property - def end(self): + def end(self) -> int: """The position directly after the last byte in this byte stream. i.e. the example corresponds to the half open region [start, end). """ @@ -159,7 +192,7 @@ def trivial(self): return self.index in self.owner.trivial @property - def discarded(self): + def discarded(self) -> bool: """True if this is example's ``stop_example`` call had ``discard`` set to ``True``. This means we believe that the shrinker should be able to delete this example completely, without affecting the value produced by its enclosing @@ -168,12 +201,12 @@ def discarded(self): return self.index in self.owner.discarded @property - def length(self): + def length(self) -> int: """The number of bytes in this example.""" return self.end - self.start @property - def children(self): + def children(self) -> "List[Example]": """The list of all examples with this as a parent, in increasing index order.""" return [self.owner[i] for i in self.owner.children[self.index]] @@ -188,14 +221,14 @@ class ExampleProperty: to calculate these properties. """ - def __init__(self, examples): - self.example_stack = [] + def __init__(self, examples: "Examples"): + self.example_stack: "List[int]" = [] self.examples = examples self.bytes_read = 0 self.example_count = 0 self.block_count = 0 - def run(self): + def run(self) -> Any: """Rerun the test case with this visitor and return the results of ``self.finish()``.""" self.begin() @@ -217,41 +250,41 @@ def run(self): self.__pop(record == STOP_EXAMPLE_DISCARD_RECORD) return self.finish() - def __push(self, label_index): + def __push(self, label_index: int) -> None: i = self.example_count assert i < len(self.examples) self.start_example(i, label_index) self.example_count += 1 self.example_stack.append(i) - def __pop(self, discarded): + def __pop(self, discarded: bool) -> None: i = self.example_stack.pop() self.stop_example(i, discarded) - def begin(self): + def begin(self) -> None: """Called at the beginning of the run to initialise any relevant state.""" self.result = IntList.of_length(len(self.examples)) - def start_example(self, i, label_index): + def start_example(self, i: int, label_index: int) -> None: """Called at the start of each example, with ``i`` the index of the example and ``label_index`` the index of its label in ``self.examples.labels``.""" - def block(self, i): + def block(self, i: int) -> None: """Called with each ``draw_bits`` call, with ``i`` the index of the corresponding block in ``self.examples.blocks``""" - def stop_example(self, i, discarded): + def stop_example(self, i: int, discarded: bool) -> None: """Called at the end of each example, with ``i`` the index of the example and ``discarded`` being ``True`` if ``stop_example`` was called with ``discard=True``.""" - def finish(self): + def finish(self) -> Any: return self.result -def calculated_example_property(cls): +def calculated_example_property(cls: Type[ExampleProperty]) -> Any: """Given an ``ExampleProperty`` as above we use this decorator to transform it into a lazy property on the ``Examples`` class, which has as its value the result of calling ``cls.run()``, @@ -262,7 +295,7 @@ def calculated_example_property(cls): name = cls.__name__ cache_name = "__" + name - def lazy_calculate(self): + def lazy_calculate(self: "Examples") -> IntList: result = getattr(self, cache_name, None) if result is None: result = cls(self).run() @@ -270,7 +303,7 @@ def lazy_calculate(self): return result lazy_calculate.__name__ = cls.__name__ - lazy_calculate.__qualname__ = getattr(cls, "__qualname__", cls.__name__) + lazy_calculate.__qualname__ = cls.__qualname__ return property(lazy_calculate) @@ -291,15 +324,16 @@ class ExampleRecord: they currently have slightly different functions and implementations. """ - def __init__(self): + def __init__(self) -> None: self.labels = [DRAW_BYTES_LABEL] - self.__index_of_labels = {DRAW_BYTES_LABEL: 0} + self.__index_of_labels: "Optional[Dict[int, int]]" = {DRAW_BYTES_LABEL: 0} self.trail = IntList() - def freeze(self): + def freeze(self) -> None: self.__index_of_labels = None - def start_example(self, label): + def start_example(self, label: int) -> None: + assert self.__index_of_labels is not None try: i = self.__index_of_labels[label] except KeyError: @@ -307,13 +341,13 @@ def start_example(self, label): self.labels.append(label) self.trail.append(START_EXAMPLE_RECORD + i) - def stop_example(self, discard): + def stop_example(self, discard: bool) -> None: if discard: self.trail.append(STOP_EXAMPLE_DISCARD_RECORD) else: self.trail.append(STOP_EXAMPLE_NO_DISCARD_RECORD) - def draw_bits(self, n, forced): + def draw_bits(self, n: int, forced: Optional[int]) -> None: self.trail.append(DRAW_BITS_RECORD) @@ -328,7 +362,7 @@ class Examples: described there. """ - def __init__(self, record, blocks): + def __init__(self, record: ExampleRecord, blocks: "Blocks") -> None: self.trail = record.trail self.labels = record.labels self.__length = ( @@ -336,117 +370,125 @@ def __init__(self, record, blocks): + record.trail.count(STOP_EXAMPLE_NO_DISCARD_RECORD) + record.trail.count(DRAW_BITS_RECORD) ) - self.__example_lengths = None - self.blocks = blocks - self.__children = None + self.__children: "Optional[List[Sequence[int]]]" = None - @calculated_example_property - class starts_and_ends(ExampleProperty): + class _starts_and_ends(ExampleProperty): def begin(self): self.starts = IntList.of_length(len(self.examples)) self.ends = IntList.of_length(len(self.examples)) - def start_example(self, i, label_index): + def start_example(self, i: int, label_index: int) -> None: self.starts[i] = self.bytes_read - def stop_example(self, i, label_index): + def stop_example(self, i: int, discarded: bool) -> None: self.ends[i] = self.bytes_read - def finish(self): + def finish(self) -> Tuple[IntList, IntList]: return (self.starts, self.ends) + starts_and_ends: "Tuple[IntList, IntList]" = calculated_example_property( + _starts_and_ends + ) + @property - def starts(self): + def starts(self) -> IntList: return self.starts_and_ends[0] @property - def ends(self): + def ends(self) -> IntList: return self.starts_and_ends[1] - @calculated_example_property - class discarded(ExampleProperty): - def begin(self): - self.result = set() + class _discarded(ExampleProperty): + def begin(self) -> None: + self.result: "Set[int]" = set() # type: ignore # IntList in parent class - def finish(self): + def finish(self) -> FrozenSet[int]: return frozenset(self.result) - def stop_example(self, i, discarded): + def stop_example(self, i: int, discarded: bool) -> None: if discarded: self.result.add(i) - @calculated_example_property - class trivial(ExampleProperty): - def begin(self): + discarded: FrozenSet[int] = calculated_example_property(_discarded) + + class _trivial(ExampleProperty): + def begin(self) -> None: self.nontrivial = IntList.of_length(len(self.examples)) - self.result = set() + self.result: "Set[int]" = set() # type: ignore # IntList in parent class - def block(self, i): + def block(self, i: int) -> None: if not self.examples.blocks.trivial(i): self.nontrivial[self.example_stack[-1]] = 1 - def stop_example(self, i, discarded): + def stop_example(self, i: int, discarded: bool) -> None: if self.nontrivial[i]: if self.example_stack: self.nontrivial[self.example_stack[-1]] = 1 else: self.result.add(i) - def finish(self): + def finish(self) -> FrozenSet[int]: return frozenset(self.result) - @calculated_example_property - class parentage(ExampleProperty): - def stop_example(self, i, discarded): + trivial: FrozenSet[int] = calculated_example_property(_trivial) + + class _parentage(ExampleProperty): + def stop_example(self, i: int, discarded: bool) -> None: if i > 0: self.result[i] = self.example_stack[-1] - @calculated_example_property - class depths(ExampleProperty): + parentage: IntList = calculated_example_property(_parentage) + + class _depths(ExampleProperty): def begin(self): self.result = IntList.of_length(len(self.examples)) - def start_example(self, i, label_index): + def start_example(self, i: int, label_index: int) -> None: self.result[i] = len(self.example_stack) - @calculated_example_property - class label_indices(ExampleProperty): - def start_example(self, i, label_index): + depths: IntList = calculated_example_property(_depths) + + class _label_indices(ExampleProperty): + def start_example(self, i: int, label_index: int) -> None: self.result[i] = label_index - @calculated_example_property - class mutator_groups(ExampleProperty): - def begin(self): - self.groups = defaultdict(list) + label_indices: IntList = calculated_example_property(_label_indices) + + class _mutator_groups(ExampleProperty): + def begin(self) -> None: + self.groups: "Dict[Tuple[int, int], List[int]]" = defaultdict(list) - def start_example(self, i, label_index): + def start_example(self, i: int, label_index: int) -> None: depth = len(self.example_stack) self.groups[label_index, depth].append(i) - def finish(self): + def finish(self) -> Iterable[Iterable[int]]: # Discard groups with only one example, since the mutator can't # do anything useful with them. return [g for g in self.groups.values() if len(g) >= 2] + mutator_groups: List[List[int]] = calculated_example_property(_mutator_groups) + @property - def children(self): + def children(self) -> List[Sequence[int]]: if self.__children is None: - self.__children = [IntList() for _ in range(len(self))] + children = [IntList() for _ in range(len(self))] for i, p in enumerate(self.parentage): if i > 0: - self.__children[p].append(i) + children[p].append(i) # Replace empty children lists with a tuple to reduce # memory usage. - for i, c in enumerate(self.__children): + for i, c in enumerate(children): if not c: - self.__children[i] = () - return self.__children + children[i] = () # type: ignore + self.__children = children # type: ignore + return self.__children # type: ignore - def __len__(self): + def __len__(self) -> int: return self.__length - def __getitem__(self, i): + def __getitem__(self, i: int) -> Example: assert isinstance(i, int) n = len(self) if i < -n or i >= n: @@ -456,6 +498,7 @@ def __getitem__(self, i): return Example(self, i) +@dataclass_transform() @attr.s(slots=True, frozen=True) class Block: """Blocks track the flat list of lowest-level draws from the byte stream, @@ -466,31 +509,31 @@ class Block: individual call to ``draw_bits``. """ - start = attr.ib() - end = attr.ib() + start: int = attr.ib() + end: int = attr.ib() # Index of this block inside the overall list of blocks. - index = attr.ib() + index: int = attr.ib() # True if this block's byte values were forced by a write operation. # As long as the bytes before this block remain the same, modifying this # block's bytes will have no effect. - forced = attr.ib(repr=False) + forced: bool = attr.ib(repr=False) # True if this block's byte values are all 0. Reading this flag can be # more convenient than explicitly checking a slice for non-zero bytes. - all_zero = attr.ib(repr=False) + all_zero: bool = attr.ib(repr=False) @property - def bounds(self): + def bounds(self) -> Tuple[int, int]: return (self.start, self.end) @property - def length(self): + def length(self) -> int: return self.end - self.start @property - def trivial(self): + def trivial(self) -> bool: return self.forced or self.all_zero @@ -512,20 +555,22 @@ class Blocks: have to allocate the actual object.""" __slots__ = ("endpoints", "owner", "__blocks", "__count", "__sparse") + owner: "Union[ConjectureData, ConjectureResult, None]" + __blocks: Union[Dict[int, Block], List[Optional[Block]]] - def __init__(self, owner): + def __init__(self, owner: "ConjectureData") -> None: self.owner = owner self.endpoints = IntList() self.__blocks = {} self.__count = 0 self.__sparse = True - def add_endpoint(self, n): + def add_endpoint(self, n: int) -> None: """Add n to the list of endpoints.""" assert isinstance(self.owner, ConjectureData) self.endpoints.append(n) - def transfer_ownership(self, new_owner): + def transfer_ownership(self, new_owner: "ConjectureResult") -> None: """Used to move ``Blocks`` over to a ``ConjectureResult`` object when that is read to be used and we no longer want to keep the whole ``ConjectureData`` around.""" @@ -533,7 +578,7 @@ def transfer_ownership(self, new_owner): self.owner = new_owner self.__check_completion() - def start(self, i): + def start(self, i: int) -> int: """Equivalent to self[i].start.""" i = self._check_index(i) @@ -542,15 +587,15 @@ def start(self, i): else: return self.end(i - 1) - def end(self, i): + def end(self, i: int) -> int: """Equivalent to self[i].end.""" return self.endpoints[i] - def bounds(self, i): + def bounds(self, i: int) -> Tuple[int, int]: """Equivalent to self[i].bounds.""" return (self.start(i), self.end(i)) - def all_bounds(self): + def all_bounds(self) -> Iterable[Tuple[int, int]]: """Equivalent to [(b.start, b.end) for b in self].""" prev = 0 for e in self.endpoints: @@ -561,16 +606,16 @@ def all_bounds(self): def last_block_length(self): return self.end(-1) - self.start(-1) - def __len__(self): + def __len__(self) -> int: return len(self.endpoints) - def __known_block(self, i): + def __known_block(self, i: int) -> Optional[Block]: try: return self.__blocks[i] except (KeyError, IndexError): return None - def trivial(self, i): + def trivial(self, i: int) -> Any: """Equivalent to self.blocks[i].trivial.""" if self.owner is not None: return self.start(i) in self.owner.forced_indices or not any( @@ -579,7 +624,7 @@ def trivial(self, i): else: return self[i].trivial - def _check_index(self, i): + def _check_index(self, i: int) -> int: n = len(self) if i < -n or i >= n: raise IndexError(f"Index {i} out of range [-{n}, {n})") @@ -587,7 +632,7 @@ def _check_index(self, i): i += n return i - def __getitem__(self, i): + def __getitem__(self, i: int) -> Block: i = self._check_index(i) assert i >= 0 result = self.__known_block(i) @@ -599,7 +644,8 @@ def __getitem__(self, i): # stop being sparse and want to use most of the blocks. Switch # over to a list at that point. if self.__sparse and len(self.__blocks) * 2 >= len(self): - new_blocks = [None] * len(self) + new_blocks: "List[Optional[Block]]" = [None] * len(self) + assert isinstance(self.__blocks, dict) for k, v in self.__blocks.items(): new_blocks[k] = v self.__sparse = False @@ -618,6 +664,7 @@ def __getitem__(self, i): # Integrity check: We can't have allocated more blocks than we have # positions for blocks. assert self.__count <= len(self) + assert self.owner is not None result = Block( start=start, end=end, @@ -649,12 +696,12 @@ def __check_completion(self): if self.__count == len(self) and isinstance(self.owner, ConjectureResult): self.owner = None - def __iter__(self): + def __iter__(self) -> Iterator[Block]: for i in range(len(self)): yield self[i] - def __repr__(self): - parts = [] + def __repr__(self) -> str: + parts: "List[str]" = [] for i in range(len(self)): b = self.__known_block(i) if b is None: @@ -670,7 +717,7 @@ class _Overrun: def __repr__(self): return "Overrun" - def as_result(self): + def as_result(self) -> "_Overrun": return self @@ -687,14 +734,18 @@ class DataObserver: ConjectureData object, primarily used for tracking the behaviour in the tree cache.""" - def conclude_test(self, status, interesting_origin): + def conclude_test( + self, + status: Status, + interesting_origin: Optional[InterestingOrigin], + ) -> None: """Called when ``conclude_test`` is called on the observed ``ConjectureData``, with the same arguments. Note that this is called after ``freeze`` has completed. """ - def draw_bits(self, n_bits, forced, value): + def draw_bits(self, n_bits: int, forced: bool, value: int) -> None: """Called when ``draw_bits`` is called on on the observed ``ConjectureData``. * ``n_bits`` is the number of bits drawn. @@ -703,35 +754,36 @@ def draw_bits(self, n_bits, forced, value): * ``value`` is the result that ``draw_bits`` returned. """ - def kill_branch(self): + def kill_branch(self) -> None: """Mark this part of the tree as not worth re-exploring.""" +@dataclass_transform() @attr.s(slots=True) class ConjectureResult: """Result class storing the parts of ConjectureData that we will care about after the original ConjectureData has outlived its usefulness.""" - status = attr.ib() - interesting_origin = attr.ib() - buffer = attr.ib() - blocks = attr.ib() - output = attr.ib() - extra_information = attr.ib() - has_discards = attr.ib() - target_observations = attr.ib() - tags = attr.ib() - forced_indices = attr.ib(repr=False) - examples = attr.ib(repr=False) - - index = attr.ib(init=False) - - def __attrs_post_init__(self): + status: Status = attr.ib() + interesting_origin: Optional[InterestingOrigin] = attr.ib() + buffer: bytes = attr.ib() + blocks: Blocks = attr.ib() + output: str = attr.ib() + extra_information: Optional[ExtraInformation] = attr.ib() + has_discards: bool = attr.ib() + target_observations: TargetObservations = attr.ib() + tags: FrozenSet[StructuralCoverageTag] = attr.ib() + forced_indices: FrozenSet[int] = attr.ib(repr=False) + examples: Examples = attr.ib(repr=False) + + index: int = attr.ib(init=False) + + def __attrs_post_init__(self) -> None: self.index = len(self.buffer) self.forced_indices = frozenset(self.forced_indices) - def as_result(self): + def as_result(self) -> "ConjectureResult": return self @@ -743,12 +795,20 @@ def as_result(self): class ConjectureData: @classmethod - def for_buffer(self, buffer, observer=None): - return ConjectureData( - prefix=buffer, max_length=len(buffer), random=None, observer=observer - ) - - def __init__(self, max_length, prefix, random, observer=None): + def for_buffer( + cls, + buffer: Union[List[int], bytes], + observer: Optional[DataObserver] = None, + ) -> "ConjectureData": + return cls(len(buffer), buffer, random=None, observer=observer) + + def __init__( + self, + max_length: int, + prefix: Union[List[int], bytes, bytearray], + random: Optional[Random], + observer: Optional[DataObserver] = None, + ) -> None: if observer is None: observer = DataObserver() assert isinstance(observer, DataObserver) @@ -757,15 +817,13 @@ def __init__(self, max_length, prefix, random, observer=None): self.max_length = max_length self.is_find = False self.overdraw = 0 - self.__block_starts = defaultdict(list) - self.__block_starts_calculated_to = 0 - self.__prefix = prefix + self.__prefix = bytes(prefix) self.__random = random assert random is not None or max_length <= len(prefix) self.blocks = Blocks(self) - self.buffer = bytearray() + self.buffer: "Union[bytes, bytearray]" = bytearray() self.index = 0 self.output = "" self.status = Status.VALID @@ -774,28 +832,28 @@ def __init__(self, max_length, prefix, random, observer=None): self.testcounter = global_test_counter global_test_counter += 1 self.start_time = time.perf_counter() - self.events = set() - self.forced_indices = set() - self.interesting_origin = None - self.draw_times = [] + self.events: "Union[Set[Hashable], FrozenSet[Hashable]]" = set() + self.forced_indices: "Set[int]" = set() + self.interesting_origin: Optional[InterestingOrigin] = None + self.draw_times: "List[float]" = [] self.max_depth = 0 self.has_discards = False - self.__result = None + self.__result: "Optional[ConjectureResult]" = None # Observations used for targeted search. They'll be aggregated in # ConjectureRunner.generate_new_examples and fed to TargetSelector. - self.target_observations = {} + self.target_observations: TargetObservations = {} # Tags which indicate something about which part of the search space # this example is in. These are used to guide generation. - self.tags = set() - self.labels_for_structure_stack = [] + self.tags: "Set[StructuralCoverageTag]" = set() + self.labels_for_structure_stack: "List[Set[int]]" = [] # Normally unpopulated but we need this in the niche case # that self.as_result() is Overrun but we still want the # examples for reporting purposes. - self.__examples = None + self.__examples: "Optional[Examples]" = None # We want the top level example to have depth 0, so we start # at -1. @@ -813,7 +871,7 @@ def __repr__(self): ", frozen" if self.frozen else "", ) - def as_result(self): + def as_result(self) -> Union[ConjectureResult, _Overrun]: """Convert the result of running this test into either an Overrun object or a ConjectureResult.""" @@ -834,32 +892,31 @@ def as_result(self): has_discards=self.has_discards, target_observations=self.target_observations, tags=frozenset(self.tags), - forced_indices=self.forced_indices, + forced_indices=frozenset(self.forced_indices), ) + assert self.__result is not None self.blocks.transfer_ownership(self.__result) return self.__result - def __assert_not_frozen(self, name): + def __assert_not_frozen(self, name: str) -> None: if self.frozen: raise Frozen(f"Cannot call {name} on frozen ConjectureData") - def note(self, value): + def note(self, value: Any) -> None: self.__assert_not_frozen("note") if not isinstance(value, str): value = repr(value) self.output += value - def draw(self, strategy, label=None): + def draw(self, strategy: "SearchStrategy[Ex]", label: Optional[int] = None) -> "Ex": if self.is_find and not strategy.supports_find: raise InvalidArgument( - ( - "Cannot use strategy %r within a call to find (presumably " - "because it would be invalid after the call had ended)." - ) - % (strategy,) + f"Cannot use strategy {strategy!r} within a call to find " + "(presumably because it would be invalid after the call had ended)." ) at_top_level = self.depth == 0 + start_time = None if at_top_level: # We start this timer early, because accessing attributes on a LazyStrategy # can be almost arbitrarily slow. In cases like characters() and text() @@ -876,12 +933,14 @@ def draw(self, strategy, label=None): self.mark_invalid() if label is None: + assert isinstance(strategy.label, int) label = strategy.label self.start_example(label=label) try: if not at_top_level: return strategy.do_draw(self) else: + assert start_time is not None strategy.validate() try: return strategy.do_draw(self) @@ -890,7 +949,7 @@ def draw(self, strategy, label=None): finally: self.stop_example() - def start_example(self, label): + def start_example(self, label: int) -> None: self.__assert_not_frozen("start_example") self.depth += 1 # Logically it would make sense for this to just be @@ -904,7 +963,7 @@ def start_example(self, label): self.__example_record.start_example(label) self.labels_for_structure_stack.append({label}) - def stop_example(self, discard=False): + def stop_example(self, discard: bool = False) -> None: if self.frozen: return if discard: @@ -948,17 +1007,18 @@ def stop_example(self, discard=False): self.observer.kill_branch() - def note_event(self, event): + def note_event(self, event: Hashable) -> None: + assert isinstance(self.events, set) self.events.add(event) @property - def examples(self): + def examples(self) -> Examples: assert self.frozen if self.__examples is None: self.__examples = Examples(record=self.__example_record, blocks=self.blocks) return self.__examples - def freeze(self): + def freeze(self) -> None: if self.frozen: assert isinstance(self.buffer, bytes) return @@ -978,7 +1038,7 @@ def freeze(self): self.events = frozenset(self.events) self.observer.conclude_test(self.status, self.interesting_origin) - def draw_bits(self, n, *, forced=None): + def draw_bits(self, n: int, *, forced: Optional[int] = None) -> int: """Return an ``n``-bit integer from the underlying source of bytes. If ``forced`` is set to an integer will instead ignore the underlying source and simulate a draw as if it had @@ -996,8 +1056,10 @@ def draw_bits(self, n, *, forced=None): index = self.__bytes_drawn buf = self.__prefix[index : index + n_bytes] if len(buf) < n_bytes: + assert self.__random is not None buf += uniform(self.__random, n_bytes - len(buf)) else: + assert self.__random is not None buf = uniform(self.__random, n_bytes) buf = bytearray(buf) self.__bytes_drawn += n_bytes @@ -1015,6 +1077,7 @@ def draw_bits(self, n, *, forced=None): initial = self.index + assert isinstance(self.buffer, bytearray) self.buffer.extend(buf) self.index = len(self.buffer) @@ -1023,27 +1086,31 @@ def draw_bits(self, n, *, forced=None): self.blocks.add_endpoint(self.index) - assert bit_length(result) <= n + assert result.bit_length() <= n return result - def draw_bytes(self, n): + def draw_bytes(self, n: int) -> bytes: """Draw n bytes from the underlying source.""" return int_to_bytes(self.draw_bits(8 * n), n) - def write(self, string): + def write(self, string: bytes) -> Optional[bytes]: """Write ``string`` to the output buffer.""" self.__assert_not_frozen("write") string = bytes(string) if not string: - return + return None self.draw_bits(len(string) * 8, forced=int_from_bytes(string)) return self.buffer[-len(string) :] - def __check_capacity(self, n): + def __check_capacity(self, n: int) -> None: if self.index + n > self.max_length: self.mark_overrun() - def conclude_test(self, status, interesting_origin=None): + def conclude_test( + self, + status: Status, + interesting_origin: Optional[InterestingOrigin] = None, + ) -> None: assert (interesting_origin is None) or (status == Status.INTERESTING) self.__assert_not_frozen("conclude_test") self.interesting_origin = interesting_origin @@ -1051,7 +1118,9 @@ def conclude_test(self, status, interesting_origin=None): self.freeze() raise StopTest(self.testcounter) - def mark_interesting(self, interesting_origin=None): + def mark_interesting( + self, interesting_origin: Optional[InterestingOrigin] = None + ) -> None: self.conclude_test(Status.INTERESTING, interesting_origin) def mark_invalid(self): @@ -1061,7 +1130,7 @@ def mark_overrun(self): self.conclude_test(Status.OVERRUN) -def bits_to_bytes(n): +def bits_to_bytes(n: int) -> int: """The number of bytes required to represent an n-bit number. Equivalent to (n + 7) // 8, but slightly faster. This really is called enough times that that matters.""" diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/datatree.py b/hypothesis-python/src/hypothesis/internal/conjecture/datatree.py index 3def58a4da..e61f33732a 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/datatree.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/datatree.py @@ -1,27 +1,21 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import attr -from hypothesis.errors import Flaky, HypothesisException +from hypothesis.errors import Flaky, HypothesisException, StopTest from hypothesis.internal.compat import int_to_bytes from hypothesis.internal.conjecture.data import ( ConjectureData, DataObserver, Status, - StopTest, bits_to_bytes, ) from hypothesis.internal.conjecture.junkdrawer import IntList @@ -39,7 +33,7 @@ def inconsistent_generation(): ) -EMPTY = frozenset() +EMPTY: frozenset = frozenset() @attr.s(slots=True) @@ -73,14 +67,6 @@ class Conclusion: interesting_origin = attr.ib() -CONCLUSIONS = {} - - -def conclusion(status, interesting_origin): - result = Conclusion(status, interesting_origin) - return CONCLUSIONS.setdefault(result, result) - - @attr.s(slots=True) class TreeNode: """Node in a tree that corresponds to previous interactions with @@ -264,7 +250,7 @@ def append_int(n_bits, value): # on, hence the pragma. assert ( # pragma: no cover check_counter != 1000 - or len(branch.children) < (2 ** n_bits) + or len(branch.children) < (2**n_bits) or any(not v.is_exhausted for v in branch.children.values()) ) @@ -307,8 +293,8 @@ def simulate_test_function(self, data): v = data.draw_bits(node.transition.bit_length) try: node = node.transition.children[v] - except KeyError: - raise PreviouslyUnseenBehaviour() + except KeyError as err: + raise PreviouslyUnseenBehaviour() from err else: assert isinstance(node.transition, Killed) data.observer.kill_branch() @@ -407,7 +393,7 @@ def conclude_test(self, status, interesting_origin): if i < len(node.values) or isinstance(node.transition, Branch): inconsistent_generation() - new_transition = conclusion(status, interesting_origin) + new_transition = Conclusion(status, interesting_origin) if node.transition is not None and node.transition != new_transition: # As an, I'm afraid, horrible bodge, we deliberately ignore flakiness @@ -418,8 +404,8 @@ def conclude_test(self, status, interesting_origin): or new_transition.status != Status.VALID ): raise Flaky( - "Inconsistent test results! Test case was %r on first run but %r on second" - % (node.transition, new_transition) + f"Inconsistent test results! Test case was {node.transition!r} " + f"on first run but {new_transition!r} on second" ) else: node.transition = new_transition diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/dfa/__init__.py b/hypothesis-python/src/hypothesis/internal/conjecture/dfa/__init__.py index ea934363df..40b87f69b6 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/dfa/__init__.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/dfa/__init__.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import threading from collections import Counter, defaultdict, deque @@ -395,8 +390,8 @@ def all_matching_strings_of_length(self, k): states.append(j) path.append(c) break - else: # pragma: no cover - assert False + else: + raise NotImplementedError("Should be unreachable") assert self.is_accepting(states[-1]) assert len(states) == len(path) + 1 yield bytes(path) @@ -496,7 +491,7 @@ def equivalent(self, other): """Checks whether this DFA and other match precisely the same language. - Uses the classic algorith of Hopcroft and Karp (more or less): + Uses the classic algorithm of Hopcroft and Karp (more or less): Hopcroft, John E. A linear algorithm for testing equivalence of finite automata. Vol. 114. Defense Technical Information Center, 1971. """ diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/dfa/lstar.py b/hypothesis-python/src/hypothesis/internal/conjecture/dfa/lstar.py index 9374f9fe82..1f79e02a5c 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/dfa/lstar.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/dfa/lstar.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from bisect import bisect_right, insort from collections import Counter @@ -147,7 +142,7 @@ def __init__(self, member): # When we're trying to figure out what state a string leads to we will # end up searching to find a suitable candidate. By putting states in - # a self-organisating list we ideally minimise the number of lookups. + # a self-organising list we ideally minimise the number of lookups. self.__self_organising_states = SelfOrganisingList(self.__states) self.start = 0 @@ -280,7 +275,7 @@ def learn(self, string): # First we make sure that normalization is not the source of the # failure to match. while True: - normalized = bytes([self.normalizer.normalize(c) for c in string]) + normalized = bytes(self.normalizer.normalize(c) for c in string) # We can correctly replace the string with its normalized version # so normalization is not the problem here. if self.member(normalized) == correct_outcome: @@ -293,7 +288,7 @@ def learn(self, string): def replace(b): if a == b: return target - return bytes([b if c == a else c for c in target]) + return bytes(b if c == a else c for c in target) self.normalizer.distinguish(a, lambda x: self.member(replace(x))) target = replace(self.normalizer.normalize(a)) @@ -389,7 +384,7 @@ class LearnedDFA(DFA): distinguished by a membership test and a set of experiments.""" def __init__(self, lstar): - DFA.__init__(self) + super().__init__() self.__lstar = lstar self.__generation = lstar.generation diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py index 96135a980c..750e68edd4 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py @@ -1,23 +1,19 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import math import sys import time from collections import defaultdict from contextlib import contextmanager +from datetime import timedelta from enum import Enum from random import Random, getrandbits from weakref import WeakKeyDictionary @@ -26,6 +22,7 @@ from hypothesis import HealthCheck, Phase, Verbosity, settings as Settings from hypothesis._settings import local_settings +from hypothesis.errors import StopTest from hypothesis.internal.cache import LRUReusedCache from hypothesis.internal.compat import ceil, int_from_bytes from hypothesis.internal.conjecture.data import ( @@ -34,7 +31,6 @@ DataObserver, Overrun, Status, - StopTest, ) from hypothesis.internal.conjecture.datatree import ( DataTree, @@ -47,11 +43,6 @@ from hypothesis.internal.healthcheck import fail_health_check from hypothesis.reporting import base_report, report -# Tell pytest to omit the body of this module from tracebacks -# https://docs.pytest.org/en/latest/example/simple.html#writing-well-integrated-assertion-helpers -__tracebackhide__ = True - - MAX_SHRINKS = 500 CACHE_SIZE = 10000 MUTATION_POOL_SIZE = 100 @@ -298,12 +289,12 @@ def test_function(self, data): ): # See https://github.com/HypothesisWorks/hypothesis/issues/2340 report( - "WARNING: Hypothesis has spent more than five minutes working to shrink " - "a failing example, and stopped because it is making very slow " - "progress. When you re-run your tests, shrinking will resume and " - "may take this long before aborting again.\n" - "PLEASE REPORT THIS if you can provide a reproducing example, so that " - "we can improve shrinking performance for everyone." + "WARNING: Hypothesis has spent more than five minutes working to shrink" + " a failing example, and stopped because it is making very slow" + " progress. When you re-run your tests, shrinking will resume and may" + " take this long before aborting again.\nPLEASE REPORT THIS if you can" + " provide a reproducing example, so that we can improve shrinking" + " performance for everyone." ) self.exit_with(ExitReason.very_slow_shrinking) @@ -374,51 +365,39 @@ def record_for_health_check(self, data): if state.overrun_examples == max_overrun_draws: fail_health_check( self.settings, - ( - "Examples routinely exceeded the max allowable size. " - "(%d examples overran while generating %d valid ones)" - ". Generating examples this large will usually lead to" - " bad results. You could try setting max_size parameters " - "on your collections and turning " - "max_leaves down on recursive() calls." - ) - % (state.overrun_examples, state.valid_examples), + "Examples routinely exceeded the max allowable size. " + f"({state.overrun_examples} examples overran while generating " + f"{state.valid_examples} valid ones). Generating examples this large " + "will usually lead to bad results. You could try setting max_size " + "parameters on your collections and turning max_leaves down on " + "recursive() calls.", HealthCheck.data_too_large, ) if state.invalid_examples == max_invalid_draws: fail_health_check( self.settings, - ( - "It looks like your strategy is filtering out a lot " - "of data. Health check found %d filtered examples but " - "only %d good ones. This will make your tests much " - "slower, and also will probably distort the data " - "generation quite a lot. You should adapt your " - "strategy to filter less. This can also be caused by " - "a low max_leaves parameter in recursive() calls" - ) - % (state.invalid_examples, state.valid_examples), + "It looks like your strategy is filtering out a lot of data. Health " + f"check found {state.invalid_examples} filtered examples but only " + f"{state.valid_examples} good ones. This will make your tests much " + "slower, and also will probably distort the data generation quite a " + "lot. You should adapt your strategy to filter less. This can also " + "be caused by a low max_leaves parameter in recursive() calls", HealthCheck.filter_too_much, ) draw_time = sum(state.draw_times) - if draw_time > 1.0: + # Allow at least the greater of one second or 5x the deadline. If deadline + # is None, allow 30s - the user can disable the healthcheck too if desired. + draw_time_limit = 5 * (self.settings.deadline or timedelta(seconds=6)) + if draw_time > max(1.0, draw_time_limit.total_seconds()): fail_health_check( self.settings, - ( - "Data generation is extremely slow: Only produced " - "%d valid examples in %.2f seconds (%d invalid ones " - "and %d exceeded maximum size). Try decreasing " - "size of the data you're generating (with e.g." - "max_size or max_leaves parameters)." - ) - % ( - state.valid_examples, - draw_time, - state.invalid_examples, - state.overrun_examples, - ), + "Data generation is extremely slow: Only produced " + f"{state.valid_examples} valid examples in {draw_time:.2f} seconds " + f"({state.invalid_examples} invalid ones and {state.overrun_examples} " + "exceeded maximum size). Try decreasing size of the data you're " + "generating (with e.g. max_size or max_leaves parameters).", HealthCheck.too_slow, ) @@ -602,8 +581,12 @@ def should_generate_more(self): # the run. if not self.interesting_examples: return True + # Users who disable shrinking probably want to exit as fast as possible. # If we've found a bug and won't report more than one, stop looking. - elif not self.settings.report_multiple_bugs: + elif ( + Phase.shrink not in self.settings.phases + or not self.settings.report_multiple_bugs + ): return False assert self.first_bug_found_at <= self.last_bug_found_at <= self.call_count # Otherwise, keep searching for between ten and 'a heuristic' calls. @@ -799,9 +782,9 @@ def generate_mutations_from(self, data): group = self.random.choice(groups) - ex1, ex2 = [ + ex1, ex2 = ( data.examples[i] for i in sorted(self.random.sample(group, 2)) - ] + ) assert ex1.end <= ex2.start replacements = [data.buffer[e.start : e.end] for e in [ex1, ex2]] @@ -1084,10 +1067,13 @@ def event_to_string(self, event): return event try: return self.events_to_strings[event] - except KeyError: + except (KeyError, TypeError): pass result = str(event) - self.events_to_strings[event] = result + try: + self.events_to_strings[event] = result + except TypeError: + pass return result diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/floats.py b/hypothesis-python/src/hypothesis/internal/conjecture/floats.py index a18860b617..5608570130 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/floats.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/floats.py @@ -1,23 +1,22 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from array import array +from typing import TYPE_CHECKING, Optional from hypothesis.internal.conjecture.utils import calc_label_from_name from hypothesis.internal.floats import float_to_int, int_to_float +if TYPE_CHECKING: + from hypothesis.internal.conjecture.data import ConjectureData + """ This module implements support for arbitrary floating point numbers in Conjecture. It doesn't make any attempt to get a good distribution, only to @@ -81,15 +80,13 @@ MAX_EXPONENT = 0x7FF -SPECIAL_EXPONENTS = (0, MAX_EXPONENT) - BIAS = 1023 MAX_POSITIVE_EXPONENT = MAX_EXPONENT - 1 - BIAS DRAW_FLOAT_LABEL = calc_label_from_name("drawing a float") -def exponent_key(e): +def exponent_key(e: int) -> float: if e == MAX_EXPONENT: return float("inf") unbiased = e - BIAS @@ -108,21 +105,21 @@ def exponent_key(e): del i, b -def decode_exponent(e): +def decode_exponent(e: int) -> int: """Take draw_bits(11) and turn it into a suitable floating point exponent such that lexicographically simpler leads to simpler floats.""" assert 0 <= e <= MAX_EXPONENT return ENCODING_TABLE[e] -def encode_exponent(e): +def encode_exponent(e: int) -> int: """Take a floating point exponent and turn it back into the equivalent result from conjecture.""" assert 0 <= e <= MAX_EXPONENT return DECODING_TABLE[e] -def reverse_byte(b): +def reverse_byte(b: int) -> int: result = 0 for _ in range(8): result <<= 1 @@ -138,7 +135,7 @@ def reverse_byte(b): REVERSE_BITS_TABLE = bytearray(map(reverse_byte, range(256))) -def reverse64(v): +def reverse64(v: int) -> int: """Reverse a 64-bit integer bitwise. We do this by breaking it up into 8 bytes. The 64-bit integer is then the @@ -165,14 +162,14 @@ def reverse64(v): MANTISSA_MASK = (1 << 52) - 1 -def reverse_bits(x, n): +def reverse_bits(x: int, n: int) -> int: assert x.bit_length() <= n <= 64 x = reverse64(x) x >>= 64 - n return x -def update_mantissa(unbiased_exponent, mantissa): +def update_mantissa(unbiased_exponent: int, mantissa: int) -> int: if unbiased_exponent <= 0: mantissa = reverse_bits(mantissa, 52) elif unbiased_exponent <= 51: @@ -183,7 +180,7 @@ def update_mantissa(unbiased_exponent, mantissa): return mantissa -def lex_to_float(i): +def lex_to_float(i: int) -> float: assert i.bit_length() <= 64 has_fractional_part = i >> 63 if has_fractional_part: @@ -200,14 +197,14 @@ def lex_to_float(i): return float(integral_part) -def float_to_lex(f): +def float_to_lex(f: float) -> int: if is_simple(f): assert f >= 0 return int(f) return base_float_to_lex(f) -def base_float_to_lex(f): +def base_float_to_lex(f: float) -> int: i = float_to_int(f) i &= (1 << 63) - 1 exponent = i >> 52 @@ -219,7 +216,7 @@ def base_float_to_lex(f): return (1 << 63) | (exponent << 52) | mantissa -def is_simple(f): +def is_simple(f: float) -> int: try: i = int(f) except (ValueError, OverflowError): @@ -229,18 +226,17 @@ def is_simple(f): return i.bit_length() <= 56 -def draw_float(data): +def draw_float(data: "ConjectureData", forced_sign_bit: Optional[int] = None) -> float: try: data.start_example(DRAW_FLOAT_LABEL) + is_negative = data.draw_bits(1, forced=forced_sign_bit) f = lex_to_float(data.draw_bits(64)) - if data.draw_bits(1): - f = -f - return f + return -f if is_negative else f finally: data.stop_example() -def write_float(data, f): - data.draw_bits(64, forced=float_to_lex(abs(f))) +def write_float(data: "ConjectureData", f: float) -> None: sign = float_to_int(f) >> 63 data.draw_bits(1, forced=sign) + data.draw_bits(64, forced=float_to_lex(abs(f))) diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/junkdrawer.py b/hypothesis-python/src/hypothesis/internal/conjecture/junkdrawer.py index d1191d3a71..dc3d0a0f9c 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/junkdrawer.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/junkdrawer.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """A module for miscellaneous useful bits and bobs that don't obviously belong anywhere else. If you spot a better home for @@ -19,15 +14,37 @@ import array import sys +from random import Random +from typing import ( + Callable, + Dict, + Generic, + Iterable, + Iterator, + List, + Optional, + Sequence, + Tuple, + TypeVar, + Union, + overload, +) + +ARRAY_CODES = ["B", "H", "I", "L", "Q", "O"] -def array_or_list(code, contents): +def array_or_list( + code: str, contents: Iterable[int] +) -> "Union[List[int], array.ArrayType[int]]": if code == "O": return list(contents) return array.array(code, contents) -def replace_all(buffer, replacements): +def replace_all( + buffer: Sequence[int], + replacements: Iterable[Tuple[int, int, Sequence[int]]], +) -> bytes: """Substitute multiple replacement values into a buffer. Replacements is a list of (start, end, value) triples. @@ -46,11 +63,10 @@ def replace_all(buffer, replacements): return bytes(result) -ARRAY_CODES = ["B", "H", "I", "L", "Q", "O"] NEXT_ARRAY_CODE = dict(zip(ARRAY_CODES, ARRAY_CODES[1:])) -class IntList: +class IntList(Sequence[int]): """Class for storing a list of non-negative integers compactly. We store them as the smallest size integer array we can get @@ -60,67 +76,78 @@ class IntList: __slots__ = ("__underlying",) - def __init__(self, values=()): + __underlying: "Union[List[int], array.ArrayType[int]]" + + def __init__(self, values: Sequence[int] = ()): for code in ARRAY_CODES: try: - self.__underlying = array_or_list(code, values) + underlying = array_or_list(code, values) break except OverflowError: pass else: # pragma: no cover raise AssertionError(f"Could not create storage for {values!r}") - if isinstance(self.__underlying, list): - for v in self.__underlying: - if v < 0 or not isinstance(v, int): + if isinstance(underlying, list): + for v in underlying: + if not isinstance(v, int) or v < 0: raise ValueError(f"Could not create IntList for {values!r}") + self.__underlying = underlying @classmethod - def of_length(self, n): - return IntList(array_or_list("B", [0]) * n) + def of_length(cls, n: int) -> "IntList": + return cls(array_or_list("B", [0]) * n) - def count(self, n): - return self.__underlying.count(n) + def count(self, value: int) -> int: + return self.__underlying.count(value) def __repr__(self): - return f"IntList({list(self)!r})" + return f"IntList({list(self.__underlying)!r})" def __len__(self): return len(self.__underlying) - def __getitem__(self, i): + @overload + def __getitem__(self, i: int) -> int: + ... # pragma: no cover + + @overload + def __getitem__(self, i: slice) -> "IntList": + ... # pragma: no cover + + def __getitem__(self, i: Union[int, slice]) -> "Union[int, IntList]": if isinstance(i, slice): return IntList(self.__underlying[i]) return self.__underlying[i] - def __delitem__(self, i): + def __delitem__(self, i: int) -> None: del self.__underlying[i] - def insert(self, i, v): + def insert(self, i: int, v: int) -> None: self.__underlying.insert(i, v) - def __iter__(self): + def __iter__(self) -> Iterator[int]: return iter(self.__underlying) - def __eq__(self, other): + def __eq__(self, other: object) -> bool: if self is other: return True if not isinstance(other, IntList): return NotImplemented return self.__underlying == other.__underlying - def __ne__(self, other): + def __ne__(self, other: object) -> bool: if self is other: return False if not isinstance(other, IntList): return NotImplemented return self.__underlying != other.__underlying - def append(self, n): + def append(self, n: int) -> None: i = len(self) self.__underlying.append(0) self[i] = n - def __setitem__(self, i, n): + def __setitem__(self, i: int, n: int) -> None: while True: try: self.__underlying[i] = n @@ -129,16 +156,17 @@ def __setitem__(self, i, n): assert n > 0 self.__upgrade() - def extend(self, ls): + def extend(self, ls: Iterable[int]) -> None: for n in ls: self.append(n) - def __upgrade(self): + def __upgrade(self) -> None: + assert isinstance(self.__underlying, array.array) code = NEXT_ARRAY_CODE[self.__underlying.typecode] self.__underlying = array_or_list(code, self.__underlying) -def binary_search(lo, hi, f): +def binary_search(lo: int, hi: int, f: Callable[[int], bool]) -> int: """Binary searches in [lo , hi) to find n such that f(n) == f(lo) but f(n + 1) != f(lo). It is implicitly assumed and will not be checked @@ -156,11 +184,14 @@ def binary_search(lo, hi, f): return lo -def uniform(random, n): +def uniform(random: Random, n: int) -> bytes: """Returns a bytestring of length n, distributed uniformly at random.""" return random.getrandbits(n * 8).to_bytes(n, "big") +T = TypeVar("T") + + class LazySequenceCopy: """A "copy" of a sequence that works by inserting a mask in front of the underlying sequence, so that you can mutate it without changing @@ -168,15 +199,17 @@ class LazySequenceCopy: in O(1) time. The full list API is not supported yet but there's no reason in principle it couldn't be.""" - def __init__(self, values): + __mask: Optional[Dict[int, int]] + + def __init__(self, values: Sequence[int]): self.__values = values self.__len = len(values) self.__mask = None - def __len__(self): + def __len__(self) -> int: return self.__len - def pop(self): + def pop(self) -> int: if len(self) == 0: raise IndexError("Cannot pop from empty list") result = self[-1] @@ -185,7 +218,7 @@ def pop(self): self.__mask.pop(self.__len, None) return result - def __getitem__(self, i): + def __getitem__(self, i: int) -> int: i = self.__check_index(i) default = self.__values[i] if self.__mask is None: @@ -193,13 +226,13 @@ def __getitem__(self, i): else: return self.__mask.get(i, default) - def __setitem__(self, i, v): + def __setitem__(self, i: int, v: int) -> None: i = self.__check_index(i) if self.__mask is None: self.__mask = {} self.__mask[i] = v - def __check_index(self, i): + def __check_index(self, i: int) -> int: n = len(self) if i < -n or i >= n: raise IndexError(f"Index {i} out of range [0, {n})") @@ -209,20 +242,20 @@ def __check_index(self, i): return i -def clamp(lower, value, upper): +def clamp(lower: int, value: int, upper: int) -> int: """Given a value and lower/upper bounds, 'clamp' the value so that it satisfies lower <= value <= upper.""" return max(lower, min(value, upper)) -def swap(ls, i, j): +def swap(ls: LazySequenceCopy, i: int, j: int) -> None: """Swap the elements ls[i], ls[j].""" if i == j: return ls[i], ls[j] = ls[j], ls[i] -def stack_depth_of_caller(): +def stack_depth_of_caller() -> int: """Get stack size for caller's frame. From https://stackoverflow.com/a/47956089/9297601 , this is a simple @@ -233,12 +266,12 @@ def stack_depth_of_caller(): frame = sys._getframe(2) size = 1 while frame: - frame = frame.f_back + frame = frame.f_back # type: ignore[assignment] size += 1 return size -def find_integer(f): +def find_integer(f: Callable[[int], bool]) -> int: """Finds a (hopefully large) integer such that f(n) is True and f(n + 1) is False. @@ -276,7 +309,7 @@ def find_integer(f): return lo -def pop_random(random, seq): +def pop_random(random: Random, seq: LazySequenceCopy) -> int: """Remove and return a random element of seq. This runs in O(1) but leaves the sequence in an arbitrary order.""" i = random.randrange(0, len(seq)) @@ -288,7 +321,7 @@ class NotFound(Exception): pass -class SelfOrganisingList: +class SelfOrganisingList(Generic[T]): """A self-organising list with the move-to-front heuristic. A self-organising list is a collection which we want to retrieve items @@ -305,17 +338,17 @@ class SelfOrganisingList: """ - def __init__(self, values=()): + def __init__(self, values: Iterable[T] = ()) -> None: self.__values = list(values) - def __repr__(self): + def __repr__(self) -> str: return f"SelfOrganisingList({self.__values!r})" - def add(self, value): + def add(self, value: T) -> None: """Add a value to this list.""" self.__values.append(value) - def find(self, condition): + def find(self, condition: Callable[[T], bool]) -> T: """Returns some value in this list such that ``condition(value)`` is True. If no such value exists raises ``NotFound``.""" for i in range(len(self.__values) - 1, -1, -1): diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/optimiser.py b/hypothesis-python/src/hypothesis/internal/conjecture/optimiser.py index a3af0d8224..4594979199 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/optimiser.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/optimiser.py @@ -1,22 +1,18 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis.internal.compat import int_from_bytes, int_to_bytes from hypothesis.internal.conjecture.data import Status -from hypothesis.internal.conjecture.engine import BUFFER_SIZE, NO_SCORE +from hypothesis.internal.conjecture.engine import BUFFER_SIZE from hypothesis.internal.conjecture.junkdrawer import find_integer +from hypothesis.internal.conjecture.pareto import NO_SCORE class Optimiser: diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/pareto.py b/hypothesis-python/src/hypothesis/internal/conjecture/pareto.py index 9e1507e685..d82408f97e 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/pareto.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/pareto.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from enum import Enum @@ -54,7 +49,7 @@ def dominance(left, right): return DominanceRelation.EQUAL if sort_key(right.buffer) < sort_key(left.buffer): - result = dominance(right, left) + result = dominance(left=right, right=left) if result == DominanceRelation.LEFT_DOMINATES: return DominanceRelation.RIGHT_DOMINATES else: diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py b/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py index b352a6bb2a..2ee5d5f4cb 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py @@ -1,19 +1,15 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from collections import defaultdict +from typing import Dict import attr @@ -35,10 +31,7 @@ replace_all, ) from hypothesis.internal.conjecture.shrinking import Float, Integer, Lexical, Ordering -from hypothesis.internal.conjecture.shrinking.dfas import SHRINKING_DFAS - -if False: - from typing import Dict # noqa +from hypothesis.internal.conjecture.shrinking.learned_dfas import SHRINKING_DFAS def sort_key(buffer): @@ -67,7 +60,7 @@ def sort_key(buffer): return (len(buffer), buffer) -SHRINK_PASS_DEFINITIONS = {} # type: Dict[str, ShrinkPassDefinition] +SHRINK_PASS_DEFINITIONS: Dict[str, "ShrinkPassDefinition"] = {} @attr.s() @@ -103,8 +96,8 @@ def defines_shrink_pass(): def accept(run_step): ShrinkPassDefinition(run_with_chooser=run_step) - def run(self): # pragma: no cover - raise AssertionError("Shrink passes should not be run directly") + def run(self): + raise NotImplementedError("Shrink passes should not be run directly") run.__name__ = run_step.__name__ run.is_shrink_pass = True @@ -222,7 +215,7 @@ class Shrinker: The two easiest ways to do this are: * Just run the N steps in random order. As long as a - reasonably large proportion of the operations suceed, this + reasonably large proportion of the operations succeed, this guarantees the expected stall length is quite short. The book keeping for making sure this does the right thing when it succeeds can be quite annoying. @@ -248,7 +241,7 @@ class Shrinker: """ - def derived_value(fn): + def derived_value(fn): # noqa: B902 """It's useful during shrinking to have access to derived values of the current shrink target. @@ -285,8 +278,8 @@ def __init__(self, engine, initial, predicate, allow_transition): # We keep track of the current best example on the shrink_target # attribute. - self.shrink_target = None - self.update_shrink_target(initial) + self.shrink_target = initial + self.clear_change_tracking() self.shrinks = 0 # We terminate shrinks that seem to have reached their logical @@ -304,7 +297,7 @@ def __init__(self, engine, initial, predicate, allow_transition): # testing and learning purposes. self.extra_dfas = {} - @derived_value + @derived_value # type: ignore def cached_calculations(self): return {} @@ -340,7 +333,7 @@ def shrink_pass(self, name): self.add_new_pass(name) return self.passes_by_name[name] - @derived_value + @derived_value # type: ignore def match_cache(self): return {} @@ -454,25 +447,15 @@ def s(n): return "s" if n != 1 else "" total_deleted = self.initial_size - len(self.shrink_target.buffer) - - self.debug("---------------------") - self.debug("Shrink pass profiling") - self.debug("---------------------") - self.debug("") calls = self.engine.call_count - self.initial_calls + self.debug( - ( - "Shrinking made a total of %d call%s " - "of which %d shrank. This deleted %d byte%s out of %d." - ) - % ( - calls, - s(calls), - self.shrinks, - total_deleted, - s(total_deleted), - self.initial_size, - ) + "---------------------\n" + "Shrink pass profiling\n" + "---------------------\n\n" + f"Shrinking made a total of {calls} call{s(calls)} of which " + f"{self.shrinks} shrank. This deleted {total_deleted} bytes out " + f"of {self.initial_size}." ) for useful in [True, False]: self.debug("") @@ -490,10 +473,8 @@ def s(n): continue self.debug( - ( - " * %s made %d call%s of which " - "%d shrank, deleting %d byte%s." - ) + " * %s made %d call%s of which " + "%d shrank, deleting %d byte%s." % ( p.name, p.calls, @@ -532,7 +513,7 @@ def greedy_shrink(self): + [dfa_replacement(n) for n in SHRINKING_DFAS] ) - @derived_value + @derived_value # type: ignore def shrink_pass_choice_trees(self): return defaultdict(ChoiceTree) @@ -647,7 +628,7 @@ def examples(self): def all_block_bounds(self): return self.shrink_target.blocks.all_bounds() - @derived_value + @derived_value # type: ignore def examples_by_label(self): """An index of all examples grouped by their label, with the examples stored in their normal index order.""" @@ -657,7 +638,7 @@ def examples_by_label(self): examples_by_label[ex.label].append(ex) return dict(examples_by_label) - @derived_value + @derived_value # type: ignore def distinct_labels(self): return sorted(self.examples_by_label, key=str) @@ -839,22 +820,17 @@ def __changed_blocks(self): def update_shrink_target(self, new_target): assert isinstance(new_target, ConjectureResult) - if self.shrink_target is not None: - self.shrinks += 1 - # If we are just taking a long time to shrink we don't want to - # trigger this heuristic, so whenever we shrink successfully - # we give ourselves a bit of breathing room to make sure we - # would find a shrink that took that long to find the next time. - # The case where we're taking a long time but making steady - # progress is handled by `finish_shrinking_deadline` in engine.py - self.max_stall = max( - self.max_stall, (self.calls - self.calls_at_last_shrink) * 2 - ) - self.calls_at_last_shrink = self.calls - else: - self.__all_changed_blocks = set() - self.__last_checked_changed_at = new_target - + self.shrinks += 1 + # If we are just taking a long time to shrink we don't want to + # trigger this heuristic, so whenever we shrink successfully + # we give ourselves a bit of breathing room to make sure we + # would find a shrink that took that long to find the next time. + # The case where we're taking a long time but making steady + # progress is handled by `finish_shrinking_deadline` in engine.py + self.max_stall = max( + self.max_stall, (self.calls - self.calls_at_last_shrink) * 2 + ) + self.calls_at_last_shrink = self.calls self.shrink_target = new_target self.__derived_values = {} @@ -1009,7 +985,7 @@ def remove_discarded(self): return False return True - @derived_value + @derived_value # type: ignore def blocks_by_non_zero_suffix(self): """Returns a list of blocks grouped by their non-zero suffix, as a list of (suffix, indices) pairs, skipping all groupings @@ -1024,7 +1000,7 @@ def blocks_by_non_zero_suffix(self): ) return duplicates - @derived_value + @derived_value # type: ignore def duplicated_block_suffixes(self): return sorted(self.blocks_by_non_zero_suffix) @@ -1083,12 +1059,12 @@ def minimize_floats(self, chooser): lambda ex: ( ex.label == DRAW_FLOAT_LABEL and len(ex.children) == 2 - and ex.children[0].length == 8 + and ex.children[1].length == 8 ), ) - u = ex.children[0].start - v = ex.children[0].end + u = ex.children[1].start + v = ex.children[1].end buf = self.shrink_target.buffer b = buf[u:v] f = lex_to_float(int_from_bytes(b)) @@ -1329,8 +1305,8 @@ def run_block_program(self, i, description, original, repeats=1): attempt[u:v] = int_to_bytes(value - 1, v - u) elif d == "X": del attempt[u:v] - else: # pragma: no cover - raise AssertionError(f"Unrecognised command {d!r}") + else: + raise NotImplementedError(f"Unrecognised command {d!r}") return self.incorporate_new_buffer(attempt) diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/__init__.py b/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/__init__.py index 6b92094350..556e77461e 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/__init__.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/__init__.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis.internal.conjecture.shrinking.floats import Float from hypothesis.internal.conjecture.shrinking.integer import Integer diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/common.py b/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/common.py index 8db4f88aa3..24df7b90ab 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/common.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/common.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """This module implements various useful common functions for shrinking tasks.""" diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/dfas.py b/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/dfas.py index 848feb9538..8fb42a33b7 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/dfas.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/dfas.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import hashlib import math @@ -238,6 +233,7 @@ def normalize( required_successes=100, allowed_to_update=False, max_dfas=10, + random=None, ): """Attempt to ensure that this test function successfully normalizes - i.e. whenever it declares a test case to be interesting, we are able @@ -267,6 +263,7 @@ def normalize( test_function, settings=settings(database=None, suppress_health_check=HealthCheck.all()), ignore_limits=True, + random=random, ) seen = set() @@ -310,14 +307,14 @@ def shrinking_predicate(d): if not allowed_to_update: raise FailedToNormalise( - "Shrinker failed to normalize %r to %r and we are not allowed to learn new DFAs." - % (previous.buffer, current.buffer) + f"Shrinker failed to normalize {previous.buffer!r} to " + f"{current.buffer!r} and we are not allowed to learn new DFAs." ) if dfas_added >= max_dfas: raise FailedToNormalise( - "Test function is too hard to learn: Added %d DFAs and still not done." - % (dfas_added,) + f"Test function is too hard to learn: Added {dfas_added} " + "DFAs and still not done." ) dfas_added += 1 @@ -326,11 +323,7 @@ def shrinking_predicate(d): runner, previous.buffer, current.buffer, shrinking_predicate ) - name = ( - base_name - + "-" - + hashlib.sha256(repr(new_dfa).encode("utf-8")).hexdigest()[:10] - ) + name = base_name + "-" + hashlib.sha256(repr(new_dfa).encode()).hexdigest()[:10] # If there is a name collision this DFA should already be being # used for shrinking, so we should have already been able to shrink diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/floats.py b/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/floats.py index 3180e4ea7d..ceab3f5f0f 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/floats.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/floats.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import math import sys @@ -20,7 +15,7 @@ from hypothesis.internal.conjecture.shrinking.common import Shrinker from hypothesis.internal.conjecture.shrinking.integer import Integer -MAX_PRECISE_INTEGER = 2 ** 53 +MAX_PRECISE_INTEGER = 2**53 class Float(Shrinker): diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/integer.py b/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/integer.py index ab0f82c192..06ba9c0564 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/integer.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/integer.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis.internal.conjecture.junkdrawer import find_integer from hypothesis.internal.conjecture.shrinking.common import Shrinker diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/learned_dfas.py b/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/learned_dfas.py index 8e5aa83507..3a414de534 100755 --- a/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/learned_dfas.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/learned_dfas.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis.internal.conjecture.dfa import ConcreteDFA diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/lexical.py b/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/lexical.py index 23c792bde5..c755f456b4 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/lexical.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/lexical.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis.internal.compat import int_from_bytes, int_to_bytes from hypothesis.internal.conjecture.shrinking.common import Shrinker diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/ordering.py b/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/ordering.py index ece2b42b2d..f112308f5c 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/ordering.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/shrinking/ordering.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis.internal.conjecture.junkdrawer import find_integer from hypothesis.internal.conjecture.shrinking.common import Shrinker diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/utils.py b/hypothesis-python/src/hypothesis/internal/conjecture/utils.py index 0160f22648..0133ceaaf4 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/utils.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/utils.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import enum import hashlib @@ -19,30 +14,30 @@ import math import sys from collections import OrderedDict, abc +from functools import lru_cache +from typing import TYPE_CHECKING, List, Optional, Sequence, Tuple, Type, TypeVar, Union from hypothesis.errors import InvalidArgument -from hypothesis.internal.compat import ( - bit_length, - floor, - int_from_bytes, - qualname, - str_to_bytes, -) -from hypothesis.internal.floats import int_to_float +from hypothesis.internal.compat import floor, int_from_bytes +from hypothesis.internal.floats import int_to_float, next_up + +if TYPE_CHECKING: + from hypothesis.internal.conjecture.data import ConjectureData + -LABEL_MASK = 2 ** 64 - 1 +LABEL_MASK = 2**64 - 1 -def calc_label_from_name(name): - hashed = hashlib.sha384(str_to_bytes(name)).digest() +def calc_label_from_name(name: str) -> int: + hashed = hashlib.sha384(name.encode()).digest() return int_from_bytes(hashed[:8]) -def calc_label_from_cls(cls): - return calc_label_from_name(qualname(cls)) +def calc_label_from_cls(cls: type) -> int: + return calc_label_from_name(cls.__qualname__) -def combine_labels(*labels): +def combine_labels(*labels: int) -> int: label = 0 for l in labels: label = (label << 1) & LABEL_MASK @@ -53,12 +48,29 @@ def combine_labels(*labels): INTEGER_RANGE_DRAW_LABEL = calc_label_from_name("another draw in integer_range()") BIASED_COIN_LABEL = calc_label_from_name("biased_coin()") BIASED_COIN_INNER_LABEL = calc_label_from_name("inside biased_coin()") -SAMPLE_IN_SAMPLER_LABLE = calc_label_from_name("a sample() in Sampler") +SAMPLE_IN_SAMPLER_LABEL = calc_label_from_name("a sample() in Sampler") ONE_FROM_MANY_LABEL = calc_label_from_name("one more from many()") -def integer_range(data, lower, upper, center=None): +def unbounded_integers(data: "ConjectureData") -> int: + size = INT_SIZES[INT_SIZES_SAMPLER.sample(data)] + r = data.draw_bits(size) + sign = r & 1 + r >>= 1 + if sign: + r = -r + return int(r) + + +def integer_range( + data: "ConjectureData", + lower: int, + upper: int, + center: Optional[int] = None, + forced: Optional[int] = None, +) -> int: assert lower <= upper + assert forced is None or lower <= forced <= upper if lower == upper: # Write a value even when this is trivial so that when a bound depends # on other values we don't suddenly disappear when the gap shrinks to @@ -76,7 +88,8 @@ def integer_range(data, lower, upper, center=None): elif center == lower: above = True else: - above = boolean(data) + force_above = None if forced is None else forced < center + above = not data.draw_bits(1, forced=force_above) if above: gap = upper - center @@ -85,20 +98,21 @@ def integer_range(data, lower, upper, center=None): assert gap > 0 - bits = bit_length(gap) + bits = gap.bit_length() probe = gap + 1 - if bits > 24 and data.draw_bits(3): + if bits > 24 and data.draw_bits(3, forced=None if forced is None else 0): # For large ranges, we combine the uniform random distribution from draw_bits - # with the weighting scheme used by WideRangeIntStrategy with moderate chance. - # Cutoff at 2 ** 24 so unicode choice is uniform but 32bit distribution is not. - idx = Sampler([4.0, 8.0, 1.0, 1.0, 0.5]).sample(data) - sizes = [8, 16, 32, 64, 128] - bits = min(bits, sizes[idx]) + # with a weighting scheme with moderate chance. Cutoff at 2 ** 24 so that our + # choice of unicode characters is uniform but the 32bit distribution is not. + idx = INT_SIZES_SAMPLER.sample(data) + bits = min(bits, INT_SIZES[idx]) while probe > gap: data.start_example(INTEGER_RANGE_DRAW_LABEL) - probe = data.draw_bits(bits) + probe = data.draw_bits( + bits, forced=None if forced is None else abs(forced - center) + ) data.stop_example(discard=probe > gap) if above: @@ -107,42 +121,44 @@ def integer_range(data, lower, upper, center=None): result = center - probe assert lower <= result <= upper - return int(result) + assert forced is None or result == forced, (result, forced, center, above) + return result + + +T = TypeVar("T") -def check_sample(values, strategy_name): +def check_sample( + values: Union[Type[enum.Enum], Sequence[T]], strategy_name: str +) -> Sequence[T]: if "numpy" in sys.modules and isinstance(values, sys.modules["numpy"].ndarray): if values.ndim != 1: raise InvalidArgument( - ( - "Only one-dimensional arrays are supported for sampling, " - "and the given value has {ndim} dimensions (shape " - "{shape}). This array would give samples of array slices " - "instead of elements! Use np.ravel(values) to convert " - "to a one-dimensional array, or tuple(values) if you " - "want to sample slices." - ).format(ndim=values.ndim, shape=values.shape) + "Only one-dimensional arrays are supported for sampling, " + f"and the given value has {values.ndim} dimensions (shape " + f"{values.shape}). This array would give samples of array slices " + "instead of elements! Use np.ravel(values) to convert " + "to a one-dimensional array, or tuple(values) if you " + "want to sample slices." ) elif not isinstance(values, (OrderedDict, abc.Sequence, enum.EnumMeta)): raise InvalidArgument( - "Cannot sample from {values}, not an ordered collection. " - "Hypothesis goes to some length to ensure that the {strategy} " + f"Cannot sample from {values!r}, not an ordered collection. " + f"Hypothesis goes to some length to ensure that the {strategy_name} " "strategy has stable results between runs. To replay a saved " "example, the sampled values must have the same iteration order " "on every run - ruling out sets, dicts, etc due to hash " - "randomisation. Most cases can simply use `sorted(values)`, but " + "randomization. Most cases can simply use `sorted(values)`, but " "mixed types or special values such as math.nan require careful " "handling - and note that when simplifying an example, " - "Hypothesis treats earlier values as simpler.".format( - values=repr(values), strategy=strategy_name - ) + "Hypothesis treats earlier values as simpler." ) if isinstance(values, range): return values return tuple(values) -def choice(data, values): +def choice(data: "ConjectureData", values: Sequence[T]) -> T: return values[integer_range(data, 0, len(values) - 1)] @@ -150,15 +166,13 @@ def choice(data, values): FULL_FLOAT = int_to_float(FLOAT_PREFIX | ((2 << 53) - 1)) - 1 -def fractional_float(data): +def fractional_float(data: "ConjectureData") -> float: return (int_to_float(FLOAT_PREFIX | data.draw_bits(52)) - 1) / FULL_FLOAT -def boolean(data): - return bool(data.draw_bits(1)) - - -def biased_coin(data, p, *, forced=None): +def biased_coin( + data: "ConjectureData", p: float, *, forced: Optional[bool] = None +) -> bool: """Return True with probability p (assuming a uniform generator), shrinking towards False. If ``forced`` is set to a non-None value, this will always return that value but will write choices appropriate to having @@ -186,7 +200,7 @@ def biased_coin(data, p, *, forced=None): p = 0.0 bits = 1 - size = 2 ** bits + size = 2**bits data.start_example(BIASED_COIN_LABEL) while True: @@ -294,30 +308,32 @@ class Sampler: shrinking the chosen element. """ - def __init__(self, weights): + table: List[Tuple[int, int, float]] # (base_idx, alt_idx, alt_chance) + + def __init__(self, weights: Sequence[float]): n = len(weights) - self.table = [[i, None, None] for i in range(n)] + table: "list[list[int | float | None]]" = [[i, None, None] for i in range(n)] total = sum(weights) num_type = type(total) - zero = num_type(0) - one = num_type(1) + zero = num_type(0) # type: ignore + one = num_type(1) # type: ignore - small = [] - large = [] + small: "List[int]" = [] + large: "List[int]" = [] probabilities = [w / total for w in weights] - scaled_probabilities = [] + scaled_probabilities: "List[float]" = [] - for i, p in enumerate(probabilities): - scaled = p * n + for i, alternate_chance in enumerate(probabilities): + scaled = alternate_chance * n scaled_probabilities.append(scaled) if scaled == 1: - self.table[i][2] = zero + table[i][2] = zero elif scaled < 1: small.append(i) else: @@ -331,9 +347,9 @@ def __init__(self, weights): assert lo != hi assert scaled_probabilities[hi] > one - assert self.table[lo][1] is None - self.table[lo][1] = hi - self.table[lo][2] = one - scaled_probabilities[lo] + assert table[lo][1] is None + table[lo][1] = hi + table[lo][2] = one - scaled_probabilities[lo] scaled_probabilities[hi] = ( scaled_probabilities[hi] + scaled_probabilities[lo] ) - one @@ -341,27 +357,29 @@ def __init__(self, weights): if scaled_probabilities[hi] < 1: heapq.heappush(small, hi) elif scaled_probabilities[hi] == 1: - self.table[hi][2] = zero + table[hi][2] = zero else: heapq.heappush(large, hi) while large: - self.table[large.pop()][2] = zero + table[large.pop()][2] = zero while small: - self.table[small.pop()][2] = zero - - for entry in self.table: - assert entry[2] is not None - if entry[1] is None: - entry[1] = entry[0] - elif entry[1] < entry[0]: - entry[0], entry[1] = entry[1], entry[0] - entry[2] = one - entry[2] + table[small.pop()][2] = zero + + self.table: "List[Tuple[int, int, float]]" = [] + for base, alternate, alternate_chance in table: # type: ignore + assert isinstance(base, int) + assert isinstance(alternate, int) or alternate is None + if alternate is None: + self.table.append((base, base, alternate_chance)) + elif alternate < base: + self.table.append((alternate, base, one - alternate_chance)) + else: + self.table.append((base, alternate, alternate_chance)) self.table.sort() - def sample(self, data): - data.start_example(SAMPLE_IN_SAMPLER_LABLE) - i = integer_range(data, 0, len(self.table) - 1) - base, alternate, alternate_chance = self.table[i] + def sample(self, data: "ConjectureData") -> int: + data.start_example(SAMPLE_IN_SAMPLER_LABEL) + base, alternate, alternate_chance = choice(data, self.table) use_alternate = biased_coin(data, alternate_chance) data.stop_example() if use_alternate: @@ -370,6 +388,10 @@ def sample(self, data): return base +INT_SIZES = (8, 16, 32, 64, 128) +INT_SIZES_SAMPLER = Sampler((4.0, 8.0, 1.0, 1.0, 0.5)) + + class many: """Utility class for collections. Bundles up the logic we use for "should I keep drawing more values?" and handles starting and stopping examples in @@ -382,19 +404,25 @@ class many: add_stuff_to_result() """ - def __init__(self, data, min_size, max_size, average_size): + def __init__( + self, + data: "ConjectureData", + min_size: int, + max_size: Union[int, float], + average_size: Union[int, float], + ) -> None: assert 0 <= min_size <= average_size <= max_size self.min_size = min_size self.max_size = max_size self.data = data - self.stopping_value = 1 - 1.0 / (1 + average_size) + self.p_continue = _calc_p_continue(average_size - min_size, max_size - min_size) self.count = 0 self.rejections = 0 self.drawn = False self.force_stop = False self.rejected = False - def more(self): + def more(self) -> bool: """Should I draw another element to add to the collection?""" if self.drawn: self.data.stop_example(discard=self.rejected) @@ -415,7 +443,7 @@ def more(self): elif self.count >= self.max_size: forced_result = False should_continue = biased_coin( - self.data, self.stopping_value, forced=forced_result + self.data, self.p_continue, forced=forced_result ) if should_continue: @@ -439,3 +467,52 @@ def reject(self): self.data.mark_invalid() else: self.force_stop = True + + +SMALLEST_POSITIVE_FLOAT: float = next_up(0.0) or sys.float_info.min + + +@lru_cache() +def _calc_p_continue(desired_avg: float, max_size: int) -> float: + """Return the p_continue which will generate the desired average size.""" + assert desired_avg <= max_size, (desired_avg, max_size) + if desired_avg == max_size: + return 1.0 + p_continue = 1 - 1.0 / (1 + desired_avg) + if p_continue == 0 or max_size == float("inf"): + assert 0 <= p_continue < 1, p_continue + return p_continue + assert 0 < p_continue < 1, p_continue + # For small max_size, the infinite-series p_continue is a poor approximation, + # and while we can't solve the polynomial a few rounds of iteration quickly + # gets us a good approximate solution in almost all cases (sometimes exact!). + while _p_continue_to_avg(p_continue, max_size) > desired_avg: + # This is impossible over the reals, but *can* happen with floats. + p_continue -= 0.0001 + # If we've reached zero or gone negative, we want to break out of this loop, + # and do so even if we're on a system with the unsafe denormals-are-zero flag. + # We make that an explicit error in st.floats(), but here we'd prefer to + # just get somewhat worse precision on collection lengths. + if p_continue < SMALLEST_POSITIVE_FLOAT: + p_continue = SMALLEST_POSITIVE_FLOAT + break + # Let's binary-search our way to a better estimate! We tried fancier options + # like gradient descent, but this is numerically stable and works better. + hi = 1.0 + while desired_avg - _p_continue_to_avg(p_continue, max_size) > 0.01: + assert 0 < p_continue < hi, (p_continue, hi) + mid = (p_continue + hi) / 2 + if _p_continue_to_avg(mid, max_size) <= desired_avg: + p_continue = mid + else: + hi = mid + assert 0 < p_continue < 1, p_continue + assert _p_continue_to_avg(p_continue, max_size) <= desired_avg + return p_continue + + +def _p_continue_to_avg(p_continue: float, max_size: int) -> float: + """Return the average_size generated by this p_continue and max_size.""" + if p_continue >= 1: + return max_size + return (1.0 / (1 - p_continue) - 1) * (1 - p_continue**max_size) diff --git a/hypothesis-python/src/hypothesis/internal/coverage.py b/hypothesis-python/src/hypothesis/internal/coverage.py index ebfe10c113..56a7f4fd9a 100644 --- a/hypothesis-python/src/hypothesis/internal/coverage.py +++ b/hypothesis-python/src/hypothesis/internal/coverage.py @@ -1,23 +1,18 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import json import os import sys from contextlib import contextmanager -from typing import Dict, Set, Tuple +from typing import Callable, Dict, Set, Tuple, TypeVar from hypothesis.internal.reflection import proxies @@ -34,7 +29,8 @@ itself and has essentially no overhead. """ -pretty_file_name_cache = {} # type: Dict[str, str] +Func = TypeVar("Func", bound=Callable) +pretty_file_name_cache: Dict[str, str] = {} def pretty_file_name(f): @@ -44,7 +40,7 @@ def pretty_file_name(f): pass parts = f.split(os.path.sep) - if "hypothesis" in parts: + if "hypothesis" in parts: # pragma: no branch parts = parts[-parts[::-1].index("hypothesis") :] result = os.path.sep.join(parts) pretty_file_name_cache[f] = result @@ -58,7 +54,7 @@ def pretty_file_name(f): # By this point, "branch-check" should have already been deleted by the # tox config. We can't delete it here because of #1718. - written = set() # type: Set[Tuple[str, bool]] + written: Set[Tuple[str, bool]] = set() def record_branch(name, value): key = (name, value) @@ -76,11 +72,8 @@ def check_block(name, depth): # function, one for our actual caller, so we want to go two extra # stack frames up. caller = sys._getframe(depth + 2) - local_description = "%s at %s:%d" % ( - name, - pretty_file_name(caller.f_code.co_filename), - caller.f_lineno, - ) + fname = pretty_file_name(caller.f_code.co_filename) + local_description = f"{name} at {fname}:{caller.f_lineno}" try: description_stack.append(local_description) description = " in ".join(reversed(description_stack)) + " passed" @@ -97,7 +90,7 @@ def check(name): with check_block(name, 2): yield - def check_function(f): + def check_function(f: Func) -> Func: @proxies(f) def accept(*args, **kwargs): # depth of 2 because of the proxy function calling us. @@ -106,13 +99,11 @@ def accept(*args, **kwargs): return accept +else: # pragma: no cover -else: - - def check_function(f): + def check_function(f: Func) -> Func: return f - # Mypy bug: https://github.com/python/mypy/issues/4117 - @contextmanager # type: ignore + @contextmanager def check(name): yield diff --git a/hypothesis-python/src/hypothesis/internal/detection.py b/hypothesis-python/src/hypothesis/internal/detection.py index fb9f482ecc..9d3496176a 100644 --- a/hypothesis-python/src/hypothesis/internal/detection.py +++ b/hypothesis-python/src/hypothesis/internal/detection.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from types import MethodType diff --git a/hypothesis-python/src/hypothesis/internal/entropy.py b/hypothesis-python/src/hypothesis/internal/entropy.py index a9548f8e6c..d477477924 100644 --- a/hypothesis-python/src/hypothesis/internal/entropy.py +++ b/hypothesis-python/src/hypothesis/internal/entropy.py @@ -1,25 +1,47 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import contextlib +import gc import random import sys - -from hypothesis.errors import InvalidArgument - -RANDOMS_TO_MANAGE = [random] # type: list +import warnings +from itertools import count +from typing import TYPE_CHECKING, Any, Callable, Hashable, Tuple +from weakref import WeakValueDictionary + +import hypothesis.core +from hypothesis.errors import HypothesisWarning, InvalidArgument +from hypothesis.internal.compat import PYPY + +if TYPE_CHECKING: + if sys.version_info >= (3, 8): # pragma: no cover + from typing import Protocol + else: + from typing_extensions import Protocol + + # we can't use this at runtime until from_type supports + # protocols -- breaks ghostwriter tests + class RandomLike(Protocol): + seed: Callable[..., Any] + getstate: Callable[[], Any] + setstate: Callable[..., Any] + +else: # pragma: no cover + RandomLike = random.Random + +# This is effectively a WeakSet, which allows us to associate the saved states +# with their respective Random instances even as new ones are registered and old +# ones go out of scope and get garbage collected. Keys are ascending integers. +_RKEY = count() +RANDOMS_TO_MANAGE: WeakValueDictionary = WeakValueDictionary({next(_RKEY): random}) class NumpyRandomWrapper: @@ -34,26 +56,97 @@ def __init__(self): self.setstate = numpy.random.set_state -def register_random(r: random.Random) -> None: - """Register the given Random instance for management by Hypothesis. +NP_RANDOM = None + + +if not PYPY: - You can pass ``random.Random`` instances (or other objects with seed, - getstate, and setstate methods) to ``register_random(r)`` to have their - states seeded and restored in the same way as the global PRNGs from the - ``random`` and ``numpy.random`` modules. + def _get_platform_base_refcount(r: Any) -> int: + return sys.getrefcount(r) + + # Determine the number of refcounts created by function scope for + # the given platform / version of Python. + _PLATFORM_REF_COUNT = _get_platform_base_refcount(object()) +else: # pragma: no cover + # PYPY doesn't have `sys.getrefcount` + _PLATFORM_REF_COUNT = -1 + + +def register_random(r: RandomLike) -> None: + """Register (a weakref to) the given Random-like instance for management by + Hypothesis. + + You can pass instances of structural subtypes of ``random.Random`` + (i.e., objects with seed, getstate, and setstate methods) to + ``register_random(r)`` to have their states seeded and restored in the same + way as the global PRNGs from the ``random`` and ``numpy.random`` modules. All global PRNGs, from e.g. simulation or scheduling frameworks, should - be registered to prevent flaky tests. Hypothesis will ensure that the - PRNG state is consistent for all test runs, or reproducibly varied if you + be registered to prevent flaky tests. Hypothesis will ensure that the + PRNG state is consistent for all test runs, always seeding them to zero and + restoring the previous state after the test, or, reproducibly varied if you choose to use the :func:`~hypothesis.strategies.random_module` strategy. + + ``register_random`` only makes `weakrefs + `_ to ``r``, + thus ``r`` will only be managed by Hypothesis as long as it has active + references elsewhere at runtime. The pattern ``register_random(MyRandom())`` + will raise a ``ReferenceError`` to help protect users from this issue. + This check does not occur for the PyPy interpreter. See the following example for + an illustration of this issue + + .. code-block:: python + + + def my_BROKEN_hook(): + r = MyRandomLike() + + # `r` will be garbage collected after the hook resolved + # and Hypothesis will 'forget' that it was registered + register_random(r) # Hypothesis will emit a warning + + + rng = MyRandomLike() + + + def my_WORKING_hook(): + register_random(rng) """ if not (hasattr(r, "seed") and hasattr(r, "getstate") and hasattr(r, "setstate")): raise InvalidArgument(f"r={r!r} does not have all the required methods") - if r not in RANDOMS_TO_MANAGE: - RANDOMS_TO_MANAGE.append(r) - -def get_seeder_and_restorer(seed=0): + if r in RANDOMS_TO_MANAGE.values(): + return + + if not PYPY: # pragma: no branch + # PYPY does not have `sys.getrefcount` + gc.collect() + if not gc.get_referrers(r): + if sys.getrefcount(r) <= _PLATFORM_REF_COUNT: + raise ReferenceError( + f"`register_random` was passed `r={r}` which will be " + "garbage collected immediately after `register_random` creates a " + "weakref to it. This will prevent Hypothesis from managing this " + "source of RNG. See the docs for `register_random` for more " + "details." + ) + else: + warnings.warn( + HypothesisWarning( + "It looks like `register_random` was passed an object " + "that could be garbage collected immediately after " + "`register_random` creates a weakref to it. This will " + "prevent Hypothesis from managing this source of RNG. " + "See the docs for `register_random` for more details." + ) + ) + + RANDOMS_TO_MANAGE[next(_RKEY)] = r + + +def get_seeder_and_restorer( + seed: Hashable = 0, +) -> Tuple[Callable[[], None], Callable[[], None]]: """Return a pair of functions which respectively seed all and restore the state of all registered PRNGs. @@ -63,25 +156,27 @@ def get_seeder_and_restorer(seed=0): to force determinism on simulation or scheduling frameworks which avoid using the global random state. See e.g. #1709. """ - assert isinstance(seed, int) and 0 <= seed < 2 ** 32 - states = [] # type: list + assert isinstance(seed, int) and 0 <= seed < 2**32 + states: dict = {} - if "numpy" in sys.modules and not any( - isinstance(x, NumpyRandomWrapper) for x in RANDOMS_TO_MANAGE - ): - RANDOMS_TO_MANAGE.append(NumpyRandomWrapper()) + if "numpy" in sys.modules: + global NP_RANDOM + if NP_RANDOM is None: + # Protect this from garbage-collection by adding it to global scope + NP_RANDOM = RANDOMS_TO_MANAGE[next(_RKEY)] = NumpyRandomWrapper() def seed_all(): assert not states - for r in RANDOMS_TO_MANAGE: - states.append(r.getstate()) + for k, r in RANDOMS_TO_MANAGE.items(): + states[k] = r.getstate() r.seed(seed) def restore_all(): - assert len(states) == len(RANDOMS_TO_MANAGE) - for r, state in zip(RANDOMS_TO_MANAGE, states): - r.setstate(state) - del states[:] + for k, state in states.items(): + r = RANDOMS_TO_MANAGE.get(k) + if r is not None: # i.e., hasn't been garbage-collected + r.setstate(state) + states.clear() return seed_all, restore_all @@ -95,6 +190,10 @@ def deterministic_PRNG(seed=0): bad idea in principle, and breaks all kinds of independence assumptions in practice. """ + if hypothesis.core._hypothesis_global_random is None: # pragma: no cover + hypothesis.core._hypothesis_global_random = random.Random() + register_random(hypothesis.core._hypothesis_global_random) + seed_all, restore_all = get_seeder_and_restorer(seed) seed_all() try: diff --git a/hypothesis-python/src/hypothesis/internal/escalation.py b/hypothesis-python/src/hypothesis/internal/escalation.py index a23e306e96..64f214a0c5 100644 --- a/hypothesis-python/src/hypothesis/internal/escalation.py +++ b/hypothesis-python/src/hypothesis/internal/escalation.py @@ -1,18 +1,14 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER +import contextlib import os import sys import traceback @@ -24,10 +20,12 @@ from hypothesis.errors import ( DeadlineExceeded, HypothesisException, - MultipleFailures, StopTest, UnsatisfiedAssumption, + _Trimmable, ) +from hypothesis.internal.compat import BaseExceptionGroup +from hypothesis.utils.dynamicvariables import DynamicVariable def belongs_to(package): @@ -57,7 +55,7 @@ def accept(filepath): PREVENT_ESCALATION = os.getenv("HYPOTHESIS_DO_NOT_ESCALATE") == "true" -FILE_CACHE = {} # type: Dict[bytes, bool] +FILE_CACHE: Dict[bytes, bool] = {} is_hypothesis_file = belongs_to(hypothesis) @@ -69,12 +67,12 @@ def escalate_hypothesis_internal_error(): if PREVENT_ESCALATION: return - error_type, e, tb = sys.exc_info() + _, e, tb = sys.exc_info() if getattr(e, "hypothesis_internal_never_escalate", False): return - filepath = traceback.extract_tb(tb)[-1][0] + filepath = None if tb is None else traceback.extract_tb(tb)[-1][0] if is_hypothesis_file(filepath) and not isinstance( e, (HypothesisException,) + HYPOTHESIS_CONTROL_EXCEPTIONS ): @@ -88,15 +86,17 @@ def get_trimmed_traceback(exception=None): else: tb = exception.__traceback__ # Avoid trimming the traceback if we're in verbose mode, or the error - # was raised inside Hypothesis (and is not a MultipleFailures) - if hypothesis.settings.default.verbosity >= hypothesis.Verbosity.debug or ( - is_hypothesis_file(traceback.extract_tb(tb)[-1][0]) - and not isinstance(exception, MultipleFailures) + # was raised inside Hypothesis + if ( + tb is None + or hypothesis.settings.default.verbosity >= hypothesis.Verbosity.debug + or is_hypothesis_file(traceback.extract_tb(tb)[-1][0]) + and not isinstance(exception, _Trimmable) ): return tb - while tb is not None and ( + while tb.tb_next is not None and ( # If the frame is from one of our files, it's been added by Hypothesis. - is_hypothesis_file(getframeinfo(tb.tb_frame)[0]) + is_hypothesis_file(getframeinfo(tb.tb_frame).filename) # But our `@proxies` decorator overrides the source location, # so we check for an attribute it injects into the frame too. or tb.tb_frame.f_globals.get("__hypothesistracebackhide__") is True @@ -111,9 +111,12 @@ def get_interesting_origin(exception): # if report_multiple_bugs=False). We traditionally use the exception type and # location, but have extracted this logic in order to see through `except ...:` # blocks and understand the __cause__ (`raise x from y`) or __context__ that - # first raised an exception. + # first raised an exception as well as PEP-654 exception groups. tb = get_trimmed_traceback(exception) - filename, lineno, *_ = traceback.extract_tb(tb)[-1] + if tb is None: + filename, lineno = None, None + else: + filename, lineno, *_ = traceback.extract_tb(tb)[-1] return ( type(exception), filename, @@ -121,4 +124,42 @@ def get_interesting_origin(exception): # Note that if __cause__ is set it is always equal to __context__, explicitly # to support introspection when debugging, so we can use that unconditionally. get_interesting_origin(exception.__context__) if exception.__context__ else (), + # We distinguish exception groups by the inner exceptions, as for __context__ + tuple( + map(get_interesting_origin, exception.exceptions) + if isinstance(exception, BaseExceptionGroup) + else [] + ), ) + + +current_pytest_item = DynamicVariable(None) + + +def _get_exceptioninfo(): + # ExceptionInfo was moved to the top-level namespace in Pytest 7.0 + if "pytest" in sys.modules: + with contextlib.suppress(Exception): + # From Pytest 7, __init__ warns on direct calls. + return sys.modules["pytest"].ExceptionInfo.from_exc_info + if "_pytest._code" in sys.modules: # pragma: no cover # old versions only + with contextlib.suppress(Exception): + return sys.modules["_pytest._code"].ExceptionInfo + return None # pragma: no cover # coverage tests always use pytest + + +def format_exception(err, tb): + # Try using Pytest to match the currently configured traceback style + ExceptionInfo = _get_exceptioninfo() + if current_pytest_item.value is not None and ExceptionInfo is not None: + item = current_pytest_item.value + return str(item.repr_failure(ExceptionInfo((type(err), err, tb)))) + "\n" + + # Or use better_exceptions, if that's installed and enabled + if "better_exceptions" in sys.modules: + better_exceptions = sys.modules["better_exceptions"] + if sys.excepthook is better_exceptions.excepthook: + return "".join(better_exceptions.format_exception(type(err), err, tb)) + + # If all else fails, use the standard-library formatting tools + return "".join(traceback.format_exception(type(err), err, tb)) diff --git a/hypothesis-python/src/hypothesis/internal/filtering.py b/hypothesis-python/src/hypothesis/internal/filtering.py new file mode 100644 index 0000000000..8f15ba9ecf --- /dev/null +++ b/hypothesis-python/src/hypothesis/internal/filtering.py @@ -0,0 +1,297 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +"""Tools for understanding predicates, to satisfy them by construction. + +For example:: + + integers().filter(lambda x: x >= 0) -> integers(min_value=0) + +This is intractable in general, but reasonably easy for simple cases involving +numeric bounds, strings with length or regex constraints, and collection lengths - +and those are precisely the most common cases. When they arise in e.g. Pandas +dataframes, it's also pretty painful to do the constructive version by hand in +a library; so we prefer to share all the implementation effort here. +See https://github.com/HypothesisWorks/hypothesis/issues/2701 for details. +""" + +import ast +import inspect +import math +import operator +from decimal import Decimal +from fractions import Fraction +from functools import partial +from typing import Any, Callable, Dict, NamedTuple, Optional, TypeVar + +from hypothesis.internal.compat import ceil, floor +from hypothesis.internal.floats import next_down, next_up +from hypothesis.internal.reflection import extract_lambda_source + +Ex = TypeVar("Ex") +Predicate = Callable[[Ex], bool] + + +class ConstructivePredicate(NamedTuple): + """Return kwargs to the appropriate strategy, and the predicate if needed. + + For example:: + + integers().filter(lambda x: x >= 0) + -> {"min_value": 0"}, None + + integers().filter(lambda x: x >= 0 and x % 7) + -> {"min_value": 0}, lambda x: x % 7 + + At least in principle - for now we usually return the predicate unchanged + if needed. + + We have a separate get-predicate frontend for each "group" of strategies; e.g. + for each numeric type, for strings, for bytes, for collection sizes, etc. + """ + + kwargs: Dict[str, Any] + predicate: Optional[Predicate] + + @classmethod + def unchanged(cls, predicate: Predicate) -> "ConstructivePredicate": + return cls({}, predicate) + + +ARG = object() + + +def convert(node: ast.AST, argname: str) -> object: + if isinstance(node, ast.Name): + if node.id != argname: + raise ValueError("Non-local variable") + return ARG + return ast.literal_eval(node) + + +def comp_to_kwargs(x: ast.AST, op: ast.AST, y: ast.AST, *, argname: str) -> dict: + a = convert(x, argname) + b = convert(y, argname) + num = (int, float) + if not (a is ARG and isinstance(b, num)) and not (isinstance(a, num) and b is ARG): + # It would be possible to work out if comparisons between two literals + # are always true or false, but it's too rare to be worth the complexity. + # (and we can't even do `arg == arg`, because what if it's NaN?) + raise ValueError("Can't analyse this comparison") + + if isinstance(op, ast.Lt): + if a is ARG: + return {"max_value": b, "exclude_max": True} + return {"min_value": a, "exclude_min": True} + elif isinstance(op, ast.LtE): + if a is ARG: + return {"max_value": b} + return {"min_value": a} + elif isinstance(op, ast.Eq): + if a is ARG: + return {"min_value": b, "max_value": b} + return {"min_value": a, "max_value": a} + elif isinstance(op, ast.GtE): + if a is ARG: + return {"min_value": b} + return {"max_value": a} + elif isinstance(op, ast.Gt): + if a is ARG: + return {"min_value": b, "exclude_min": True} + return {"max_value": a, "exclude_max": True} + raise ValueError("Unhandled comparison operator") # e.g. ast.Ne + + +def merge_preds(*con_predicates: ConstructivePredicate) -> ConstructivePredicate: + # This function is just kinda messy. Unfortunately the neatest way + # to do this is just to roll out each case and handle them in turn. + base = { + "min_value": -math.inf, + "max_value": math.inf, + "exclude_min": False, + "exclude_max": False, + } + predicate = None + for kw, p in con_predicates: + predicate = p or predicate + if "min_value" in kw: + if kw["min_value"] > base["min_value"]: + base["exclude_min"] = kw.get("exclude_min", False) + base["min_value"] = kw["min_value"] + elif kw["min_value"] == base["min_value"]: + base["exclude_min"] |= kw.get("exclude_min", False) + if "max_value" in kw: + if kw["max_value"] < base["max_value"]: + base["exclude_max"] = kw.get("exclude_max", False) + base["max_value"] = kw["max_value"] + elif kw["max_value"] == base["max_value"]: + base["exclude_max"] |= kw.get("exclude_max", False) + + if not base["exclude_min"]: + del base["exclude_min"] + if base["min_value"] == -math.inf: + del base["min_value"] + if not base["exclude_max"]: + del base["exclude_max"] + if base["max_value"] == math.inf: + del base["max_value"] + return ConstructivePredicate(base, predicate) + + +def numeric_bounds_from_ast( + tree: ast.AST, argname: str, fallback: ConstructivePredicate +) -> ConstructivePredicate: + """Take an AST; return a ConstructivePredicate. + + >>> lambda x: x >= 0 + {"min_value": 0}, None + >>> lambda x: x < 10 + {"max_value": 10, "exclude_max": True}, None + >>> lambda x: x >= y + {}, lambda x: x >= y + + See also https://greentreesnakes.readthedocs.io/en/latest/ + """ + if isinstance(tree, ast.Compare): + ops = tree.ops + vals = tree.comparators + comparisons = [(tree.left, ops[0], vals[0])] + for i, (op, val) in enumerate(zip(ops[1:], vals[1:]), start=1): + comparisons.append((vals[i - 1], op, val)) + bounds = [] + for comp in comparisons: + try: + kwargs = comp_to_kwargs(*comp, argname=argname) + bounds.append(ConstructivePredicate(kwargs, None)) + except ValueError: + bounds.append(fallback) + return merge_preds(*bounds) + + if isinstance(tree, ast.BoolOp) and isinstance(tree.op, ast.And): + return merge_preds( + *(numeric_bounds_from_ast(node, argname, fallback) for node in tree.values) + ) + + return fallback + + +def get_numeric_predicate_bounds(predicate: Predicate) -> ConstructivePredicate: + """Shared logic for understanding numeric bounds. + + We then specialise this in the other functions below, to ensure that e.g. + all the values are representable in the types that we're planning to generate + so that the strategy validation doesn't complain. + """ + unchanged = ConstructivePredicate.unchanged(predicate) + if ( + isinstance(predicate, partial) + and len(predicate.args) == 1 + and not predicate.keywords + ): + arg = predicate.args[0] + if ( + (isinstance(arg, Decimal) and Decimal.is_snan(arg)) + or not isinstance(arg, (int, float, Fraction, Decimal)) + or math.isnan(arg) + ): + return unchanged + options = { + # We're talking about op(arg, x) - the reverse of our usual intuition! + operator.lt: {"min_value": arg, "exclude_min": True}, # lambda x: arg < x + operator.le: {"min_value": arg}, # lambda x: arg <= x + operator.eq: {"min_value": arg, "max_value": arg}, # lambda x: arg == x + operator.ge: {"max_value": arg}, # lambda x: arg >= x + operator.gt: {"max_value": arg, "exclude_max": True}, # lambda x: arg > x + } + if predicate.func in options: + return ConstructivePredicate(options[predicate.func], None) + + # This section is a little complicated, but stepping through with comments should + # help to clarify it. We start by finding the source code for our predicate and + # parsing it to an abstract syntax tree; if this fails for any reason we bail out + # and fall back to standard rejection sampling (a running theme). + try: + if predicate.__name__ == "": + source = extract_lambda_source(predicate) + else: + source = inspect.getsource(predicate) + tree: ast.AST = ast.parse(source) + except Exception: + return unchanged + + # Dig down to the relevant subtree - our tree is probably a Module containing + # either a FunctionDef, or an Expr which in turn contains a lambda definition. + while isinstance(tree, ast.Module) and len(tree.body) == 1: + tree = tree.body[0] + while isinstance(tree, ast.Expr): + tree = tree.value + + if isinstance(tree, ast.Lambda) and len(tree.args.args) == 1: + return numeric_bounds_from_ast(tree.body, tree.args.args[0].arg, unchanged) + elif isinstance(tree, ast.FunctionDef) and len(tree.args.args) == 1: + if len(tree.body) != 1 or not isinstance(tree.body[0], ast.Return): + # If the body of the function is anything but `return `, + # i.e. as simple as a lambda, we can't process it (yet). + return unchanged + argname = tree.args.args[0].arg + body = tree.body[0].value + assert isinstance(body, ast.AST) + return numeric_bounds_from_ast(body, argname, unchanged) + return unchanged + + +def get_integer_predicate_bounds(predicate: Predicate) -> ConstructivePredicate: + kwargs, predicate = get_numeric_predicate_bounds(predicate) # type: ignore + + if "min_value" in kwargs: + if kwargs["min_value"] == -math.inf: + del kwargs["min_value"] + elif math.isinf(kwargs["min_value"]): + return ConstructivePredicate({"min_value": 1, "max_value": -1}, None) + elif kwargs["min_value"] != int(kwargs["min_value"]): + kwargs["min_value"] = ceil(kwargs["min_value"]) + elif kwargs.get("exclude_min", False): + kwargs["min_value"] = int(kwargs["min_value"]) + 1 + + if "max_value" in kwargs: + if kwargs["max_value"] == math.inf: + del kwargs["max_value"] + elif math.isinf(kwargs["max_value"]): + return ConstructivePredicate({"min_value": 1, "max_value": -1}, None) + elif kwargs["max_value"] != int(kwargs["max_value"]): + kwargs["max_value"] = floor(kwargs["max_value"]) + elif kwargs.get("exclude_max", False): + kwargs["max_value"] = int(kwargs["max_value"]) - 1 + + kwargs = {k: v for k, v in kwargs.items() if k in {"min_value", "max_value"}} + return ConstructivePredicate(kwargs, predicate) + + +def get_float_predicate_bounds(predicate: Predicate) -> ConstructivePredicate: + kwargs, predicate = get_numeric_predicate_bounds(predicate) # type: ignore + + if "min_value" in kwargs: + min_value = kwargs["min_value"] + kwargs["min_value"] = float(kwargs["min_value"]) + if min_value < kwargs["min_value"] or ( + min_value == kwargs["min_value"] and kwargs.get("exclude_min", False) + ): + kwargs["min_value"] = next_up(kwargs["min_value"]) + + if "max_value" in kwargs: + max_value = kwargs["max_value"] + kwargs["max_value"] = float(kwargs["max_value"]) + if max_value > kwargs["max_value"] or ( + max_value == kwargs["max_value"] and kwargs.get("exclude_max", False) + ): + kwargs["max_value"] = next_down(kwargs["max_value"]) + + kwargs = {k: v for k, v in kwargs.items() if k in {"min_value", "max_value"}} + return ConstructivePredicate(kwargs, predicate) diff --git a/hypothesis-python/src/hypothesis/internal/floats.py b/hypothesis-python/src/hypothesis/internal/floats.py index 35c820cb8d..d2cb2c94f8 100644 --- a/hypothesis-python/src/hypothesis/internal/floats.py +++ b/hypothesis-python/src/hypothesis/internal/floats.py @@ -1,20 +1,17 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import math import struct +from sys import float_info +from typing import Callable, Optional, SupportsFloat # Format codes for (int, float) sized types, used for byte-wise casts. # See https://docs.python.org/3/library/struct.html#format-characters @@ -39,19 +36,15 @@ def float_of(x, width): return reinterpret_bits(float(x), "!e", "!e") -def sign(x): +def is_negative(x: SupportsFloat) -> bool: try: - return math.copysign(1.0, x) + return math.copysign(1.0, x) < 0 except TypeError: raise TypeError( f"Expected float but got {x!r} of type {type(x).__name__}" ) from None -def is_negative(x): - return sign(x) < 0 - - def count_between_floats(x, y, width=64): assert x <= y if is_negative(x): @@ -81,7 +74,7 @@ def next_up(value, width=64): From https://stackoverflow.com/a/10426033, with thanks to Mark Dickinson. """ - assert isinstance(value, float) + assert isinstance(value, float), f"{value!r} of type {type(value)}" if math.isnan(value) or (math.isinf(value) and value > 0): return value if value == 0.0 and is_negative(value): @@ -99,3 +92,59 @@ def next_up(value, width=64): def next_down(value, width=64): return -next_up(-value, width) + + +def next_down_normal(value, width, allow_subnormal): + value = next_down(value, width) + if (not allow_subnormal) and 0 < abs(value) < width_smallest_normals[width]: + return 0.0 if value > 0 else -width_smallest_normals[width] + return value + + +def next_up_normal(value, width, allow_subnormal): + return -next_down_normal(-value, width, allow_subnormal) + + +# Smallest positive non-zero numbers that is fully representable by an +# IEEE-754 float, calculated with the width's associated minimum exponent. +# Values from https://en.wikipedia.org/wiki/IEEE_754#Basic_and_interchange_formats +width_smallest_normals = { + 16: 2 ** -(2 ** (5 - 1) - 2), + 32: 2 ** -(2 ** (8 - 1) - 2), + 64: 2 ** -(2 ** (11 - 1) - 2), +} +assert width_smallest_normals[64] == float_info.min + + +def make_float_clamper( + min_float: float = 0.0, + max_float: float = math.inf, + allow_zero: bool = False, # Allows +0.0 (even if minfloat > 0) +) -> Optional[Callable[[float], float]]: + """ + Return a function that clamps positive floats into the given bounds. + + Returns None when no values are allowed (min > max and zero is not allowed). + """ + if max_float < min_float: + if allow_zero: + min_float = max_float = 0.0 + else: + return None + + range_size = min(max_float - min_float, float_info.max) + mantissa_mask = (1 << 52) - 1 + + def float_clamper(float_val: float) -> float: + if min_float <= float_val <= max_float: + return float_val + if float_val == 0.0 and allow_zero: + return float_val + # Outside bounds; pick a new value, sampled from the allowed range, + # using the mantissa bits. + mant = float_to_int(float_val) & mantissa_mask + float_val = min_float + range_size * (mant / mantissa_mask) + # Re-enforce the bounds (just in case of floating point arithmetic error) + return max(min_float, min(max_float, float_val)) + + return float_clamper diff --git a/hypothesis-python/src/hypothesis/internal/healthcheck.py b/hypothesis-python/src/hypothesis/internal/healthcheck.py index 503fd5c019..d43742e00b 100644 --- a/hypothesis-python/src/hypothesis/internal/healthcheck.py +++ b/hypothesis-python/src/hypothesis/internal/healthcheck.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis.errors import FailedHealthCheck @@ -26,7 +21,7 @@ def fail_health_check(settings, message, label): message += ( "\nSee https://hypothesis.readthedocs.io/en/latest/health" "checks.html for more information about this. " - "If you want to disable just this health check, add %s " + f"If you want to disable just this health check, add {label} " "to the suppress_health_check settings for this test." - ) % (label,) - raise FailedHealthCheck(message, label) + ) + raise FailedHealthCheck(message) diff --git a/hypothesis-python/src/hypothesis/internal/intervalsets.py b/hypothesis-python/src/hypothesis/internal/intervalsets.py index 426ad420fd..5bdd731d2d 100644 --- a/hypothesis-python/src/hypothesis/internal/intervalsets.py +++ b/hypothesis-python/src/hypothesis/internal/intervalsets.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER class IntervalSet: diff --git a/hypothesis-python/src/hypothesis/internal/lazyformat.py b/hypothesis-python/src/hypothesis/internal/lazyformat.py index 8c131f26cf..9a728c6380 100644 --- a/hypothesis-python/src/hypothesis/internal/lazyformat.py +++ b/hypothesis-python/src/hypothesis/internal/lazyformat.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER class lazyformat: diff --git a/hypothesis-python/src/hypothesis/internal/reflection.py b/hypothesis-python/src/hypothesis/internal/reflection.py index 6f22fa0aee..736d4ede9e 100644 --- a/hypothesis-python/src/hypothesis/internal/reflection.py +++ b/hypothesis-python/src/hypothesis/internal/reflection.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """This file can approximately be considered the collection of hypothesis going to really unreasonable lengths to produce pretty output.""" @@ -21,32 +16,24 @@ import inspect import os import re +import sys +import textwrap import types from functools import wraps +from keyword import iskeyword from tokenize import detect_encoding from types import ModuleType -from typing import Callable, TypeVar - -from hypothesis.internal.compat import ( - is_typed_named_tuple, - qualname, - str_to_bytes, - to_unicode, - update_code_location, -) -from hypothesis.vendor.pretty import pretty +from typing import TYPE_CHECKING, Callable +from unittest.mock import _patch as PatchType -C = TypeVar("C", bound=Callable) -READTHEDOCS = os.environ.get("READTHEDOCS", None) == "True" +from hypothesis.internal.compat import is_typed_named_tuple, update_code_location +from hypothesis.utils.conventions import not_set +from hypothesis.vendor.pretty import pretty +if TYPE_CHECKING: + from hypothesis.strategies._internal.strategies import T -def fully_qualified_name(f): - """Returns a unique identifier for f pointing to the module it was defined - on, and an containing functions.""" - if f.__module__ is not None: - return f.__module__ + "." + qualname(f) - else: - return qualname(f) +READTHEDOCS = os.environ.get("READTHEDOCS", None) == "True" def is_mock(obj): @@ -70,19 +57,21 @@ def function_digest(function): """ hasher = hashlib.sha384() try: - hasher.update(to_unicode(inspect.getsource(function)).encode("utf-8")) + hasher.update(inspect.getsource(function).encode()) except (OSError, TypeError): pass try: - hasher.update(str_to_bytes(function.__name__)) - except AttributeError: - pass - try: - hasher.update(function.__module__.__name__.encode("utf-8")) + hasher.update(function.__name__.encode()) except AttributeError: pass try: - hasher.update(str_to_bytes(repr(inspect.getfullargspec(function)))) + # We prefer to use the modern signature API, but left this for compatibility. + # While we don't promise stability of the database, there's no advantage to + # using signature here, so we might as well keep the existing keys for now. + spec = inspect.getfullargspec(function) + if inspect.ismethod(function): + del spec.args[0] + hasher.update(repr(spec).encode()) except TypeError: pass try: @@ -92,6 +81,67 @@ def function_digest(function): return hasher.digest() +def check_signature(sig: inspect.Signature) -> None: + # Backport from Python 3.11; see https://github.com/python/cpython/pull/92065 + for p in sig.parameters.values(): + if iskeyword(p.name) and p.kind is not p.POSITIONAL_ONLY: + raise ValueError( + f"Signature {sig!r} contains a parameter named {p.name!r}, " + f"but this is a SyntaxError because `{p.name}` is a keyword. " + "You, or a library you use, must have manually created an " + "invalid signature - this will be an error in Python 3.11+" + ) + + +def get_signature(target, *, follow_wrapped=True): + # Special case for use of `@unittest.mock.patch` decorator, mimicking the + # behaviour of getfullargspec instead of reporting unusable arguments. + patches = getattr(target, "patchings", None) + if isinstance(patches, list) and all(isinstance(p, PatchType) for p in patches): + P = inspect.Parameter + return inspect.Signature( + [P("args", P.VAR_POSITIONAL), P("keywargs", P.VAR_KEYWORD)] + ) + + if isinstance(getattr(target, "__signature__", None), inspect.Signature): + # This special case covers unusual codegen like Pydantic models + sig = target.__signature__ + check_signature(sig) + # And *this* much more complicated block ignores the `self` argument + # if that's been (incorrectly) included in the custom signature. + if sig.parameters and (inspect.isclass(target) or inspect.ismethod(target)): + selfy = next(iter(sig.parameters.values())) + if ( + selfy.name == "self" + and selfy.default is inspect.Parameter.empty + and selfy.kind.name.startswith("POSITIONAL_") + ): + return sig.replace( + parameters=[v for k, v in sig.parameters.items() if k != "self"] + ) + return sig + if sys.version_info[:2] <= (3, 8) and inspect.isclass(target): + # Workaround for subclasses of typing.Generic on Python <= 3.8 + from hypothesis.strategies._internal.types import is_generic_type + + if is_generic_type(target): + sig = inspect.signature(target.__init__) + check_signature(sig) + return sig.replace( + parameters=[v for k, v in sig.parameters.items() if k != "self"] + ) + sig = inspect.signature(target, follow_wrapped=follow_wrapped) + check_signature(sig) + return sig + + +def arg_is_required(param): + return param.default is inspect.Parameter.empty and param.kind in ( + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.KEYWORD_ONLY, + ) + + def required_args(target, args=(), kwargs=()): """Return a set of names of required args to target that were not supplied in args or kwargs. @@ -105,30 +155,16 @@ def required_args(target, args=(), kwargs=()): if inspect.isclass(target) and is_typed_named_tuple(target): provided = set(kwargs) | set(target._fields[: len(args)]) return set(target._fields) - provided - # Then we try to do the right thing with inspect.getfullargspec - # Note that for classes we inspect the __init__ method, *unless* the class - # has an explicit __signature__ attribute. This allows us to support - # runtime-generated constraints on **kwargs, as for e.g. Pydantic models. + # Then we try to do the right thing with inspect.signature try: - spec = inspect.getfullargspec( - getattr(target, "__init__", target) - if inspect.isclass(target) and not hasattr(target, "__signature__") - else target - ) - except TypeError: # pragma: no cover - return None - # self appears in the argspec of __init__ and bound methods, but it's an - # error to explicitly supply it - so we might skip the first argument. - skip_self = int(inspect.isclass(target) or inspect.ismethod(target)) - # Start with the args that were not supplied and all kwonly arguments, - # then remove all positional arguments with default values, and finally - # remove kwonly defaults and any supplied keyword arguments - return ( - set(spec.args[skip_self + len(args) :] + spec.kwonlyargs) - - set(spec.args[len(spec.args) - len(spec.defaults or ()) :]) - - set(spec.kwonlydefaults or ()) - - set(kwargs) - ) + sig = get_signature(target) + except (ValueError, TypeError): + return set() + return { + name + for name, param in list(sig.parameters.items())[len(args) :] + if arg_is_required(param) and name not in kwargs + } def convert_keyword_arguments(function, args, kwargs): @@ -136,105 +172,78 @@ def convert_keyword_arguments(function, args, kwargs): passed as positional and keyword args to the function. Unless function has kwonlyargs or **kwargs the dictionary will always be empty. """ - argspec = inspect.getfullargspec(function) - new_args = [] - kwargs = dict(kwargs) - - defaults = dict(argspec.kwonlydefaults or {}) - - if argspec.defaults: - for name, value in zip( - argspec.args[-len(argspec.defaults) :], argspec.defaults - ): - defaults[name] = value - - n = max(len(args), len(argspec.args)) - - for i in range(n): - if i < len(args): - new_args.append(args[i]) - else: - arg_name = argspec.args[i] - if arg_name in kwargs: - new_args.append(kwargs.pop(arg_name)) - elif arg_name in defaults: - new_args.append(defaults[arg_name]) - else: - raise TypeError("No value provided for argument %r" % (arg_name)) - - if kwargs and not (argspec.varkw or argspec.kwonlyargs): - if len(kwargs) > 1: - raise TypeError( - "%s() got unexpected keyword arguments %s" - % (function.__name__, ", ".join(map(repr, kwargs))) - ) - else: - bad_kwarg = next(iter(kwargs)) - raise TypeError( - "%s() got an unexpected keyword argument %r" - % (function.__name__, bad_kwarg) - ) - return tuple(new_args), kwargs + sig = inspect.signature(function, follow_wrapped=False) + bound = sig.bind(*args, **kwargs) + return bound.args, bound.kwargs def convert_positional_arguments(function, args, kwargs): """Return a tuple (new_args, new_kwargs) where all possible arguments have been moved to kwargs. - new_args will only be non-empty if function has a variadic argument. + new_args will only be non-empty if function has pos-only args or *args. """ - argspec = inspect.getfullargspec(function) - new_kwargs = dict(argspec.kwonlydefaults or {}) - new_kwargs.update(kwargs) - if not argspec.varkw: - for k in new_kwargs.keys(): - if k not in argspec.args and k not in argspec.kwonlyargs: - raise TypeError( - "%s() got an unexpected keyword argument %r" - % (function.__name__, k) - ) - if len(args) < len(argspec.args): - for i in range(len(args), len(argspec.args) - len(argspec.defaults or ())): - if argspec.args[i] not in kwargs: - raise TypeError(f"No value provided for argument {argspec.args[i]}") - for kw in argspec.kwonlyargs: - if kw not in new_kwargs: - raise TypeError(f"No value provided for argument {kw}") - - if len(args) > len(argspec.args) and not argspec.varargs: - raise TypeError( - "{}() takes at most {} positional arguments ({} given)".format( - function.__name__, len(argspec.args), len(args) - ) - ) - - for arg, name in zip(args, argspec.args): - if name in new_kwargs: - raise TypeError( - f"{function.__name__}() got multiple values for keyword argument {name!r}" - ) - else: - new_kwargs[name] = arg - - return (tuple(args[len(argspec.args) :]), new_kwargs) + sig = inspect.signature(function, follow_wrapped=False) + bound = sig.bind(*args, **kwargs) + new_args = [] + new_kwargs = dict(bound.arguments) + for p in sig.parameters.values(): + if p.name in new_kwargs: + if p.kind is p.POSITIONAL_ONLY: + new_args.append(new_kwargs.pop(p.name)) + elif p.kind is p.VAR_POSITIONAL: + new_args.extend(new_kwargs.pop(p.name)) + elif p.kind is p.VAR_KEYWORD: + assert set(new_kwargs[p.name]).isdisjoint(set(new_kwargs) - {p.name}) + new_kwargs.update(new_kwargs.pop(p.name)) + return tuple(new_args), new_kwargs + + +def ast_arguments_matches_signature(args, sig): + assert isinstance(args, ast.arguments) + assert isinstance(sig, inspect.Signature) + expected = [] + for node in getattr(args, "posonlyargs", ()): # New in Python 3.8 + expected.append((node.arg, inspect.Parameter.POSITIONAL_ONLY)) + for node in args.args: + expected.append((node.arg, inspect.Parameter.POSITIONAL_OR_KEYWORD)) + if args.vararg is not None: + expected.append((args.vararg.arg, inspect.Parameter.VAR_POSITIONAL)) + for node in args.kwonlyargs: + expected.append((node.arg, inspect.Parameter.KEYWORD_ONLY)) + if args.kwarg is not None: + expected.append((args.kwarg.arg, inspect.Parameter.VAR_KEYWORD)) + return expected == [(p.name, p.kind) for p in sig.parameters.values()] + + +def is_first_param_referenced_in_function(f): + """Is the given name referenced within f?""" + try: + tree = ast.parse(textwrap.dedent(inspect.getsource(f))) + except Exception: + return True # Assume it's OK unless we know otherwise + name = list(get_signature(f).parameters)[0] + return any( + isinstance(node, ast.Name) + and node.id == name + and isinstance(node.ctx, ast.Load) + for node in ast.walk(tree) + ) -def extract_all_lambdas(tree): +def extract_all_lambdas(tree, matching_signature): lambdas = [] class Visitor(ast.NodeVisitor): def visit_Lambda(self, node): - lambdas.append(node) + if ast_arguments_matches_signature(node.args, matching_signature): + lambdas.append(node) Visitor().visit(tree) return lambdas -def args_for_lambda_ast(l): - return [n.arg for n in l.args.args] - - LINE_CONTINUATION = re.compile(r"\\\n") WHITESPACE = re.compile(r"\s+") PROBABLY_A_COMMENT = re.compile("""#[^'"]*$""") @@ -249,24 +258,10 @@ def extract_lambda_source(f): This is not a good function and I am sorry for it. Forgive me my sins, oh lord """ - argspec = inspect.getfullargspec(f) - arg_strings = [] - for a in argspec.args: - assert isinstance(a, str) - arg_strings.append(a) - if argspec.varargs: - arg_strings.append("*" + argspec.varargs) - elif argspec.kwonlyargs: - arg_strings.append("*") - for a in argspec.kwonlyargs or []: - default = (argspec.kwonlydefaults or {}).get(a) - if default: - arg_strings.append(f"{a}={default}") - else: - arg_strings.append(a) - - if arg_strings: - if_confused = "lambda {}: ".format(", ".join(arg_strings)) + sig = inspect.signature(f) + assert sig.return_annotation is inspect.Parameter.empty + if sig.parameters: + if_confused = f"lambda {str(sig)[1:-1]}: " else: if_confused = "lambda: " try: @@ -277,6 +272,8 @@ def extract_lambda_source(f): source = LINE_CONTINUATION.sub(" ", source) source = WHITESPACE.sub(" ", source) source = source.strip() + if "lambda" not in source and sys.platform == "emscripten": # pragma: no cover + return if_confused # work around Pyodide bug in inspect.getsource() assert "lambda" in source tree = None @@ -314,8 +311,7 @@ def extract_lambda_source(f): if tree is None: return if_confused - all_lambdas = extract_all_lambdas(tree) - aligned_lambdas = [l for l in all_lambdas if args_for_lambda_ast(l) == argspec.args] + aligned_lambdas = extract_all_lambdas(tree, matching_signature=sig) if len(aligned_lambdas) != 1: return if_confused lambda_ast = aligned_lambdas[0] @@ -396,6 +392,9 @@ def get_pretty_function_description(f): # their module as __self__. This might include c-extensions generally? if not (self is None or inspect.isclass(self) or inspect.ismodule(self)): return f"{self!r}.{name}" + elif isinstance(name, str) and getattr(dict, name, object()) is f: + # special case for keys/values views in from_type() / ghostwriter output + return f"dict.{name}" return name @@ -405,31 +404,27 @@ def nicerepr(v): elif isinstance(v, type): return v.__name__ else: - return pretty(v) + # With TypeVar T, show List[T] instead of TypeError on List[~T] + return re.sub(r"(\[)~([A-Z][a-z]*\])", r"\g<1>\g<2>", pretty(v)) -def arg_string(f, args, kwargs, reorder=True): +def repr_call(f, args, kwargs, reorder=True): if reorder: args, kwargs = convert_positional_arguments(f, args, kwargs) - argspec = inspect.getfullargspec(f) + bits = [nicerepr(x) for x in args] - bits = [] - - for a in argspec.args: - if a in kwargs: - bits.append("{}={}".format(a, nicerepr(kwargs.pop(a)))) + for p in get_signature(f).parameters.values(): + if p.name in kwargs and not p.kind.name.startswith("VAR_"): + bits.append(f"{p.name}={nicerepr(kwargs.pop(p.name))}") if kwargs: for a in sorted(kwargs): - bits.append("{}={}".format(a, nicerepr(kwargs[a]))) - - return ", ".join([nicerepr(x) for x in args] + bits) + bits.append(f"{a}={nicerepr(kwargs[a])}") - -def unbind_method(f): - """Take something that might be a method or a function and return the - underlying function.""" - return getattr(f, "im_func", getattr(f, "__func__", f)) + rep = nicerepr(f) + if rep.startswith("lambda") and ":" in rep: + rep = f"({rep})" + return rep + "(" + ", ".join(bits) + ")" def check_valid_identifier(identifier): @@ -437,7 +432,7 @@ def check_valid_identifier(identifier): raise ValueError(f"{identifier!r} is not a valid python identifier") -eval_cache = {} # type: dict +eval_cache: dict = {} def source_exec_as_module(source): @@ -446,79 +441,78 @@ def source_exec_as_module(source): except KeyError: pass - result = ModuleType( - "hypothesis_temporary_module_%s" - % (hashlib.sha384(str_to_bytes(source)).hexdigest(),) - ) + hexdigest = hashlib.sha384(source.encode()).hexdigest() + result = ModuleType("hypothesis_temporary_module_" + hexdigest) assert isinstance(source, str) exec(source, result.__dict__) eval_cache[source] = result return result -COPY_ARGSPEC_SCRIPT = """ +COPY_SIGNATURE_SCRIPT = """ from hypothesis.utils.conventions import not_set -def accept(%(funcname)s): - def %(name)s(%(argspec)s): - return %(funcname)s(%(invocation)s) - return %(name)s +def accept({funcname}): + def {name}{signature}: + return {funcname}({invocation}) + return {name} """.lstrip() -def define_function_signature(name, docstring, argspec): - """A decorator which sets the name, argspec and docstring of the function +def get_varargs(sig, kind=inspect.Parameter.VAR_POSITIONAL): + for p in sig.parameters.values(): + if p.kind is kind: + return p + return None + + +def define_function_signature(name, docstring, signature): + """A decorator which sets the name, signature and docstring of the function passed into it.""" + if name == "": + name = "_lambda_" check_valid_identifier(name) - for a in argspec.args: + for a in signature.parameters: check_valid_identifier(a) - if argspec.varargs is not None: - check_valid_identifier(argspec.varargs) - if argspec.varkw is not None: - check_valid_identifier(argspec.varkw) - n_defaults = len(argspec.defaults or ()) - if n_defaults: - parts = [] - for a in argspec.args[:-n_defaults]: - parts.append(a) - for a in argspec.args[-n_defaults:]: - parts.append(f"{a}=not_set") - else: - parts = list(argspec.args) - used_names = list(argspec.args) + list(argspec.kwonlyargs) - used_names.append(name) - for a in argspec.kwonlyargs: - check_valid_identifier(a) + used_names = list(signature.parameters) + [name] + + newsig = signature.replace( + parameters=[ + p if p.default is signature.empty else p.replace(default=not_set) + for p in ( + p.replace(annotation=signature.empty) + for p in signature.parameters.values() + ) + ], + return_annotation=signature.empty, + ) + + pos_args = [ + p + for p in signature.parameters.values() + if p.kind.name.startswith("POSITIONAL_") + ] def accept(f): - fargspec = inspect.getfullargspec(f) + fsig = inspect.signature(f, follow_wrapped=False) must_pass_as_kwargs = [] invocation_parts = [] - for a in argspec.args: - if a not in fargspec.args and not fargspec.varargs: - must_pass_as_kwargs.append(a) + for p in pos_args: + if p.name not in fsig.parameters and get_varargs(fsig) is None: + must_pass_as_kwargs.append(p.name) else: - invocation_parts.append(a) - if argspec.varargs: - used_names.append(argspec.varargs) - parts.append("*" + argspec.varargs) - invocation_parts.append("*" + argspec.varargs) - elif argspec.kwonlyargs: - parts.append("*") + invocation_parts.append(p.name) + if get_varargs(signature) is not None: + invocation_parts.append("*" + get_varargs(signature).name) for k in must_pass_as_kwargs: invocation_parts.append(f"{k}={k}") - - for k in argspec.kwonlyargs: - invocation_parts.append(f"{k}={k}") - if k in (argspec.kwonlydefaults or []): - parts.append(f"{k}=not_set") - else: - parts.append(k) - if argspec.varkw: - used_names.append(argspec.varkw) - parts.append("**" + argspec.varkw) - invocation_parts.append("**" + argspec.varkw) + for p in signature.parameters.values(): + if p.kind is p.KEYWORD_ONLY: + invocation_parts.append(f"{p.name}={p.name}") + varkw = get_varargs(signature, kind=inspect.Parameter.VAR_KEYWORD) + if varkw: + invocation_parts.append("**" + varkw.name) candidate_names = ["f"] + [f"f_{i}" for i in range(1, len(used_names) + 2)] @@ -526,23 +520,35 @@ def accept(f): if funcname not in used_names: break - base_accept = source_exec_as_module( - COPY_ARGSPEC_SCRIPT - % { - "name": name, - "funcname": funcname, - "argspec": ", ".join(parts), - "invocation": ", ".join(invocation_parts), - } - ).accept - - result = base_accept(f) + source = COPY_SIGNATURE_SCRIPT.format( + name=name, + funcname=funcname, + signature=str(newsig), + invocation=", ".join(invocation_parts), + ) + result = source_exec_as_module(source).accept(f) result.__doc__ = docstring - result.__defaults__ = argspec.defaults - if argspec.kwonlydefaults: - result.__kwdefaults__ = argspec.kwonlydefaults - if argspec.annotations: - result.__annotations__ = argspec.annotations + result.__defaults__ = tuple( + p.default + for p in signature.parameters.values() + if p.default is not signature.empty and "POSITIONAL" in p.kind.name + ) + kwdefaults = { + p.name: p.default + for p in signature.parameters.values() + if p.default is not signature.empty and p.kind is p.KEYWORD_ONLY + } + if kwdefaults: + result.__kwdefaults__ = kwdefaults + annotations = { + p.name: p.annotation + for p in signature.parameters.values() + if p.annotation is not signature.empty + } + if signature.return_annotation is not signature.empty: + annotations["return"] = signature.return_annotation + if annotations: + result.__annotations__ = annotations return result return accept @@ -569,16 +575,19 @@ def accept(f): return accept -def proxies(target): +def proxies(target: "T") -> Callable[[Callable], "T"]: + replace_sig = define_function_signature( + target.__name__.replace("", "_lambda_"), # type: ignore + target.__doc__, + get_signature(target, follow_wrapped=False), + ) + def accept(proxy): - return impersonate(target)( - wraps(target)( - define_function_signature( - target.__name__.replace("", "_lambda_"), - target.__doc__, - inspect.getfullargspec(target), - )(proxy) - ) - ) + return impersonate(target)(wraps(target)(replace_sig(proxy))) return accept + + +def is_identity_function(f): + # TODO: pattern-match the AST to handle `def ...` identity functions too + return bool(re.fullmatch(r"lambda (\w+): \1", get_pretty_function_description(f))) diff --git a/hypothesis-python/src/hypothesis/internal/scrutineer.py b/hypothesis-python/src/hypothesis/internal/scrutineer.py new file mode 100644 index 0000000000..6c5075114d --- /dev/null +++ b/hypothesis-python/src/hypothesis/internal/scrutineer.py @@ -0,0 +1,136 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import sys +from collections import defaultdict +from functools import lru_cache, reduce +from os import sep +from pathlib import Path + +from hypothesis._settings import Phase, Verbosity +from hypothesis.internal.escalation import is_hypothesis_file + + +@lru_cache(maxsize=None) +def should_trace_file(fname): + # fname.startswith("<") indicates runtime code-generation via compile, + # e.g. compile("def ...", "", "exec") in e.g. attrs methods. + return not (is_hypothesis_file(fname) or fname.startswith("<")) + + +class Tracer: + """A super-simple branch coverage tracer.""" + + __slots__ = ("branches", "_previous_location") + + def __init__(self): + self.branches = set() + self._previous_location = None + + def trace(self, frame, event, arg): + if event == "call": + return self.trace + elif event == "line": + fname = frame.f_code.co_filename + if should_trace_file(fname): + current_location = (fname, frame.f_lineno) + self.branches.add((self._previous_location, current_location)) + self._previous_location = current_location + + +UNHELPFUL_LOCATIONS = ( + # There's a branch which is only taken when an exception is active while exiting + # a contextmanager; this is probably after the fault has been triggered. + # Similar reasoning applies to a few other standard-library modules: even + # if the fault was later, these still aren't useful locations to report! + f"{sep}contextlib.py", + f"{sep}inspect.py", + f"{sep}re.py", + f"{sep}re{sep}__init__.py", # refactored in Python 3.11 + # Quite rarely, the first AFNP line is in Pytest's assertion-rewriting module. + f"{sep}_pytest{sep}assertion{sep}rewrite.py", +) + + +def get_explaining_locations(traces): + # Traces is a dict[interesting_origin | None, set[frozenset[tuple[str, int]]]] + # Each trace in the set might later become a Counter instead of frozenset. + if not traces: + return {} + + unions = {origin: set().union(*values) for origin, values in traces.items()} + seen_passing = {None}.union(*unions.pop(None, set())) + + always_failing_never_passing = { + origin: reduce(set.intersection, [set().union(*v) for v in values]) + - seen_passing + for origin, values in traces.items() + if origin is not None + } + + # Build the observed parts of the control-flow graph for each origin + cf_graphs = {origin: defaultdict(set) for origin in unions} + for origin, seen_arcs in unions.items(): + for src, dst in seen_arcs: + cf_graphs[origin][src].add(dst) + assert cf_graphs[origin][None], "Expected start node with >=1 successor" + + # For each origin, our explanation is the always_failing_never_passing lines + # which are reachable from the start node (None) without passing through another + # AFNP line. So here's a whatever-first search with early stopping: + explanations = defaultdict(set) + for origin in unions: + queue = {None} + seen = set() + while queue: + assert queue.isdisjoint(seen), f"Intersection: {queue & seen}" + src = queue.pop() + seen.add(src) + if src in always_failing_never_passing[origin]: + explanations[origin].add(src) + else: + queue.update(cf_graphs[origin][src] - seen) + + # The last step is to filter out explanations that we know would be uninformative. + # When this is the first AFNP location, we conclude that Scrutineer missed the + # real divergence (earlier in the trace) and drop that unhelpful explanation. + return { + origin: {loc for loc in afnp_locs if not loc[0].endswith(UNHELPFUL_LOCATIONS)} + for origin, afnp_locs in explanations.items() + } + + +LIB_DIR = str(Path(sys.executable).parent / "lib") +EXPLANATION_STUB = ( + "Explanation:", + " These lines were always and only run by failing examples:", +) + + +def make_report(explanations, cap_lines_at=5): + report = defaultdict(list) + for origin, locations in explanations.items(): + report_lines = [f" {fname}:{lineno}" for fname, lineno in locations] + report_lines.sort(key=lambda line: (line.startswith(LIB_DIR), line)) + if len(report_lines) > cap_lines_at + 1: + msg = " (and {} more with settings.verbosity >= verbose)" + report_lines[cap_lines_at:] = [msg.format(len(report_lines[cap_lines_at:]))] + if report_lines: # We might have filtered out every location as uninformative. + report[origin] = list(EXPLANATION_STUB) + report_lines + return report + + +def explanatory_lines(traces, settings): + if Phase.explain in settings.phases and sys.gettrace() and not traces: + return defaultdict(list) + # Return human-readable report lines summarising the traces + explanations = get_explaining_locations(traces) + max_lines = 5 if settings.verbosity <= Verbosity.normal else float("inf") + return make_report(explanations, cap_lines_at=max_lines) diff --git a/hypothesis-python/src/hypothesis/internal/validation.py b/hypothesis-python/src/hypothesis/internal/validation.py index f797b01f32..da7dfbb937 100644 --- a/hypothesis-python/src/hypothesis/internal/validation.py +++ b/hypothesis-python/src/hypothesis/internal/validation.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import decimal import math @@ -26,7 +21,7 @@ def check_type(typ, arg, name): if not isinstance(arg, typ): if isinstance(typ, tuple): assert len(typ) >= 2, "Use bare type instead of len-1 tuple" - typ_string = "one of %s" % (", ".join(t.__name__ for t in typ)) + typ_string = "one of " + ", ".join(t.__name__ for t in typ) else: typ_string = typ.__name__ @@ -38,8 +33,7 @@ def check_type(typ, arg, name): assert typ is not SearchStrategy, "use check_strategy instead" raise InvalidArgument( - "Expected %s but got %s=%r (type=%s)" - % (typ_string, name, arg, type(arg).__name__) + f"Expected {typ_string} but got {name}={arg!r} (type={type(arg).__name__})" ) @@ -90,11 +84,11 @@ def try_convert(typ, value, name): return value try: return typ(value) - except (TypeError, ValueError, ArithmeticError): + except (TypeError, ValueError, ArithmeticError) as err: raise InvalidArgument( - "Cannot convert %s=%r of type %s to type %s" - % (name, value, type(value).__name__, typ.__name__) - ) + f"Cannot convert {name}={value!r} of type " + f"{type(value).__name__} to type {typ.__name__}" + ) from err @check_function @@ -122,8 +116,7 @@ def check_valid_interval(lower_bound, upper_bound, lower_name, upper_name): return if upper_bound < lower_bound: raise InvalidArgument( - "Cannot have %s=%r < %s=%r" - % (upper_name, upper_bound, lower_name, lower_bound) + f"Cannot have {upper_name}={upper_bound!r} < {lower_name}={lower_bound!r}" ) diff --git a/hypothesis-python/src/hypothesis/provisional.py b/hypothesis-python/src/hypothesis/provisional.py index b3909065cb..914391ae4c 100644 --- a/hypothesis-python/src/hypothesis/provisional.py +++ b/hypothesis-python/src/hypothesis/provisional.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """This module contains various provisional APIs and strategies. @@ -23,47 +18,44 @@ """ # https://tools.ietf.org/html/rfc3696 -import os.path import string +from importlib import resources +from hypothesis import strategies as st from hypothesis.errors import InvalidArgument from hypothesis.internal.conjecture import utils as cu -from hypothesis.strategies._internal import core as st -from hypothesis.strategies._internal.strategies import SearchStrategy +from hypothesis.strategies._internal.utils import defines_strategy URL_SAFE_CHARACTERS = frozenset(string.ascii_letters + string.digits + "$-_.+!*'(),~") +FRAGMENT_SAFE_CHARACTERS = URL_SAFE_CHARACTERS | {"?", "/"} # This file is sourced from http://data.iana.org/TLD/tlds-alpha-by-domain.txt # The file contains additional information about the date that it was last updated. -try: - from importlib.resources import read_text # type: ignore -except ImportError: - # If we don't have importlib.resources (Python 3.7+) or the importlib_resources - # backport available, fall back to __file__ and hope we're on a filesystem. - f = os.path.join(os.path.dirname(__file__), "vendor", "tlds-alpha-by-domain.txt") - with open(f) as tld_file: - _tlds = tld_file.read().splitlines() -else: # pragma: no cover # new in Python 3.7 - _tlds = read_text("hypothesis.vendor", "tlds-alpha-by-domain.txt").splitlines() +try: # pragma: no cover + traversable = resources.files("hypothesis.vendor") / "tlds-alpha-by-domain.txt" + _tlds = traversable.read_text().splitlines() +except (AttributeError, ValueError): # .files() was added in Python 3.9 + _tlds = resources.read_text( + "hypothesis.vendor", "tlds-alpha-by-domain.txt" + ).splitlines() + assert _tlds[0].startswith("#") TOP_LEVEL_DOMAINS = ["COM"] + sorted(_tlds[1:], key=len) -class DomainNameStrategy(SearchStrategy): +class DomainNameStrategy(st.SearchStrategy): @staticmethod def clean_inputs(minimum, maximum, value, variable_name): if value is None: value = maximum elif not isinstance(value, int): raise InvalidArgument( - "Expected integer but %s is a %s" - % (variable_name, type(value).__name__) + f"Expected integer but {variable_name} is a {type(value).__name__}" ) elif not minimum <= value <= maximum: raise InvalidArgument( - "Invalid value %r < %s=%r < %r" - % (minimum, variable_name, value, maximum) + f"Invalid value {minimum!r} < {variable_name}={value!r} < {maximum!r}" ) return value @@ -111,7 +103,7 @@ def do_draw(self, data): .filter(lambda tld: len(tld) + 2 <= self.max_length) .flatmap( lambda tld: st.tuples( - *[st.sampled_from([c.lower(), c.upper()]) for c in tld] + *(st.sampled_from([c.lower(), c.upper()]) for c in tld) ).map("".join) ) ) @@ -120,7 +112,7 @@ def do_draw(self, data): # with a max of 255, that leaves 3 characters for a TLD. # Allowing any more subdomains would not leave enough # characters for even the shortest possible TLDs. - elements = cu.many(data, min_size=1, average_size=1, max_size=126) + elements = cu.many(data, min_size=1, average_size=3, max_size=126) while elements.more(): # Generate a new valid subdomain using the regex strategy. sub_domain = data.draw(st.from_regex(self.label_regex, fullmatch=True)) @@ -131,27 +123,51 @@ def do_draw(self, data): return domain -@st.defines_strategy(force_reusable_values=True) +@defines_strategy(force_reusable_values=True) def domains( *, max_length: int = 255, max_element_length: int = 63 -) -> SearchStrategy[str]: +) -> st.SearchStrategy[str]: """Generate :rfc:`1035` compliant fully qualified domain names.""" return DomainNameStrategy( max_length=max_length, max_element_length=max_element_length ) -@st.defines_strategy(force_reusable_values=True) -def urls() -> SearchStrategy[str]: +# The `urls()` strategy uses this to generate URL fragments (e.g. "#foo"). +# It has been extracted to top-level so that we can test it independently +# of `urls()`, which helps with getting non-flaky coverage of the lambda. +_url_fragments_strategy = ( + st.lists( + st.builds( + lambda char, encode: f"%{ord(char):02X}" + if (encode or char not in FRAGMENT_SAFE_CHARACTERS) + else char, + st.characters(min_codepoint=0, max_codepoint=255), + st.booleans(), + ), + min_size=1, + ) + .map("".join) + .map("#{}".format) +) + + +@defines_strategy(force_reusable_values=True) +def urls() -> st.SearchStrategy[str]: """A strategy for :rfc:`3986`, generating http/https URLs.""" def url_encode(s): return "".join(c if c in URL_SAFE_CHARACTERS else "%%%02X" % ord(c) for c in s) schemes = st.sampled_from(["http", "https"]) - ports = st.integers(min_value=0, max_value=2 ** 16 - 1).map(":{}".format) + ports = st.integers(min_value=0, max_value=2**16 - 1).map(":{}".format) paths = st.lists(st.text(string.printable).map(url_encode)).map("/".join) return st.builds( - "{}://{}{}/{}".format, schemes, domains(), st.just("") | ports, paths + "{}://{}{}/{}{}".format, + schemes, + domains(), + st.just("") | ports, + paths, + st.just("") | _url_fragments_strategy, ) diff --git a/hypothesis-python/src/hypothesis/reporting.py b/hypothesis-python/src/hypothesis/reporting.py index 9ec2af7bb6..a0f300b2bc 100644 --- a/hypothesis-python/src/hypothesis/reporting.py +++ b/hypothesis-python/src/hypothesis/reporting.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import inspect @@ -20,10 +15,6 @@ from hypothesis.utils.dynamicvariables import DynamicVariable -def silent(value): - pass - - def default(value): try: print(value) @@ -50,7 +41,7 @@ def to_text(textish): if inspect.isfunction(textish): textish = textish() if isinstance(textish, bytes): - textish = textish.decode("utf-8") + textish = textish.decode() return textish diff --git a/hypothesis-python/src/hypothesis/stateful.py b/hypothesis-python/src/hypothesis/stateful.py index 8c794b2101..1c66d88ee8 100644 --- a/hypothesis-python/src/hypothesis/stateful.py +++ b/hypothesis-python/src/hypothesis/stateful.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """This module provides support for a stateful style of testing, where tests attempt to find a sequence of operations that cause a breakage rather than just @@ -22,26 +17,48 @@ """ import inspect -from collections.abc import Iterable from copy import copy from functools import lru_cache from io import StringIO -from typing import Any, Dict, List +from typing import ( + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Sequence, + Union, + overload, +) from unittest import TestCase import attr from hypothesis import strategies as st -from hypothesis._settings import HealthCheck, Verbosity, settings as Settings +from hypothesis._settings import ( + HealthCheck, + Verbosity, + note_deprecation, + settings as Settings, +) from hypothesis.control import current_build_context -from hypothesis.core import given +from hypothesis.core import TestFunc, given from hypothesis.errors import InvalidArgument, InvalidDefinition from hypothesis.internal.conjecture import utils as cu -from hypothesis.internal.reflection import function_digest, nicerepr, proxies, qualname +from hypothesis.internal.healthcheck import fail_health_check +from hypothesis.internal.reflection import ( + function_digest, + get_pretty_function_description, + nicerepr, + proxies, +) from hypothesis.internal.validation import check_type from hypothesis.reporting import current_verbosity, report from hypothesis.strategies._internal.featureflags import FeatureStrategy from hypothesis.strategies._internal.strategies import ( + Ex, + Ex_Inv, OneOfStrategy, SearchStrategy, check_strategy, @@ -52,6 +69,11 @@ SHOULD_CONTINUE_LABEL = cu.calc_label_from_name("should we continue drawing") +class _OmittedArgument: + """Sentinel class to prevent overlapping overloads in type hints. See comments + above the overloads of @rule.""" + + class TestCaseProperty: # pragma: no cover def __get__(self, obj, typ=None): if obj is not None: @@ -95,7 +117,7 @@ def run_state_machine(factory, data): try: if print_steps: report(f"state = {machine.__class__.__name__}()") - machine.check_invariants() + machine.check_invariants(settings) max_steps = settings.stateful_step_count steps_run = 0 @@ -164,13 +186,20 @@ def run_state_machine(factory, data): ) else: machine._add_result_to_targets(rule.targets, result) + elif result is not None: + fail_health_check( + settings, + "Rules should return None if they have no target bundle, " + f"but {rule.function.__qualname__} returned {result!r}", + HealthCheck.return_value, + ) finally: if print_steps: # 'result' is only used if the step has target bundles. # If it does, and the result is a 'MultipleResult', # then 'print_step' prints a multi-variable assignment. machine._print_step(rule, data_to_print, result) - machine.check_invariants() + machine.check_invariants(settings) cd.stop_example() finally: if print_steps: @@ -194,16 +223,16 @@ def run_state_machine(factory, data): class StateMachineMeta(type): - def __setattr__(self, name, value): + def __setattr__(cls, name, value): if name == "settings" and isinstance(value, Settings): raise AttributeError( ( "Assigning {cls}.settings = {value} does nothing. Assign " "to {cls}.TestCase.settings, or use @{value} as a decorator " "on the {cls} class." - ).format(cls=self.__name__, value=value) + ).format(cls=cls.__name__, value=value) ) - return type.__setattr__(self, name, value) + return super().__setattr__(name, value) class RuleBasedStateMachine(metaclass=StateMachineMeta): @@ -216,18 +245,16 @@ class RuleBasedStateMachine(metaclass=StateMachineMeta): executed. """ - _rules_per_class = {} # type: Dict[type, List[classmethod]] - _invariants_per_class = {} # type: Dict[type, List[classmethod]] - _base_rules_per_class = {} # type: Dict[type, List[classmethod]] - _initializers_per_class = {} # type: Dict[type, List[classmethod]] - _base_initializers_per_class = {} # type: Dict[type, List[classmethod]] + _rules_per_class: Dict[type, List[classmethod]] = {} + _invariants_per_class: Dict[type, List[classmethod]] = {} + _initializers_per_class: Dict[type, List[classmethod]] = {} - def __init__(self): + def __init__(self) -> None: if not self.rules(): raise InvalidDefinition(f"Type {type(self).__name__} defines no rules") - self.bundles = {} # type: Dict[str, list] + self.bundles: Dict[str, list] = {} self.name_counter = 1 - self.names_to_values = {} # type: Dict[str, Any] + self.names_to_values: Dict[str, Any] = {} self.__stream = StringIO() self.__printer = RepresentationPrinter(self.__stream) self._initialize_rules_to_run = copy(self.initialize_rules()) @@ -246,7 +273,7 @@ def _pretty_print(self, value): return self.__stream.getvalue() def __repr__(self): - return "{}({})".format(type(self).__name__, nicerepr(self.bundles)) + return f"{type(self).__name__}({nicerepr(self.bundles)})" def _new_name(self): result = f"v{self.name_counter}" @@ -268,13 +295,11 @@ def initialize_rules(cls): except KeyError: pass + cls._initializers_per_class[cls] = [] for _, v in inspect.getmembers(cls): r = getattr(v, INITIALIZE_RULE_MARKER, None) if r is not None: - cls.define_initialize_rule( - r.targets, r.function, r.arguments, r.precondition - ) - cls._initializers_per_class[cls] = cls._base_initializers_per_class.pop(cls, []) + cls._initializers_per_class[cls].append(r) return cls._initializers_per_class[cls] @classmethod @@ -284,11 +309,11 @@ def rules(cls): except KeyError: pass + cls._rules_per_class[cls] = [] for _, v in inspect.getmembers(cls): r = getattr(v, RULE_MARKER, None) if r is not None: - cls.define_rule(r.targets, r.function, r.arguments, r.precondition) - cls._rules_per_class[cls] = cls._base_rules_per_class.pop(cls, []) + cls._rules_per_class[cls].append(r) return cls._rules_per_class[cls] @classmethod @@ -306,42 +331,18 @@ def invariants(cls): cls._invariants_per_class[cls] = target return cls._invariants_per_class[cls] - @classmethod - def define_initialize_rule(cls, targets, function, arguments, precondition=None): - converted_arguments = {} - for k, v in arguments.items(): - converted_arguments[k] = v - if cls in cls._initializers_per_class: - target = cls._initializers_per_class[cls] - else: - target = cls._base_initializers_per_class.setdefault(cls, []) - - return target.append(Rule(targets, function, converted_arguments, precondition)) - - @classmethod - def define_rule(cls, targets, function, arguments, precondition=None): - converted_arguments = {} - for k, v in arguments.items(): - converted_arguments[k] = v - if cls in cls._rules_per_class: - target = cls._rules_per_class[cls] - else: - target = cls._base_rules_per_class.setdefault(cls, []) - - return target.append(Rule(targets, function, converted_arguments, precondition)) - def _print_step(self, rule, data, result): self.step_count = getattr(self, "step_count", 0) + 1 - # If the step has target bundles, and the result is a MultipleResults - # then we want to assign to multiple variables. - if isinstance(result, MultipleResults): - n_output_vars = len(result.values) - else: - n_output_vars = 1 - if rule.targets and n_output_vars >= 1: - output_assignment = ", ".join(self._last_names(n_output_vars)) + " = " - else: - output_assignment = "" + output_assignment = "" + if rule.targets: + if isinstance(result, MultipleResults): + if len(result.values) == 1: + output_assignment = f"({self._last_names(1)[0]},) = " + elif result.values: + output_names = self._last_names(len(result.values)) + output_assignment = ", ".join(output_names) + " = " + else: + output_assignment = self._last_names(1)[0] + " = " report( "{}state.{}({})".format( output_assignment, @@ -359,11 +360,26 @@ def _add_result_to_targets(self, targets, result): for target in targets: self.bundles.setdefault(target, []).append(VarReference(name)) - def check_invariants(self): + def check_invariants(self, settings): for invar in self.invariants(): - if invar.precondition and not invar.precondition(self): + if self._initialize_rules_to_run and not invar.check_during_init: continue - invar.function(self) + if not all(precond(self) for precond in invar.preconditions): + continue + if ( + current_build_context().is_final + or settings.verbosity >= Verbosity.debug + ): + report(f"state.{invar.function.__name__}()") + result = invar.function(self) + if result is not None: + fail_health_check( + settings, + "The return value of an @invariant is always ignored, but " + f"{invar.function.__qualname__} returned {result!r} " + "instead of None", + HealthCheck.return_value, + ) def teardown(self): """Called after a run has finished executing to clean up any necessary @@ -376,26 +392,26 @@ def teardown(self): @classmethod @lru_cache() - def _to_test_case(state_machine_class): + def _to_test_case(cls): class StateMachineTestCase(TestCase): settings = Settings(deadline=None, suppress_health_check=HealthCheck.all()) def runTest(self): - run_state_machine_as_test(state_machine_class) + run_state_machine_as_test(cls) runTest.is_hypothesis_test = True - StateMachineTestCase.__name__ = state_machine_class.__name__ + ".TestCase" - StateMachineTestCase.__qualname__ = qualname(state_machine_class) + ".TestCase" + StateMachineTestCase.__name__ = cls.__name__ + ".TestCase" + StateMachineTestCase.__qualname__ = cls.__qualname__ + ".TestCase" return StateMachineTestCase @attr.s() class Rule: targets = attr.ib() - function = attr.ib(repr=qualname) + function = attr.ib(repr=get_pretty_function_description) arguments = attr.ib() - precondition = attr.ib() + preconditions = attr.ib() bundles = attr.ib(init=False) def __attrs_post_init__(self): @@ -436,8 +452,8 @@ def do_draw(self, data): return bundle[position] -class Bundle(SearchStrategy): - def __init__(self, name, consume=False): +class Bundle(SearchStrategy[Ex]): + def __init__(self, name: str, consume: bool = False) -> None: self.name = name self.__reference_strategy = BundleReferenceStrategy(name, consume) @@ -464,12 +480,12 @@ def available(self, data): return bool(machine.bundle(self.name)) -class BundleConsumer(Bundle): - def __init__(self, bundle): +class BundleConsumer(Bundle[Ex]): + def __init__(self, bundle: Bundle[Ex]) -> None: super().__init__(bundle.name, consume=True) -def consumes(bundle): +def consumes(bundle: Bundle[Ex]) -> SearchStrategy[Ex]: """When introducing a rule in a RuleBasedStateMachine, this function can be used to mark bundles from which each value used in a step with the given rule should be removed. This function returns a strategy object @@ -488,14 +504,16 @@ def consumes(bundle): @attr.s() -class MultipleResults(Iterable): +class MultipleResults(Iterable[Ex]): values = attr.ib() def __iter__(self): return iter(self.values) -def multiple(*args): +# We need to use an invariant typevar here to avoid a mypy error, as covariant +# typevars cannot be used as parameters. +def multiple(*args: Ex_Inv) -> MultipleResults[Ex_Inv]: """This function can be used to pass multiple results to the target(s) of a rule. Just use ``return multiple(result1, result2, ...)`` in your rule. @@ -506,7 +524,7 @@ def multiple(*args): def _convert_targets(targets, target): - """Single validator and convertor for target arguments.""" + """Single validator and converter for target arguments.""" if target is not None: if targets: raise InvalidArgument( @@ -529,6 +547,13 @@ def _convert_targets(targets, target): ) raise InvalidArgument(msg % (t, type(t))) while isinstance(t, Bundle): + if isinstance(t, BundleConsumer): + note_deprecation( + f"Using consumes({t.name}) doesn't makes sense in this context. " + "This will be an error in a future version of Hypothesis.", + since="2021-09-08", + has_codemod=False, + ) t = t.name converted_targets.append(t) return tuple(converted_targets) @@ -536,13 +561,74 @@ def _convert_targets(targets, target): RULE_MARKER = "hypothesis_stateful_rule" INITIALIZE_RULE_MARKER = "hypothesis_stateful_initialize_rule" -PRECONDITION_MARKER = "hypothesis_stateful_precondition" +PRECONDITIONS_MARKER = "hypothesis_stateful_preconditions" INVARIANT_MARKER = "hypothesis_stateful_invariant" -def rule(*, targets=(), target=None, **kwargs): - """Decorator for RuleBasedStateMachine. Any name present in target or - targets will define where the end result of this function should go. If +_RuleType = Callable[..., Union[MultipleResults[Ex], Ex]] +_RuleWrapper = Callable[[_RuleType[Ex]], _RuleType[Ex]] + + +# We cannot exclude `target` or `targets` from any of these signatures because +# otherwise they would be matched against the `kwargs`, either leading to +# overlapping overloads of incompatible return types, or a concrete +# implementation that does not accept all overloaded variant signatures. +# Although it is possible to reorder the variants to fix the former, it will +# always lead to the latter, as then the omitted parameter could be typed as +# a `SearchStrategy`, which the concrete implementation does not accept. +# +# Omitted `targets` parameters, where the default value is used, are typed with +# a special `_OmittedArgument` type. We cannot type them as `Tuple[()]`, because +# `Tuple[()]` is a subtype of `Sequence[Bundle[Ex]]`, leading to signature +# overlaps with incompatible return types. The `_OmittedArgument` type will never be +# encountered at runtime, and exists solely to annotate the default of `targets`. +# PEP 661 (Sentinel Values) might provide a more elegant alternative in the future. +# +# We could've also annotated `targets` as `Tuple[_OmittedArgument]`, but then when +# both `target` and `targets` are provided, mypy describes the type error as an +# invalid argument type for `targets` (expected `Tuple[_OmittedArgument]`, got ...). +# By annotating it as a bare `_OmittedArgument` type, mypy's error will warn that +# there is no overloaded signature matching the call, which is more descriptive. +# +# When `target` xor `targets` is provided, the function to decorate must return +# a value whose type matches the one stored in the bundle. When neither are +# provided, the function to decorate must return nothing. There is no variant +# for providing `target` and `targets`, as these parameters are mutually exclusive. +@overload +def rule( + *, + targets: Sequence[Bundle[Ex]], + target: None = ..., + **kwargs: SearchStrategy, +) -> _RuleWrapper[Ex]: # pragma: no cover + ... + + +@overload +def rule( + *, target: Bundle[Ex], targets: _OmittedArgument = ..., **kwargs: SearchStrategy +) -> _RuleWrapper[Ex]: # pragma: no cover + ... + + +@overload +def rule( + *, + target: None = ..., + targets: _OmittedArgument = ..., + **kwargs: SearchStrategy, +) -> Callable[[Callable[..., None]], Callable[..., None]]: # pragma: no cover + ... + + +def rule( + *, + targets: Union[Sequence[Bundle[Ex]], _OmittedArgument] = (), + target: Optional[Bundle[Ex]] = None, + **kwargs: SearchStrategy, +) -> Union[_RuleWrapper[Ex], Callable[[Callable[..., None]], Callable[..., None]]]: + """Decorator for RuleBasedStateMachine. Any Bundle present in ``target`` or + ``targets`` will define where the end result of this function should go. If both are empty then the end result will be discarded. ``target`` must be a Bundle, or if the result should go to multiple @@ -565,18 +651,23 @@ def rule(*, targets=(), target=None, **kwargs): check_strategy(v, name=k) def accept(f): + if getattr(f, INVARIANT_MARKER, None): + raise InvalidDefinition( + "A function cannot be used for both a rule and an invariant.", + Settings.default, + ) existing_rule = getattr(f, RULE_MARKER, None) existing_initialize_rule = getattr(f, INITIALIZE_RULE_MARKER, None) if existing_rule is not None or existing_initialize_rule is not None: raise InvalidDefinition( "A function cannot be used for two distinct rules. ", Settings.default ) - precondition = getattr(f, PRECONDITION_MARKER, None) + preconditions = getattr(f, PRECONDITIONS_MARKER, ()) rule = Rule( targets=converted_targets, arguments=kwargs, function=f, - precondition=precondition, + preconditions=preconditions, ) @proxies(f) @@ -589,27 +680,66 @@ def rule_wrapper(*args, **kwargs): return accept -def initialize(*, targets=(), target=None, **kwargs): +# See also comments of `rule`'s overloads. +@overload +def initialize( + *, + targets: Sequence[Bundle[Ex]], + target: None = ..., + **kwargs: SearchStrategy, +) -> _RuleWrapper[Ex]: # pragma: no cover + ... + + +@overload +def initialize( + *, target: Bundle[Ex], targets: _OmittedArgument = ..., **kwargs: SearchStrategy +) -> _RuleWrapper[Ex]: # pragma: no cover + ... + + +@overload +def initialize( + *, + target: None = ..., + targets: _OmittedArgument = ..., + **kwargs: SearchStrategy, +) -> Callable[[Callable[..., None]], Callable[..., None]]: # pragma: no cover + ... + + +def initialize( + *, + targets: Union[Sequence[Bundle[Ex]], _OmittedArgument] = (), + target: Optional[Bundle[Ex]] = None, + **kwargs: SearchStrategy, +) -> Union[_RuleWrapper[Ex], Callable[[Callable[..., None]], Callable[..., None]]]: """Decorator for RuleBasedStateMachine. - An initialize decorator behaves like a rule, but the decorated - method is called at most once in a run. All initialize decorated - methods will be called before any rule decorated methods, in an - arbitrary order. + An initialize decorator behaves like a rule, but all ``@initialize()`` decorated + methods will be called before any ``@rule()`` decorated methods, in an arbitrary + order. Each ``@initialize()`` method will be called exactly once per run, unless + one raises an exception - after which only the ``.teardown()`` method will be run. + ``@initialize()`` methods may not have preconditions. """ converted_targets = _convert_targets(targets, target) for k, v in kwargs.items(): check_strategy(v, name=k) def accept(f): + if getattr(f, INVARIANT_MARKER, None): + raise InvalidDefinition( + "A function cannot be used for both a rule and an invariant.", + Settings.default, + ) existing_rule = getattr(f, RULE_MARKER, None) existing_initialize_rule = getattr(f, INITIALIZE_RULE_MARKER, None) if existing_rule is not None or existing_initialize_rule is not None: raise InvalidDefinition( "A function cannot be used for two distinct rules. ", Settings.default ) - precondition = getattr(f, PRECONDITION_MARKER, None) - if precondition: + preconditions = getattr(f, PRECONDITIONS_MARKER, ()) + if preconditions: raise InvalidDefinition( "An initialization rule cannot have a precondition. ", Settings.default ) @@ -617,7 +747,7 @@ def accept(f): targets=converted_targets, arguments=kwargs, function=f, - precondition=precondition, + preconditions=preconditions, ) @proxies(f) @@ -635,10 +765,13 @@ class VarReference: name = attr.ib() -def precondition(precond): +# There are multiple alternatives for annotating the `precond` type, all of them +# have drawbacks. See https://github.com/HypothesisWorks/hypothesis/pull/3068#issuecomment-906642371 +def precondition(precond: Callable[[Any], bool]) -> Callable[[TestFunc], TestFunc]: """Decorator to apply a precondition for rules in a RuleBasedStateMachine. Specifies a precondition for a rule to be considered as a valid step in the - state machine. The given function will be called with the instance of + state machine, which is more efficient than using :func:`~hypothesis.assume` + within the rule. The ``precond`` function will be called with the instance of RuleBasedStateMachine and should return True or False. Usually it will need to look at attributes on that instance. @@ -652,8 +785,9 @@ class MyTestMachine(RuleBasedStateMachine): def divide_with(self, numerator): self.state = numerator / self.state - This is better than using assume in your rule since more valid rules - should be able to be run. + If multiple preconditions are applied to a single rule, it is only considered + a valid step when all of them return True. Preconditions may be applied to + invariants as well as rules. """ def decorator(f): @@ -668,21 +802,23 @@ def precondition_wrapper(*args, **kwargs): ) rule = getattr(f, RULE_MARKER, None) - if rule is None: - setattr(precondition_wrapper, PRECONDITION_MARKER, precond) - else: - new_rule = Rule( - targets=rule.targets, - arguments=rule.arguments, - function=rule.function, - precondition=precond, - ) - setattr(precondition_wrapper, RULE_MARKER, new_rule) - invariant = getattr(f, INVARIANT_MARKER, None) - if invariant is not None: - new_invariant = Invariant(function=invariant.function, precondition=precond) + if rule is not None: + assert invariant is None + new_rule = attr.evolve(rule, preconditions=rule.preconditions + (precond,)) + setattr(precondition_wrapper, RULE_MARKER, new_rule) + elif invariant is not None: + assert rule is None + new_invariant = attr.evolve( + invariant, preconditions=invariant.preconditions + (precond,) + ) setattr(precondition_wrapper, INVARIANT_MARKER, new_invariant) + else: + setattr( + precondition_wrapper, + PRECONDITIONS_MARKER, + getattr(f, PRECONDITIONS_MARKER, ()) + (precond,), + ) return precondition_wrapper @@ -691,11 +827,12 @@ def precondition_wrapper(*args, **kwargs): @attr.s() class Invariant: - function = attr.ib() - precondition = attr.ib() + function = attr.ib(repr=get_pretty_function_description) + preconditions = attr.ib() + check_during_init = attr.ib() -def invariant(): +def invariant(*, check_during_init: bool = False) -> Callable[[TestFunc], TestFunc]: """Decorator to apply an invariant for rules in a RuleBasedStateMachine. The decorated function will be run after every rule and can raise an exception to indicate failed invariants. @@ -708,23 +845,38 @@ class MyTestMachine(RuleBasedStateMachine): @invariant() def is_nonzero(self): assert self.state != 0 + + By default, invariants are only checked after all + :func:`@initialize() ` rules have been run. + Pass ``check_during_init=True`` for invariants which can also be checked + during initialization. """ + check_type(bool, check_during_init, "check_during_init") def accept(f): + if getattr(f, RULE_MARKER, None) or getattr(f, INITIALIZE_RULE_MARKER, None): + raise InvalidDefinition( + "A function cannot be used for both a rule and an invariant.", + Settings.default, + ) existing_invariant = getattr(f, INVARIANT_MARKER, None) if existing_invariant is not None: raise InvalidDefinition( "A function cannot be used for two distinct invariants.", Settings.default, ) - precondition = getattr(f, PRECONDITION_MARKER, None) - rule = Invariant(function=f, precondition=precondition) + preconditions = getattr(f, PRECONDITIONS_MARKER, ()) + invar = Invariant( + function=f, + preconditions=preconditions, + check_during_init=check_during_init, + ) @proxies(f) def invariant_wrapper(*args, **kwargs): return f(*args, **kwargs) - setattr(invariant_wrapper, INVARIANT_MARKER, rule) + setattr(invariant_wrapper, INVARIANT_MARKER, invar) return invariant_wrapper return accept @@ -735,7 +887,7 @@ def invariant_wrapper(*args, **kwargs): class RuleStrategy(SearchStrategy): def __init__(self, machine): - SearchStrategy.__init__(self) + super().__init__() self.machine = machine self.rules = list(machine.rules()) @@ -786,7 +938,7 @@ def do_draw(self, data): return (rule, data.draw(rule.arguments_strategy)) def is_valid(self, rule): - if rule.precondition and not rule.precondition(self.machine): + if not all(precond(self.machine) for precond in rule.preconditions): return False for b in rule.bundles: diff --git a/hypothesis-python/src/hypothesis/statistics.py b/hypothesis-python/src/hypothesis/statistics.py index 9974f3fea6..73377077cd 100644 --- a/hypothesis-python/src/hypothesis/statistics.py +++ b/hypothesis-python/src/hypothesis/statistics.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import math import statistics @@ -32,7 +27,7 @@ def describe_targets(best_targets): """Return a list of lines describing the results of `target`, if any.""" # These lines are included in the general statistics description below, # but also printed immediately below failing examples to alleviate the - # "threshold problem" where shrinking can make severe bug look trival. + # "threshold problem" where shrinking can make severe bug look trivial. # See https://github.com/HypothesisWorks/hypothesis/issues/2180 if not best_targets: return [] @@ -82,24 +77,17 @@ def describe_statistics(stats_dict): t["drawtime"] / t["runtime"] if t["runtime"] > 0 else 0 for t in cases ) lines.append( - " - during {} phase ({:.2f} seconds):\n" - " - Typical runtimes: {}, ~ {:.0f}% in data generation\n" - " - {} passing examples, {} failing examples, {} invalid examples".format( - phase, - d["duration-seconds"], - ms, - drawtime_percent, - statuses["valid"], - statuses["interesting"], - statuses["invalid"] + statuses["overrun"], - ) + f" - during {phase} phase ({d['duration-seconds']:.2f} seconds):\n" + f" - Typical runtimes: {ms}, ~ {drawtime_percent:.0f}% in data generation\n" + f" - {statuses['valid']} passing examples, {statuses['interesting']} " + f"failing examples, {statuses['invalid'] + statuses['overrun']} invalid examples" ) # If we've found new distinct failures in this phase, report them distinct_failures = d["distinct-failures"] - prev_failures if distinct_failures: plural = distinct_failures > 1 lines.append( - " - Found {}{} failing example{} in this phase".format( + " - Found {}{} distinct error{} in this phase".format( distinct_failures, " more" * bool(prev_failures), "s" * plural ) ) @@ -110,7 +98,7 @@ def describe_statistics(stats_dict): if events: lines.append(" - Events:") lines += [ - " * {:.2f}%, {}".format(100 * v / len(cases), k) + f" * {100 * v / len(cases):.2f}%, {k}" for k, v in sorted(events.items(), key=lambda x: (-x[1], x[0])) ] # Some additional details on the shrinking phase diff --git a/hypothesis-python/src/hypothesis/strategies/__init__.py b/hypothesis-python/src/hypothesis/strategies/__init__.py index 10286c6887..1d8b1ea76f 100644 --- a/hypothesis-python/src/hypothesis/strategies/__init__.py +++ b/hypothesis-python/src/hypothesis/strategies/__init__.py @@ -1,22 +1,18 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis.strategies._internal import SearchStrategy +from hypothesis.strategies._internal.collections import tuples from hypothesis.strategies._internal.core import ( DataObject, - _strategies, + DrawFn, binary, booleans, builds, @@ -29,19 +25,13 @@ dictionaries, emails, fixed_dictionaries, - floats, fractions, from_regex, from_type, frozensets, functions, - integers, iterables, - just, lists, - none, - nothing, - one_of, permutations, random_module, randoms, @@ -53,7 +43,6 @@ shared, slices, text, - tuples, uuids, ) from hypothesis.strategies._internal.datetime import ( @@ -65,6 +54,10 @@ timezones, ) from hypothesis.strategies._internal.ipaddress import ip_addresses +from hypothesis.strategies._internal.misc import just, none, nothing +from hypothesis.strategies._internal.numbers import floats, integers +from hypothesis.strategies._internal.strategies import one_of +from hypothesis.strategies._internal.utils import _strategies # The implementation of all of these lives in `_strategies.py`, but we # re-export them via this module to avoid exposing implementation details @@ -85,6 +78,7 @@ "decimals", "deferred", "dictionaries", + "DrawFn", "emails", "fixed_dictionaries", "floats", @@ -121,10 +115,27 @@ "SearchStrategy", ] -assert set(_strategies).issubset(__all__), ( - set(_strategies) - set(__all__), - set(__all__) - set(_strategies), -) -_public = {n for n in dir() if n[0] not in "_@"} -assert set(__all__) == _public, (set(__all__) - _public, _public - set(__all__)) -del _public + +def _check_exports(_public): + assert set(__all__) == _public, (set(__all__) - _public, _public - set(__all__)) + + # Verify that all exported strategy functions were registered with + # @declares_strategy. + + existing_strategies = set(_strategies) - {"_maybe_nil_uuids"} + + exported_strategies = set(__all__) - { + "DataObject", + "DrawFn", + "SearchStrategy", + "composite", + "register_type_strategy", + } + assert existing_strategies == exported_strategies, ( + existing_strategies - exported_strategies, + exported_strategies - existing_strategies, + ) + + +_check_exports({n for n in dir() if n[0] not in "_@"}) +del _check_exports diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/__init__.py b/hypothesis-python/src/hypothesis/strategies/_internal/__init__.py index 05aad29ab3..c65cc85558 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/__init__.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/__init__.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """Package defining SearchStrategy, which is the core type that Hypothesis uses to explore data.""" diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/attrs.py b/hypothesis-python/src/hypothesis/strategies/_internal/attrs.py index f75d517a6a..d4f56a1f3a 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/attrs.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/attrs.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from functools import reduce from itertools import chain @@ -21,6 +16,7 @@ from hypothesis import strategies as st from hypothesis.errors import ResolutionFailed from hypothesis.internal.compat import get_type_hints +from hypothesis.strategies._internal.core import BuildsStrategy from hypothesis.strategies._internal.types import is_a_type, type_sorting_key from hypothesis.utils.conventions import infer @@ -34,15 +30,13 @@ def from_attrs(target, args, kwargs, to_infer): # We might make this strategy more efficient if we added a layer here that # retries drawing if validation fails, for improved composition. # The treatment of timezones in datetimes() provides a precedent. - return st.tuples(st.tuples(*args), st.fixed_dictionaries(kwargs)).map( - lambda value: target(*value[0], **value[1]) - ) + return BuildsStrategy(target, args, kwargs) def from_attrs_attribute(attrib, target): """Infer a strategy from the metadata on an attr.Attribute object.""" # Try inferring from the default argument. Note that this will only help if - # the user passed `infer` to builds() for this attribute, but in that case + # the user passed `...` to builds() for this attribute, but in that case # we use it as the minimal example. default = st.nothing() if isinstance(attrib.default, attr.Factory): @@ -90,7 +84,7 @@ def from_attrs_attribute(attrib, target): if strat.is_empty: raise ResolutionFailed( "Cannot infer a strategy from the default, validator, type, or " - "converter for attribute=%r of class=%r" % (attrib, target) + f"converter for attribute={attrib!r} of class={target!r}" ) return strat diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/collections.py b/hypothesis-python/src/hypothesis/strategies/_internal/collections.py index c1e987f223..5476239ad7 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/collections.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/collections.py @@ -1,37 +1,41 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER -from collections import OrderedDict +import copy +from typing import Any, Iterable, Tuple, overload from hypothesis.errors import InvalidArgument from hypothesis.internal.conjecture import utils as cu from hypothesis.internal.conjecture.junkdrawer import LazySequenceCopy from hypothesis.internal.conjecture.utils import combine_labels +from hypothesis.internal.reflection import is_identity_function from hypothesis.strategies._internal.strategies import ( + T3, + T4, + T5, + Ex, MappedSearchStrategy, SearchStrategy, + T, + check_strategy, filter_not_satisfied, ) +from hypothesis.strategies._internal.utils import cacheable, defines_strategy class TupleStrategy(SearchStrategy): - """A strategy responsible for fixed length tuples based on heterogenous + """A strategy responsible for fixed length tuples based on heterogeneous strategies for each of their elements.""" - def __init__(self, strategies): - SearchStrategy.__init__(self) + def __init__(self, strategies: Iterable[SearchStrategy[Any]]): + super().__init__() self.element_strategies = tuple(strategies) def do_validate(self): @@ -40,14 +44,11 @@ def do_validate(self): def calc_label(self): return combine_labels( - self.class_label, *[s.label for s in self.element_strategies] + self.class_label, *(s.label for s in self.element_strategies) ) def __repr__(self): - if len(self.element_strategies) == 1: - tuple_string = f"{self.element_strategies[0]!r}," - else: - tuple_string = ", ".join(map(repr, self.element_strategies)) + tuple_string = ", ".join(map(repr, self.element_strategies)) return f"TupleStrategy(({tuple_string}))" def calc_has_reusable_values(self, recur): @@ -60,12 +61,83 @@ def calc_is_empty(self, recur): return any(recur(e) for e in self.element_strategies) +@overload +def tuples() -> SearchStrategy[Tuple[()]]: # pragma: no cover + ... + + +@overload # noqa: F811 +def tuples(__a1: SearchStrategy[Ex]) -> SearchStrategy[Tuple[Ex]]: # pragma: no cover + ... + + +@overload # noqa: F811 +def tuples( + __a1: SearchStrategy[Ex], __a2: SearchStrategy[T] +) -> SearchStrategy[Tuple[Ex, T]]: # pragma: no cover + ... + + +@overload # noqa: F811 +def tuples( + __a1: SearchStrategy[Ex], __a2: SearchStrategy[T], __a3: SearchStrategy[T3] +) -> SearchStrategy[Tuple[Ex, T, T3]]: # pragma: no cover + ... + + +@overload # noqa: F811 +def tuples( + __a1: SearchStrategy[Ex], + __a2: SearchStrategy[T], + __a3: SearchStrategy[T3], + __a4: SearchStrategy[T4], +) -> SearchStrategy[Tuple[Ex, T, T3, T4]]: # pragma: no cover + ... + + +@overload # noqa: F811 +def tuples( + __a1: SearchStrategy[Ex], + __a2: SearchStrategy[T], + __a3: SearchStrategy[T3], + __a4: SearchStrategy[T4], + __a5: SearchStrategy[T5], +) -> SearchStrategy[Tuple[Ex, T, T3, T4, T5]]: # pragma: no cover + ... + + +@overload # noqa: F811 +def tuples( + *args: SearchStrategy[Any], +) -> SearchStrategy[Tuple[Any, ...]]: # pragma: no cover + ... + + +@cacheable +@defines_strategy() +def tuples(*args: SearchStrategy[Any]) -> SearchStrategy[Tuple[Any, ...]]: # noqa: F811 + """Return a strategy which generates a tuple of the same length as args by + generating the value at index i from args[i]. + + e.g. tuples(integers(), integers()) would generate a tuple of length + two with both values an integer. + + Examples from this strategy shrink by shrinking their component parts. + """ + for arg in args: + check_strategy(arg) + + return TupleStrategy(args) + + class ListStrategy(SearchStrategy): """A strategy for lists which takes a strategy for its elements and the allowed lengths, and generates lists with the correct size and contents.""" + _nonempty_filters: tuple = (bool, len, tuple, list) + def __init__(self, elements, min_size=0, max_size=float("inf")): - SearchStrategy.__init__(self) + super().__init__() self.min_size = min_size or 0 self.max_size = max_size if max_size is not None else float("inf") assert 0 <= self.min_size <= self.max_size @@ -82,17 +154,14 @@ def do_validate(self): self.element_strategy.validate() if self.is_empty: raise InvalidArgument( - ( - "Cannot create non-empty lists with elements drawn from " - "strategy %r because it has no values." - ) - % (self.element_strategy,) + "Cannot create non-empty lists with elements drawn from " + f"strategy {self.element_strategy!r} because it has no values." ) if self.element_strategy.is_empty and 0 < self.max_size < float("inf"): raise InvalidArgument( - "Cannot create a collection of max_size=%r, because no " - "elements can be drawn from the element strategy %r" - % (self.max_size, self.element_strategy) + f"Cannot create a collection of max_size={self.max_size!r}, " + "because no elements can be drawn from the element strategy " + f"{self.element_strategy!r}" ) def calc_is_empty(self, recur): @@ -122,6 +191,16 @@ def __repr__(self): self.__class__.__name__, self.element_strategy, self.min_size, self.max_size ) + def filter(self, condition): + if condition in self._nonempty_filters or is_identity_function(condition): + assert self.max_size >= 1, "Always-empty is special cased in st.lists()" + if self.min_size >= 1: + return self + new = copy.copy(self) + new.min_size = 1 + return new + return super().filter(condition) + class UniqueListStrategy(ListStrategy): def __init__(self, elements, min_size, max_size, keys, tuple_suffixes): @@ -149,11 +228,11 @@ def do_draw(self, data): def not_yet_in_unique_list(val): return all(key(val) not in seen for key, seen in zip(self.keys, seen_sets)) - filtered = self.element_strategy.filter(not_yet_in_unique_list) + filtered = self.element_strategy._filter_for_filtered_draw( + not_yet_in_unique_list + ) while elements.more(): - value = filtered.filtered_strategy.do_filtered_draw( - data=data, filter_strategy=filtered - ) + value = filtered.do_filtered_draw(data) if value is filter_not_satisfied: elements.reject() else: @@ -210,14 +289,7 @@ class FixedKeysDictStrategy(MappedSearchStrategy): def __init__(self, strategy_dict): self.dict_type = type(strategy_dict) - - if isinstance(strategy_dict, OrderedDict): - self.keys = tuple(strategy_dict.keys()) - else: - try: - self.keys = tuple(sorted(strategy_dict.keys())) - except TypeError: - self.keys = tuple(sorted(strategy_dict.keys(), key=repr)) + self.keys = tuple(strategy_dict.keys()) super().__init__(strategy=TupleStrategy(strategy_dict[k] for k in self.keys)) def calc_is_empty(self, recur): @@ -243,14 +315,6 @@ def __init__(self, strategy_dict, optional): self.fixed = FixedKeysDictStrategy(strategy_dict) self.optional = optional - if isinstance(self.optional, OrderedDict): - self.optional_keys = tuple(self.optional.keys()) - else: - try: - self.optional_keys = tuple(sorted(self.optional.keys())) - except TypeError: - self.optional_keys = tuple(sorted(self.optional.keys(), key=repr)) - def calc_is_empty(self, recur): return recur(self.fixed) @@ -259,7 +323,7 @@ def __repr__(self): def do_draw(self, data): result = data.draw(self.fixed) - remaining = [k for k in self.optional_keys if not self.optional[k].is_empty] + remaining = [k for k, v in self.optional.items() if not v.is_empty] should_draw = cu.many( data, min_size=0, max_size=len(remaining), average_size=len(remaining) / 2 ) diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/core.py b/hypothesis-python/src/hypothesis/strategies/_internal/core.py index 86a086b178..0a77c367e5 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/core.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/core.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import enum import math @@ -20,14 +15,14 @@ import re import string import sys -import threading import typing from decimal import Context, Decimal, localcontext from fractions import Fraction -from functools import reduce -from inspect import Parameter, getfullargspec, isabstract, isclass, signature +from functools import lru_cache, reduce +from inspect import Parameter, Signature, isabstract, isclass, signature from types import FunctionType from typing import ( + TYPE_CHECKING, Any, AnyStr, Callable, @@ -50,38 +45,35 @@ import attr -from hypothesis.control import cleanup, note, reject +from hypothesis._settings import note_deprecation +from hypothesis.control import cleanup, note from hypothesis.errors import InvalidArgument, ResolutionFailed -from hypothesis.internal.cache import LRUReusedCache from hypothesis.internal.cathetus import cathetus from hypothesis.internal.charmap import as_general_categories -from hypothesis.internal.compat import ceil, floor, get_type_hints, typing_root_type +from hypothesis.internal.compat import ( + Concatenate, + ParamSpec, + ceil, + floor, + get_type_hints, + is_typed_named_tuple, +) from hypothesis.internal.conjecture.utils import ( calc_label_from_cls, check_sample, integer_range, ) from hypothesis.internal.entropy import get_seeder_and_restorer -from hypothesis.internal.floats import ( - count_between_floats, - float_of, - float_to_int, - int_to_float, - is_negative, - next_down, - next_up, -) from hypothesis.internal.reflection import ( define_function_signature, get_pretty_function_description, - is_typed_named_tuple, + get_signature, + is_first_param_referenced_in_function, nicerepr, - proxies, required_args, ) from hypothesis.internal.validation import ( check_type, - check_valid_bound, check_valid_integer, check_valid_interval, check_valid_magnitude, @@ -97,361 +89,48 @@ TupleStrategy, UniqueListStrategy, UniqueSampledListStrategy, + tuples, ) from hypothesis.strategies._internal.deferred import DeferredStrategy from hypothesis.strategies._internal.functions import FunctionStrategy -from hypothesis.strategies._internal.lazy import LazyStrategy -from hypothesis.strategies._internal.misc import JustStrategy +from hypothesis.strategies._internal.lazy import LazyStrategy, unwrap_strategies +from hypothesis.strategies._internal.misc import just, none, nothing from hypothesis.strategies._internal.numbers import ( - BoundedIntStrategy, - FixedBoundedFloatStrategy, - FloatStrategy, - WideRangeIntStrategy, + IntegersStrategy, + Real, + floats, + integers, ) from hypothesis.strategies._internal.recursive import RecursiveStrategy from hypothesis.strategies._internal.shared import SharedStrategy from hypothesis.strategies._internal.strategies import ( Ex, - OneOfStrategy, SampledFromStrategy, T, + one_of, ) from hypothesis.strategies._internal.strings import ( FixedSizeBytes, OneCharStringStrategy, + TextStrategy, ) -from hypothesis.utils.conventions import InferType, infer, not_set - -K = TypeVar("K") -V = TypeVar("V") -UniqueBy = Union[Callable[[Ex], Hashable], Tuple[Callable[[Ex], Hashable], ...]] -# See https://github.com/python/mypy/issues/3186 - numbers.Real is wrong! -Real = Union[int, float, Fraction, Decimal] - -_strategies = {} # type: Dict[str, Callable[..., SearchStrategy]] - - -class FloatKey: - def __init__(self, f): - self.value = float_to_int(f) - - def __eq__(self, other): - return isinstance(other, FloatKey) and (other.value == self.value) - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash(self.value) - - -def convert_value(v): - if isinstance(v, float): - return FloatKey(v) - return (type(v), v) - - -_CACHE = threading.local() - - -def get_cache() -> LRUReusedCache: - try: - return _CACHE.STRATEGY_CACHE - except AttributeError: - _CACHE.STRATEGY_CACHE = LRUReusedCache(1024) - return _CACHE.STRATEGY_CACHE - - -def clear_cache() -> None: - cache = get_cache() - cache.clear() - - -def cacheable(fn: T) -> T: - @proxies(fn) - def cached_strategy(*args, **kwargs): - try: - kwargs_cache_key = {(k, convert_value(v)) for k, v in kwargs.items()} - except TypeError: - return fn(*args, **kwargs) - cache_key = (fn, tuple(map(convert_value, args)), frozenset(kwargs_cache_key)) - cache = get_cache() - try: - if cache_key in cache: - return cache[cache_key] - except TypeError: - return fn(*args, **kwargs) - else: - result = fn(*args, **kwargs) - if not isinstance(result, SearchStrategy) or result.is_cacheable: - cache[cache_key] = result - return result - - cached_strategy.__clear_cache = clear_cache - return cached_strategy - - -def defines_strategy( - *, force_reusable_values: bool = False, try_non_lazy: bool = False -) -> Callable[[T], T]: - """Returns a decorator for strategy functions. - - If force_reusable is True, the generated values are assumed to be - reusable, i.e. immutable and safe to cache, across multiple test - invocations. - - If try_non_lazy is True, attempt to execute the strategy definition - function immediately, so that a LazyStrategy is only returned if this - raises an exception. - """ - - def decorator(strategy_definition): - """A decorator that registers the function as a strategy and makes it - lazily evaluated.""" - _strategies[strategy_definition.__name__] = signature(strategy_definition) - - @proxies(strategy_definition) - def accept(*args, **kwargs): - if try_non_lazy: - # Why not try this unconditionally? Because we'd end up with very - # deep nesting of recursive strategies - better to be lazy unless we - # *know* that eager evaluation is the right choice. - try: - return strategy_definition(*args, **kwargs) - except Exception: - # If invoking the strategy definition raises an exception, - # wrap that up in a LazyStrategy so it happens again later. - pass - result = LazyStrategy(strategy_definition, args, kwargs) - if force_reusable_values: - result.force_has_reusable_values = True - assert result.has_reusable_values - return result - - accept.is_hypothesis_strategy_function = True - return accept - - return decorator - +from hypothesis.strategies._internal.utils import cacheable, defines_strategy +from hypothesis.utils.conventions import not_set -class Nothing(SearchStrategy): - def calc_is_empty(self, recur): - return True +if sys.version_info >= (3, 10): # pragma: no cover + from types import EllipsisType as EllipsisType +elif typing.TYPE_CHECKING: # pragma: no cover + from builtins import ellipsis as EllipsisType +else: + EllipsisType = type(Ellipsis) - def do_draw(self, data): - # This method should never be called because draw() will mark the - # data as invalid immediately because is_empty is True. - raise NotImplementedError("This should never happen") - - def calc_has_reusable_values(self, recur): - return True - - def __repr__(self): - return "nothing()" - - def map(self, f): - return self - - def filter(self, f): - return self - - def flatmap(self, f): - return self - -NOTHING = Nothing() - - -@cacheable -def nothing() -> SearchStrategy: - """This strategy never successfully draws a value and will always reject on - an attempt to draw. - - Examples from this strategy do not shrink (because there are none). - """ - return NOTHING - - -def just(value: T) -> SearchStrategy[T]: - """Return a strategy which only generates ``value``. - - Note: ``value`` is not copied. Be wary of using mutable values. - - If ``value`` is the result of a callable, you can use - :func:`builds(callable) ` instead - of ``just(callable())`` to get a fresh value each time. - - Examples from this strategy do not shrink (because there is only one). - """ - return JustStrategy([value]) - - -@defines_strategy(force_reusable_values=True) -def none() -> SearchStrategy[None]: - """Return a strategy which only generates None. - - Examples from this strategy do not shrink (because there is only - one). - """ - return just(None) - - -T4 = TypeVar("T4") -T5 = TypeVar("T5") - - -@overload -def one_of(args: Sequence[SearchStrategy[Any]]) -> SearchStrategy[Any]: - raise NotImplementedError - - -@overload # noqa: F811 -def one_of(a1: SearchStrategy[Ex]) -> SearchStrategy[Ex]: - raise NotImplementedError - - -@overload # noqa: F811 -def one_of( - a1: SearchStrategy[Ex], a2: SearchStrategy[K] -) -> SearchStrategy[Union[Ex, K]]: - raise NotImplementedError - - -@overload # noqa: F811 -def one_of( - a1: SearchStrategy[Ex], a2: SearchStrategy[K], a3: SearchStrategy[V] -) -> SearchStrategy[Union[Ex, K, V]]: - raise NotImplementedError - - -@overload # noqa: F811 -def one_of( - a1: SearchStrategy[Ex], - a2: SearchStrategy[K], - a3: SearchStrategy[V], - a4: SearchStrategy[T4], -) -> SearchStrategy[Union[Ex, K, V, T4]]: - raise NotImplementedError - - -@overload # noqa: F811 -def one_of( - a1: SearchStrategy[Ex], - a2: SearchStrategy[K], - a3: SearchStrategy[V], - a4: SearchStrategy[T4], - a5: SearchStrategy[T5], -) -> SearchStrategy[Union[Ex, K, V, T4, T5]]: - raise NotImplementedError - - -@overload # noqa: F811 -def one_of(*args: SearchStrategy[Any]) -> SearchStrategy[Any]: - raise NotImplementedError - - -def one_of(*args): # noqa: F811 - # Mypy workaround alert: Any is too loose above; the return parameter - # should be the union of the input parameters. Unfortunately, Mypy <=0.600 - # raises errors due to incompatible inputs instead. See #1270 for links. - # v0.610 doesn't error; it gets inference wrong for 2+ arguments instead. - """Return a strategy which generates values from any of the argument - strategies. - - This may be called with one iterable argument instead of multiple - strategy arguments, in which case ``one_of(x)`` and ``one_of(*x)`` are - equivalent. - - Examples from this strategy will generally shrink to ones that come from - strategies earlier in the list, then shrink according to behaviour of the - strategy that produced them. In order to get good shrinking behaviour, - try to put simpler strategies first. e.g. ``one_of(none(), text())`` is - better than ``one_of(text(), none())``. - - This is especially important when using recursive strategies. e.g. - ``x = st.deferred(lambda: st.none() | st.tuples(x, x))`` will shrink well, - but ``x = st.deferred(lambda: st.tuples(x, x) | st.none())`` will shrink - very badly indeed. - """ - if len(args) == 1 and not isinstance(args[0], SearchStrategy): - try: - args = tuple(args[0]) - except TypeError: - pass - if len(args) == 1 and isinstance(args[0], SearchStrategy): - # This special-case means that we can one_of over lists of any size - # without incurring any performance overhead when there is only one - # strategy, and keeps our reprs simple. - return args[0] - if args and not any(isinstance(a, SearchStrategy) for a in args): - # And this special case is to give a more-specific error message if it - # seems that the user has confused `one_of()` for `sampled_from()`; - # the remaining validation is left to OneOfStrategy. See PR #2627. - raise InvalidArgument( - f"Did you mean st.sampled_from({list(args)!r})? st.one_of() is used " - "to combine strategies, but all of the arguments were of other types." - ) - return OneOfStrategy(args) - - -@cacheable -@defines_strategy(force_reusable_values=True) -def integers( - min_value: Optional[int] = None, - max_value: Optional[int] = None, -) -> SearchStrategy[int]: - """Returns a strategy which generates integers. - - If min_value is not None then all values will be >= min_value. If - max_value is not None then all values will be <= max_value - - Examples from this strategy will shrink towards zero, and negative values - will also shrink towards positive (i.e. -n may be replaced by +n). - """ - - check_valid_bound(min_value, "min_value") - check_valid_bound(max_value, "max_value") - check_valid_interval(min_value, max_value, "min_value", "max_value") - - if min_value is not None: - if min_value != int(min_value): - raise InvalidArgument( - "min_value=%r of type %r cannot be exactly represented as an integer." - % (min_value, type(min_value)) - ) - min_value = int(min_value) - if max_value is not None: - if max_value != int(max_value): - raise InvalidArgument( - "max_value=%r of type %r cannot be exactly represented as an integer." - % (max_value, type(max_value)) - ) - max_value = int(max_value) - - if min_value is None: - if max_value is None: - return WideRangeIntStrategy() - else: - if max_value > 0: - return WideRangeIntStrategy().filter(lambda x: x <= max_value) - return WideRangeIntStrategy().map(lambda x: max_value - abs(x)) - else: - if max_value is None: - if min_value < 0: - return WideRangeIntStrategy().filter(lambda x: x >= min_value) - return WideRangeIntStrategy().map(lambda x: min_value + abs(x)) - else: - assert min_value <= max_value - if min_value == max_value: - return just(min_value) - elif min_value >= 0: - return BoundedIntStrategy(min_value, max_value) - elif max_value <= 0: - return BoundedIntStrategy(-max_value, -min_value).map(lambda t: -t) - else: - return integers(min_value=0, max_value=max_value) | integers( - min_value=min_value, max_value=0 - ) +if sys.version_info >= (3, 8): # pragma: no cover + from typing import Protocol +elif TYPE_CHECKING: + from typing_extensions import Protocol +else: # pragma: no cover + Protocol = object @cacheable @@ -465,246 +144,28 @@ def booleans() -> SearchStrategy[bool]: return SampledFromStrategy([False, True], repr_="booleans()") -@cacheable -@defines_strategy(force_reusable_values=True) -def floats( - min_value: Optional[Real] = None, - max_value: Optional[Real] = None, - *, - allow_nan: Optional[bool] = None, - allow_infinity: Optional[bool] = None, - width: int = 64, - exclude_min: bool = False, - exclude_max: bool = False, -) -> SearchStrategy[float]: - """Returns a strategy which generates floats. - - - If min_value is not None, all values will be ``>= min_value`` - (or ``> min_value`` if ``exclude_min``). - - If max_value is not None, all values will be ``<= max_value`` - (or ``< max_value`` if ``exclude_max``). - - If min_value or max_value is not None, it is an error to enable - allow_nan. - - If both min_value and max_value are not None, it is an error to enable - allow_infinity. - - Where not explicitly ruled out by the bounds, all of infinity, -infinity - and NaN are possible values generated by this strategy. - - The width argument specifies the maximum number of bits of precision - required to represent the generated float. Valid values are 16, 32, or 64. - Passing ``width=32`` will still use the builtin 64-bit ``float`` class, - but always for values which can be exactly represented as a 32-bit float. - - The exclude_min and exclude_max argument can be used to generate numbers - from open or half-open intervals, by excluding the respective endpoints. - Excluding either signed zero will also exclude the other. - Attempting to exclude an endpoint which is None will raise an error; - use ``allow_infinity=False`` to generate finite floats. You can however - use e.g. ``min_value=-math.inf, exclude_min=True`` to exclude only - one infinite endpoint. - - Examples from this strategy have a complicated and hard to explain - shrinking behaviour, but it tries to improve "human readability". Finite - numbers will be preferred to infinity and infinity will be preferred to - NaN. - """ - check_type(bool, exclude_min, "exclude_min") - check_type(bool, exclude_max, "exclude_max") - - if allow_nan is None: - allow_nan = bool(min_value is None and max_value is None) - elif allow_nan and (min_value is not None or max_value is not None): - raise InvalidArgument( - "Cannot have allow_nan=%r, with min_value or max_value" % (allow_nan) - ) - - if width not in (16, 32, 64): - raise InvalidArgument( - "Got width=%r, but the only valid values are the integers 16, " - "32, and 64." % (width,) - ) - - check_valid_bound(min_value, "min_value") - check_valid_bound(max_value, "max_value") - - min_arg, max_arg = min_value, max_value - if min_value is not None: - min_value = float_of(min_value, width) - assert isinstance(min_value, float) - if max_value is not None: - max_value = float_of(max_value, width) - assert isinstance(max_value, float) - - if min_value != min_arg: - raise InvalidArgument( - f"min_value={min_arg!r} cannot be exactly represented as a float " - f"of width {width} - use min_value={min_value!r} instead." - ) - if max_value != max_arg: - raise InvalidArgument( - f"max_value={max_arg!r} cannot be exactly represented as a float " - f"of width {width} - use max_value={max_value!r} instead." - ) - - if exclude_min and (min_value is None or min_value == math.inf): - raise InvalidArgument(f"Cannot exclude min_value={min_value!r}") - if exclude_max and (max_value is None or max_value == -math.inf): - raise InvalidArgument(f"Cannot exclude max_value={max_value!r}") - - if min_value is not None and ( - exclude_min or (min_arg is not None and min_value < min_arg) - ): - min_value = next_up(min_value, width) - if min_value == min_arg: - assert min_value == min_arg == 0 - assert is_negative(min_arg) and not is_negative(min_value) - min_value = next_up(min_value, width) - assert min_value > min_arg # type: ignore - if max_value is not None and ( - exclude_max or (max_arg is not None and max_value > max_arg) - ): - max_value = next_down(max_value, width) - if max_value == max_arg: - assert max_value == max_arg == 0 - assert is_negative(max_value) and not is_negative(max_arg) - max_value = next_down(max_value, width) - assert max_value < max_arg # type: ignore - - if min_value == -math.inf: - min_value = None - if max_value == math.inf: - max_value = None - - bad_zero_bounds = ( - min_value == max_value == 0 - and is_negative(max_value) - and not is_negative(min_value) - ) - if ( - min_value is not None - and max_value is not None - and (min_value > max_value or bad_zero_bounds) - ): - # This is a custom alternative to check_valid_interval, because we want - # to include the bit-width and exclusion information in the message. - msg = ( - "There are no %s-bit floating-point values between min_value=%r " - "and max_value=%r" % (width, min_arg, max_arg) - ) - if exclude_min or exclude_max: - msg += f", exclude_min={exclude_min!r} and exclude_max={exclude_max!r}" - raise InvalidArgument(msg) - - if allow_infinity is None: - allow_infinity = bool(min_value is None or max_value is None) - elif allow_infinity: - if min_value is not None and max_value is not None: - raise InvalidArgument( - "Cannot have allow_infinity=%r, with both min_value and " - "max_value" % (allow_infinity) - ) - elif min_value == math.inf: - raise InvalidArgument("allow_infinity=False excludes min_value=inf") - elif max_value == -math.inf: - raise InvalidArgument("allow_infinity=False excludes max_value=-inf") - - unbounded_floats = FloatStrategy( - allow_infinity=allow_infinity, allow_nan=allow_nan, width=width - ) - - if min_value is None and max_value is None: - return unbounded_floats - elif min_value is not None and max_value is not None: - if min_value == max_value: - assert isinstance(min_value, float) - result = just(min_value) - elif is_negative(min_value): - if is_negative(max_value): - return floats( - min_value=-max_value, max_value=-min_value, width=width - ).map(operator.neg) - else: - return one_of( - floats(min_value=0.0, max_value=max_value, width=width), - floats(min_value=0.0, max_value=-min_value, width=width).map( - operator.neg - ), - ) - elif count_between_floats(min_value, max_value) > 1000: - return FixedBoundedFloatStrategy( - lower_bound=min_value, upper_bound=max_value, width=width - ) - else: - ub_int = float_to_int(max_value, width) - lb_int = float_to_int(min_value, width) - assert lb_int <= ub_int - result = integers(min_value=lb_int, max_value=ub_int).map( - lambda x: int_to_float(x, width) - ) - elif min_value is not None: - assert isinstance(min_value, float) - if is_negative(min_value): - return one_of( - unbounded_floats.map(abs), - floats(min_value=min_value, max_value=-0.0, width=width), - ) - else: - result = unbounded_floats.map(lambda x: min_value + abs(x)) - else: - assert isinstance(max_value, float) - if not is_negative(max_value): - return one_of( - floats(min_value=0.0, max_value=max_value, width=width), - unbounded_floats.map(lambda x: -abs(x)), - ) - else: - result = unbounded_floats.map(lambda x: max_value - abs(x)) - - if width < 64: - - def downcast(x): - try: - return float_of(x, width) - except OverflowError: # pragma: no cover - reject() - - result = result.map(downcast) - if not allow_infinity: - result = result.filter(lambda x: not math.isinf(x)) - return result - - -@cacheable -@defines_strategy() -def tuples(*args: SearchStrategy) -> SearchStrategy[tuple]: - """Return a strategy which generates a tuple of the same length as args by - generating the value at index i from args[i]. - - e.g. tuples(integers(), integers()) would generate a tuple of length - two with both values an integer. - - Examples from this strategy shrink by shrinking their component parts. - """ - for arg in args: - check_strategy(arg) - - return TupleStrategy(args) - - @overload -def sampled_from(elements: Sequence[T]) -> SearchStrategy[T]: - raise NotImplementedError +def sampled_from(elements: Sequence[T]) -> SearchStrategy[T]: # pragma: no cover + ... @overload # noqa: F811 -def sampled_from(elements: Type[enum.Enum]) -> SearchStrategy[Any]: +def sampled_from(elements: Type[enum.Enum]) -> SearchStrategy[Any]: # pragma: no cover # `SearchStrategy[Enum]` is unreliable due to metaclass issues. - raise NotImplementedError + ... + + +@overload # noqa: F811 +def sampled_from( + elements: Union[Type[enum.Enum], Sequence[Any]] +) -> SearchStrategy[Any]: # pragma: no cover + ... @defines_strategy(try_non_lazy=True) # noqa: F811 -def sampled_from(elements): +def sampled_from( + elements: Union[Type[enum.Enum], Sequence[Any]] +) -> SearchStrategy[Any]: """Returns a strategy which generates any value present in ``elements``. Note that as with :func:`~hypothesis.strategies.just`, values will not be @@ -725,6 +186,17 @@ def sampled_from(elements): """ values = check_sample(elements, "sampled_from") if not values: + if ( + isinstance(elements, type) + and issubclass(elements, enum.Enum) + and vars(elements).get("__annotations__") + ): + # See https://github.com/HypothesisWorks/hypothesis/issues/2923 + raise InvalidArgument( + f"Cannot sample from {elements.__module__}.{elements.__name__} " + "because it contains no elements. It does however have annotations, " + "so maybe you tried to write an enum as if it was a dataclass?" + ) raise InvalidArgument("Cannot sample from a length-zero sequence.") if len(values) == 1: return just(values[0]) @@ -754,7 +226,11 @@ def lists( *, min_size: int = 0, max_size: Optional[int] = None, - unique_by: Optional[UniqueBy] = None, + unique_by: Union[ + None, + Callable[[Ex], Hashable], + Tuple[Callable[[Ex], Hashable], ...], + ] = None, unique: bool = False, ) -> SearchStrategy[List[Ex]]: """Returns a list containing values drawn from elements with length in the @@ -800,7 +276,7 @@ def lists( if unique_by is not None: if not (callable(unique_by) or isinstance(unique_by, tuple)): raise InvalidArgument( - "unique_by=%r is not a callable or tuple of callables" % (unique_by) + f"unique_by={unique_by!r} is not a callable or tuple of callables" ) if callable(unique_by): unique_by = (unique_by,) @@ -808,7 +284,7 @@ def lists( raise InvalidArgument("unique_by is empty") for i, f in enumerate(unique_by): if not callable(f): - raise InvalidArgument("unique_by[%i]=%r is not a callable" % (i, f)) + raise InvalidArgument(f"unique_by[{i}]={f!r} is not a callable") # Note that lazy strategies automatically unwrap when passed to a defines_strategy # function. tuple_suffixes = None @@ -821,7 +297,7 @@ def lists( and len(unique_by) == 1 and ( # Introspection for either `itemgetter(0)`, or `lambda x: x[0]` - isinstance(unique_by[0], operator.itemgetter) # type: ignore + isinstance(unique_by[0], operator.itemgetter) and repr(unique_by[0]) == "operator.itemgetter(0)" or isinstance(unique_by[0], FunctionType) and re.fullmatch( @@ -834,6 +310,20 @@ def lists( tuple_suffixes = TupleStrategy(elements.element_strategies[1:]) elements = elements.element_strategies[0] + # UniqueSampledListStrategy offers a substantial performance improvement for + # unique arrays with few possible elements, e.g. of eight-bit integer types. + if ( + isinstance(elements, IntegersStrategy) + and None not in (elements.start, elements.end) + and (elements.end - elements.start) <= 255 + ): + elements = SampledFromStrategy( + sorted(range(elements.start, elements.end + 1), key=abs) + if elements.end < 0 or elements.start > 0 + else list(range(0, elements.end + 1)) + + list(range(-1, elements.start - 1, -1)) + ) + if isinstance(elements, SampledFromStrategy): element_count = len(elements.elements) if min_size > element_count: @@ -924,7 +414,11 @@ def iterables( *, min_size: int = 0, max_size: Optional[int] = None, - unique_by: Optional[UniqueBy] = None, + unique_by: Union[ + None, + Callable[[Ex], Hashable], + Tuple[Callable[[Ex], Hashable], ...], + ] = None, unique: bool = False, ) -> SearchStrategy[Iterable[Ex]]: """This has the same behaviour as lists, but returns iterables instead. @@ -952,13 +446,12 @@ def fixed_dictionaries( """Generates a dictionary of the same type as mapping with a fixed set of keys mapping to strategies. ``mapping`` must be a dict subclass. - Generated values have all keys present in mapping, with the - corresponding values drawn from mapping[key]. If mapping is an - instance of OrderedDict the keys will also be in the same order, - otherwise the order is arbitrary. + Generated values have all keys present in mapping, in iteration order, + with the corresponding values drawn from mapping[key]. If ``optional`` is passed, the generated value *may or may not* contain each key from ``optional`` and a value drawn from the corresponding strategy. + Generated values may contain optional keys in an arbitrary order. Examples from this strategy shrink by shrinking each individual value in the generated dictionary, and omitting optional key-value pairs. @@ -978,7 +471,7 @@ def fixed_dictionaries( if set(mapping) & set(optional): raise InvalidArgument( "The following keys were in both mapping and optional, " - "which is invalid: %r" % (set(mapping) & set(optional)) + f"which is invalid: {set(mapping) & set(optional)!r}" ) return FixedAndOptionalKeysDictStrategy(mapping, optional) return FixedKeysDictStrategy(mapping) @@ -1087,9 +580,9 @@ def characters( overlap = set(blacklist_characters).intersection(whitelist_characters) if overlap: raise InvalidArgument( - "Characters %r are present in both whitelist_characters=%r, and " - "blacklist_characters=%r" - % (sorted(overlap), whitelist_characters, blacklist_characters) + f"Characters {sorted(overlap)!r} are present in both " + f"whitelist_characters={whitelist_characters!r}, and " + f"blacklist_characters={blacklist_characters!r}" ) blacklist_categories = as_general_categories( blacklist_categories, "blacklist_categories" @@ -1110,9 +603,9 @@ def characters( both_cats = set(blacklist_categories or ()).intersection(whitelist_categories or ()) if both_cats: raise InvalidArgument( - "Categories %r are present in both whitelist_categories=%r, and " - "blacklist_categories=%r" - % (sorted(both_cats), whitelist_categories, blacklist_categories) + f"Categories {sorted(both_cats)!r} are present in both " + f"whitelist_categories={whitelist_categories!r}, and " + f"blacklist_categories={blacklist_categories!r}" ) return OneCharStringStrategy( @@ -1125,6 +618,19 @@ def characters( ) +# Cache size is limited by sys.maxunicode, but passing None makes it slightly faster. +@lru_cache(maxsize=None) +def _check_is_single_character(c): + # In order to mitigate the performance cost of this check, we use a shared cache, + # even at the cost of showing the culprit strategy in the error message. + if not isinstance(c, str): + type_ = get_pretty_function_description(type(c)) + raise InvalidArgument(f"Got non-string {c!r} (type {type_})") + if len(c) != 1: + raise InvalidArgument(f"Got {c!r} (length {len(c)} != 1)") + return c + + @cacheable @defines_strategy(force_reusable_values=True) def text( @@ -1155,20 +661,25 @@ def text( """ check_valid_sizes(min_size, max_size) if isinstance(alphabet, SearchStrategy): - char_strategy = alphabet + char_strategy = unwrap_strategies(alphabet) + if isinstance(char_strategy, SampledFromStrategy): + # Check this via the up-front validation logic below, and incidentally + # convert into a `characters()` strategy for standard text shrinking. + return text(char_strategy.elements, min_size=min_size, max_size=max_size) + elif not isinstance(char_strategy, OneCharStringStrategy): + char_strategy = char_strategy.map(_check_is_single_character) else: non_string = [c for c in alphabet if not isinstance(c, str)] if non_string: raise InvalidArgument( "The following elements in alphabet are not unicode " - "strings: %r" % (non_string,) + f"strings: {non_string!r}" ) not_one_char = [c for c in alphabet if len(c) != 1] if not_one_char: raise InvalidArgument( - "The following elements in alphabet are not of length " - "one, which leads to violation of size constraints: %r" - % (not_one_char,) + "The following elements in alphabet are not of length one, " + f"which leads to violation of size constraints: {not_one_char!r}" ) char_strategy = ( characters(whitelist_categories=(), whitelist_characters=alphabet) @@ -1177,7 +688,7 @@ def text( ) if (max_size == 0 or char_strategy.is_empty) and not min_size: return just("") - return lists(char_strategy, min_size=min_size, max_size=max_size).map("".join) + return TextStrategy(char_strategy, min_size=min_size, max_size=max_size) @cacheable @@ -1284,7 +795,11 @@ def __repr__(self): class RandomModule(SearchStrategy): def do_draw(self, data): - seed = data.draw(integers(0, 2 ** 32 - 1)) + # It would be unsafe to do run this method more than once per test case, + # because cleanup() runs tasks in FIFO order (at time of writing!). + # Fortunately, the random_module() strategy wraps us in shared(), so + # it's cached for all but the first of any number of calls. + seed = data.draw(integers(0, 2**32 - 1)) seed_all, restore_all = get_seeder_and_restorer(seed) seed_all() cleanup(restore_all) @@ -1319,7 +834,7 @@ def __init__(self, target, args, kwargs): def do_draw(self, data): try: return self.target( - *[data.draw(a) for a in self.args], + *(data.draw(a) for a in self.args), **{k: data.draw(v) for k, v in self.kwargs.items()}, ) except TypeError as err: @@ -1333,12 +848,34 @@ def do_draw(self, data): f"Calling {name} with no arguments raised an error - " f"try using sampled_from({name}) instead of builds({name})" ) from err + if not (self.args or self.kwargs): + from .types import is_a_new_type, is_generic_type + + if is_a_new_type(self.target) or is_generic_type(self.target): + raise InvalidArgument( + f"Calling {self.target!r} with no arguments raised an " + f"error - try using from_type({self.target!r}) instead " + f"of builds({self.target!r})" + ) from err + if getattr(self.target, "__no_type_check__", None) is True: + # Note: could use PEP-678 __notes__ here. Migrate over once we're + # using an `exceptiongroup` backport with support for that. + raise TypeError( + "This might be because the @no_type_check decorator prevented " + "Hypothesis from inferring a strategy for some required arguments." + ) from err raise def validate(self): tuples(*self.args).validate() fixed_dictionaries(self.kwargs).validate() + def __repr__(self): + bits = [get_pretty_function_description(self.target)] + bits.extend(map(repr, self.args)) + bits.extend(f"{k}={v!r}" for k, v in self.kwargs.items()) + return f"builds({', '.join(bits)})" + # The ideal signature builds(target, /, *args, **kwargs) is unfortunately a # SyntaxError before Python 3.8 so we emulate it with manual argument unpacking. @@ -1348,7 +885,7 @@ def validate(self): @defines_strategy() def builds( *callable_and_args: Union[Callable[..., Ex], SearchStrategy[Any]], - **kwargs: Union[SearchStrategy[Any], InferType], + **kwargs: Union[SearchStrategy[Any], EllipsisType], ) -> SearchStrategy[Ex]: """Generates values by drawing from ``args`` and ``kwargs`` and passing them to the callable (provided as the first positional argument) in the @@ -1359,9 +896,9 @@ def builds( If the callable has type annotations, they will be used to infer a strategy for required arguments that were not passed to builds. You can also tell - builds to infer a strategy for an optional argument by passing the special - value :const:`hypothesis.infer` as a keyword argument to - builds, instead of a strategy for that argument to the callable. + builds to infer a strategy for an optional argument by passing ``...`` + (:obj:`python:Ellipsis`) as a keyword argument to builds, instead of a strategy for + that argument to the callable. If the callable is a class defined with :pypi:`attrs`, missing required arguments will be inferred from the attribute on a best-effort basis, @@ -1372,7 +909,7 @@ def builds( the callable. """ if not callable_and_args: - raise InvalidArgument( + raise InvalidArgument( # pragma: no cover "builds() must be passed a callable as the first positional " "argument, but no positional arguments were given." ) @@ -1383,14 +920,14 @@ def builds( "target to construct." ) - if infer in args: + if ... in args: # type: ignore # we only annotated the allowed types # Avoid an implementation nightmare juggling tuples and worse things raise InvalidArgument( - "infer was passed as a positional argument to " + "... was passed as a positional argument to " "builds(), but is only allowed as a keyword arg" ) - required = required_args(target, args, kwargs) or set() - to_infer = {k for k, v in kwargs.items() if v is infer} + required = required_args(target, args, kwargs) + to_infer = {k for k, v in kwargs.items() if v is ...} if required or to_infer: if isinstance(target, type) and attr.has(target): # Use our custom introspection for attrs classes @@ -1400,37 +937,53 @@ def builds( # Otherwise, try using type hints hints = get_type_hints(target) if to_infer - set(hints): + badargs = ", ".join(sorted(to_infer - set(hints))) raise InvalidArgument( - "passed infer for %s, but there is no type annotation" - % (", ".join(sorted(to_infer - set(hints)))) + f"passed ... for {badargs}, but we cannot infer a strategy " + "because these arguments have no type annotation" ) - for kw in set(hints) & (required | to_infer): - kwargs[kw] = from_type(hints[kw]) - - # Mypy doesn't realise that `infer` is gone from kwargs now - # and thinks that target and args have the same (union) type. + infer_for = {k: v for k, v in hints.items() if k in (required | to_infer)} + if infer_for: + from hypothesis.strategies._internal.types import _global_type_lookup + + for kw, t in infer_for.items(): + if ( + getattr(t, "__module__", None) in ("builtins", "typing") + or t in _global_type_lookup + ): + kwargs[kw] = from_type(t) + else: + # We defer resolution of these type annotations so that the obvious + # approach to registering recursive types just works. See + # https://github.com/HypothesisWorks/hypothesis/issues/3026 + kwargs[kw] = deferred(lambda t=t: from_type(t)) # type: ignore return BuildsStrategy(target, args, kwargs) -if sys.version_info[:2] >= (3, 8): # pragma: no cover +if sys.version_info[:2] >= (3, 8): # pragma: no branch # See notes above definition - this signature is compatible and better # matches the semantics of the function. Great for documentation! sig = signature(builds) args, kwargs = sig.parameters.values() - builds.__signature__ = sig.replace( - parameters=[ - Parameter( - name="target", - kind=Parameter.POSITIONAL_ONLY, - annotation=Callable[..., Ex], - ), - args.replace(name="args", annotation=SearchStrategy[Any]), - kwargs, - ] - ) + builds = define_function_signature( + name=builds.__name__, + docstring=builds.__doc__, + signature=sig.replace( + parameters=[ + Parameter( + name="target", + kind=Parameter.POSITIONAL_ONLY, + annotation=Callable[..., Ex], + ), + args.replace(name="args", annotation=SearchStrategy[Any]), + kwargs, + ] + ), + )(builds) @cacheable +@defines_strategy(never_lazy=True) def from_type(thing: Type[Ex]) -> SearchStrategy[Ex]: """Looks up the appropriate search strategy for the given type. @@ -1503,43 +1056,18 @@ def _from_type(thing: Type[Ex]) -> SearchStrategy[Ex]: # refactoring it's hard to do without creating circular imports. from hypothesis.strategies._internal import types - if ( - hasattr(typing, "_TypedDictMeta") - and type(thing) is typing._TypedDictMeta # type: ignore - or hasattr(types.typing_extensions, "_TypedDictMeta") - and type(thing) is types.typing_extensions._TypedDictMeta # type: ignore - ): # pragma: no cover - # The __optional_keys__ attribute may or may not be present, but if there's no - # way to tell and we just have to assume that everything is required. - # See https://github.com/python/cpython/pull/17214 for details. - optional = getattr(thing, "__optional_keys__", ()) - anns = {k: from_type(v) for k, v in thing.__annotations__.items()} - return fixed_dictionaries( # type: ignore - mapping={k: v for k, v in anns.items() if k not in optional}, - optional={k: v for k, v in anns.items() if k in optional}, - ) - def as_strategy(strat_or_callable, thing, final=True): # User-provided strategies need some validation, and callables even more # of it. We do this in three places, hence the helper function if not isinstance(strat_or_callable, SearchStrategy): assert callable(strat_or_callable) # Validated in register_type_strategy - try: - # On Python 3.6, typing.Hashable is just an alias for abc.Hashable, - # and the resolver function for Type throws an AttributeError because - # Hashable has no __args__. We discard such errors when attempting - # to resolve subclasses, because the function was passed a weird arg. - strategy = strat_or_callable(thing) - except Exception: # pragma: no cover - if not final: - return NOTHING - raise + strategy = strat_or_callable(thing) else: strategy = strat_or_callable if not isinstance(strategy, SearchStrategy): raise ResolutionFailed( - "Error: %s was registered for %r, but returned non-strategy %r" - % (thing, nicerepr(strat_or_callable), strategy) + f"Error: {thing} was registered for {nicerepr(strat_or_callable)}, " + f"but returned non-strategy {strategy!r}" ) if strategy.is_empty: raise ResolutionFailed(f"Error: {thing!r} resolved to an empty strategy") @@ -1552,30 +1080,29 @@ def as_strategy(strat_or_callable, thing, final=True): if thing in types._global_type_lookup: return as_strategy(types._global_type_lookup[thing], thing) return from_type(thing.__supertype__) - # Under Python 3.6, Unions are not instances of `type` - but we - # still want to resolve them! - if getattr(thing, "__origin__", None) is typing.Union: + # Unions are not instances of `type` - but we still want to resolve them! + if types.is_a_union(thing): args = sorted(thing.__args__, key=types.type_sorting_key) return one_of([from_type(t) for t in args]) if not types.is_a_type(thing): - # The implementation of typing_extensions.Literal under Python 3.6 is - # *very strange*. Notably, `type(Literal[x]) != Literal` so we have to - # use the first form directly, and because it uses __values__ instead of - # __args__ we inline the relevant logic here until the end of life date. - if types.is_typing_literal(thing): # pragma: no cover - assert sys.version_info[:2] == (3, 6) - args_dfs_stack = list(thing.__values__) # type: ignore - literals = [] - while args_dfs_stack: - arg = args_dfs_stack.pop() - if types.is_typing_literal(arg): - args_dfs_stack.extend(reversed(arg.__values__)) - else: - literals.append(arg) - return sampled_from(literals) - raise InvalidArgument(f"thing={thing} must be a type") + if isinstance(thing, str): + # See https://github.com/HypothesisWorks/hypothesis/issues/3016 + raise InvalidArgument( + f"Got {thing!r} as a type annotation, but the forward-reference " + "could not be resolved from a string to a type. Consider using " + "`from __future__ import annotations` instead of forward-reference " + "strings." + ) + raise InvalidArgument(f"thing={thing!r} must be a type") # pragma: no cover + if thing in types.NON_RUNTIME_TYPES: + # Some code like `st.from_type(TypeAlias)` does not make sense. + # Because there are types in python that do not exist in runtime. + raise InvalidArgument( + f"Could not resolve {thing!r} to a strategy, " + f"because there is no such thing as a runtime instance of {thing!r}" + ) # Now that we know `thing` is a type, the first step is to check for an - # explicitly registered strategy. This is the best (and hopefully most + # explicitly registered strategy. This is the best (and hopefully most # common) way to resolve a type to a strategy. Note that the value in the # lookup may be a strategy or a function from type -> strategy; and we # convert empty results into an explicit error. @@ -1587,6 +1114,41 @@ def as_strategy(strat_or_callable, thing, final=True): # typing.Callable[[], foo] has __args__ = (foo,) but collections.abc.Callable # has __args__ = ([], foo); and as a result is non-hashable. pass + if ( + hasattr(typing, "_TypedDictMeta") + and type(thing) is typing._TypedDictMeta + or hasattr(types.typing_extensions, "_TypedDictMeta") # type: ignore + and type(thing) is types.typing_extensions._TypedDictMeta # type: ignore + ): # pragma: no cover + # The __optional_keys__ attribute may or may not be present, but if there's no + # way to tell and we just have to assume that everything is required. + # See https://github.com/python/cpython/pull/17214 for details. + optional = set(getattr(thing, "__optional_keys__", ())) + anns = {} + for k, v in get_type_hints(thing).items(): + origin = getattr(v, "__origin__", None) + if origin in types.RequiredTypes + types.NotRequiredTypes: + if origin in types.NotRequiredTypes: + optional.add(k) + else: + optional.discard(k) + try: + v = v.__args__[0] + except IndexError: + raise InvalidArgument( + f"`{k}: {v.__name__}` is not a valid type annotation" + ) from None + anns[k] = from_type(v) + if ( + (not anns) + and thing.__annotations__ + and ".." in getattr(thing, "__qualname__", "") + ): + raise InvalidArgument("Failed to retrieve type annotations for local type") + return fixed_dictionaries( # type: ignore + mapping={k: v for k, v in anns.items() if k not in optional}, + optional={k: v for k, v in anns.items() if k in optional}, + ) # We also have a special case for TypeVars. # They are represented as instances like `~T` when they come here. # We need to work with their type instead. @@ -1597,7 +1159,7 @@ def as_strategy(strat_or_callable, thing, final=True): # We'll start by checking if thing is from from the typing module, # because there are several special cases that don't play well with # subclass and instance checks. - if isinstance(thing, typing_root_type) or ( + if isinstance(thing, types.typing_root_type) or ( sys.version_info[:2] >= (3, 9) and isinstance(getattr(thing, "__origin__", None), type) and getattr(thing, "__args__", None) @@ -1620,20 +1182,7 @@ def as_strategy(strat_or_callable, thing, final=True): # may be able to fall back on type annotations. if issubclass(thing, enum.Enum): return sampled_from(thing) - # If we know that builds(thing) will fail, give a better error message - required = required_args(thing) - if required and not any( - [ - required.issubset(get_type_hints(thing)), - attr.has(thing), - # NamedTuples are weird enough that we need a specific check for them. - is_typed_named_tuple(thing), - ] - ): - raise ResolutionFailed( - "Could not resolve %r to a strategy; consider " - "using register_type_strategy" % (thing,) - ) + # Finally, try to build an instance by calling the type object. Unlike builds(), # this block *does* try to infer strategies for arguments with default values. # That's because of the semantic different; builds() -> "call this with ..." @@ -1641,9 +1190,20 @@ def as_strategy(strat_or_callable, thing, final=True): # me arbitrary instances" so the greater variety is acceptable. # And if it's *too* varied, express your opinions with register_type_strategy() if not isabstract(thing): + # If we know that builds(thing) will fail, give a better error message + required = required_args(thing) + if required and not ( + required.issubset(get_type_hints(thing)) + or attr.has(thing) + or is_typed_named_tuple(thing) # weird enough that we have a specific check + ): + raise ResolutionFailed( + f"Could not resolve {thing!r} to a strategy; consider " + "using register_type_strategy" + ) try: hints = get_type_hints(thing) - params = signature(thing).parameters + params = get_signature(thing).parameters except Exception: return builds(thing) kwargs = {} @@ -1660,10 +1220,20 @@ def as_strategy(strat_or_callable, thing, final=True): subclasses = thing.__subclasses__() if not subclasses: raise ResolutionFailed( - "Could not resolve %r to a strategy, because it is an abstract type " - "without any subclasses. Consider using register_type_strategy" % (thing,) + f"Could not resolve {thing!r} to a strategy, because it is an abstract " + "type without any subclasses. Consider using register_type_strategy" ) - return sampled_from(subclasses).flatmap(from_type) + subclass_strategies = nothing() + for sc in subclasses: + try: + subclass_strategies |= _from_type(sc) + except Exception: + pass + if subclass_strategies.is_empty: + # We're unable to resolve subclasses now, but we might be able to later - + # so we'll just go back to the mixed distribution. + return sampled_from(subclasses).flatmap(from_type) + return subclass_strategies @cacheable @@ -1699,16 +1269,16 @@ def fractions( if max_denominator is not None: if max_denominator < 1: - raise InvalidArgument("max_denominator=%r must be >= 1" % max_denominator) + raise InvalidArgument(f"max_denominator={max_denominator!r} must be >= 1") if min_value is not None and min_value.denominator > max_denominator: raise InvalidArgument( - "The min_value=%r has a denominator greater than the " - "max_denominator=%r" % (min_value, max_denominator) + f"The min_value={min_value!r} has a denominator greater than the " + f"max_denominator={max_denominator!r}" ) if max_value is not None and max_value.denominator > max_denominator: raise InvalidArgument( - "The max_value=%r has a denominator greater than the " - "max_denominator=%r" % (max_value, max_denominator) + f"The max_value={max_value!r} has a denominator greater than the " + f"max_denominator={max_denominator!r}" ) if min_value is not None and min_value == max_value: @@ -1808,7 +1378,7 @@ def decimals( # Convert min_value and max_value to Decimal values, and validate args check_valid_integer(places, "places") if places is not None and places < 0: - raise InvalidArgument("places=%r may not be negative" % places) + raise InvalidArgument(f"places={places!r} may not be negative") min_value = _as_finite_decimal(min_value, "min_value", allow_infinity) max_value = _as_finite_decimal(max_value, "max_value", allow_infinity) check_valid_interval(min_value, max_value, "min_value", "max_value") @@ -1854,16 +1424,17 @@ def fraction_to_decimal(val): strat = fractions(min_value, max_value).map(fraction_to_decimal) # Compose with sampled_from for infinities and NaNs as appropriate - special = [] # type: List[Decimal] + special: List[Decimal] = [] if allow_nan or (allow_nan is None and (None in (min_value, max_value))): special.extend(map(Decimal, ("NaN", "-NaN", "sNaN", "-sNaN"))) - if allow_infinity or (allow_infinity is max_value is None): + if allow_infinity or (allow_infinity is None and max_value is None): special.append(Decimal("Infinity")) - if allow_infinity or (allow_infinity is min_value is None): + if allow_infinity or (allow_infinity is None and min_value is None): special.append(Decimal("-Infinity")) return strat | (sampled_from(special) if special else nothing()) +@defines_strategy(never_lazy=True) def recursive( base: SearchStrategy[Ex], extend: Callable[[SearchStrategy[Any]], SearchStrategy[T]], @@ -1938,52 +1509,115 @@ def calc_label(self): return calc_label_from_cls(self.definition) -@cacheable -def composite(f: Callable[..., Ex]) -> Callable[..., SearchStrategy[Ex]]: - """Defines a strategy that is built out of potentially arbitrarily many - other strategies. +class DrawFn(Protocol): + """This type only exists so that you can write type hints for functions + decorated with :func:`@composite `. - This is intended to be used as a decorator. See - :ref:`the full documentation for more details ` - about how to use this function. + .. code-block:: python + + @composite + def list_and_index(draw: DrawFn) -> Tuple[int, str]: + i = draw(integers()) # type inferred as 'int' + s = draw(text()) # type inferred as 'str' + return i, s - Examples from this strategy shrink by shrinking the output of each draw - call. """ + + def __init__(self): + raise TypeError("Protocols cannot be instantiated") # pragma: no cover + + # On Python 3.8+, Protocol overrides our signature for __init__, + # so we override it right back to make the docs look nice. + __signature__: Signature = Signature(parameters=[]) + + # We define this as a callback protocol because a simple typing.Callable is + # insufficient to fully represent the interface, due to the optional `label` + # parameter. + def __call__(self, strategy: SearchStrategy[Ex], label: object = None) -> Ex: + raise NotImplementedError + + +def _composite(f): + # Wrapped below, using ParamSpec if available if isinstance(f, (classmethod, staticmethod)): special_method = type(f) f = f.__func__ else: special_method = None - argspec = getfullargspec(f) + sig = get_signature(f) + params = tuple(sig.parameters.values()) - if argspec.defaults is not None and len(argspec.defaults) == len(argspec.args): - raise InvalidArgument("A default value for initial argument will never be used") - if len(argspec.args) == 0 and not argspec.varargs: + if not (params and "POSITIONAL" in params[0].kind.name): raise InvalidArgument( "Functions wrapped with composite must take at least one " "positional argument." ) - - annots = { - k: v - for k, v in argspec.annotations.items() - if k in (argspec.args + argspec.kwonlyargs + ["return"]) - } - new_argspec = argspec._replace(args=argspec.args[1:], annotations=annots) + if params[0].default is not sig.empty: + raise InvalidArgument("A default value for initial argument will never be used") + if not is_first_param_referenced_in_function(f): + note_deprecation( + "There is no reason to use @st.composite on a function which " + + "does not call the provided draw() function internally.", + since="2022-07-17", + has_codemod=False, + ) + if params[0].kind.name != "VAR_POSITIONAL": + params = params[1:] + newsig = sig.replace( + parameters=params, + return_annotation=SearchStrategy + if sig.return_annotation is sig.empty + else SearchStrategy[sig.return_annotation], + ) @defines_strategy() - @define_function_signature(f.__name__, f.__doc__, new_argspec) + @define_function_signature(f.__name__, f.__doc__, newsig) def accept(*args, **kwargs): return CompositeStrategy(f, args, kwargs) accept.__module__ = f.__module__ + accept.__signature__ = newsig if special_method is not None: return special_method(accept) return accept +if typing.TYPE_CHECKING or ParamSpec is not None: + P = ParamSpec("P") + + def composite( + f: Callable[Concatenate[DrawFn, P], Ex] + ) -> Callable[P, SearchStrategy[Ex]]: + """Defines a strategy that is built out of potentially arbitrarily many + other strategies. + + This is intended to be used as a decorator. See + :ref:`the full documentation for more details ` + about how to use this function. + + Examples from this strategy shrink by shrinking the output of each draw + call. + """ + return _composite(f) + +else: # pragma: no cover + + @cacheable + def composite(f: Callable[..., Ex]) -> Callable[..., SearchStrategy[Ex]]: + """Defines a strategy that is built out of potentially arbitrarily many + other strategies. + + This is intended to be used as a decorator. See + :ref:`the full documentation for more details ` + about how to use this function. + + Examples from this strategy shrink by shrinking the output of each draw + call. + """ + return _composite(f) + + @defines_strategy(force_reusable_values=True) @cacheable def complex_numbers( @@ -1992,6 +1626,7 @@ def complex_numbers( max_magnitude: Optional[Real] = None, allow_infinity: Optional[bool] = None, allow_nan: Optional[bool] = None, + allow_subnormal: bool = True, ) -> SearchStrategy[complex]: """Returns a strategy that generates complex numbers. @@ -2004,7 +1639,10 @@ def complex_numbers( is an error to enable ``allow_nan``. If ``max_magnitude`` is finite, it is an error to enable ``allow_infinity``. - The magnitude contraints are respected up to a relative error + ``allow_subnormal`` is applied to each part of the complex number + separately, as for :func:`~hypothesis.strategies.floats`. + + The magnitude constraints are respected up to a relative error of (around) floating-point epsilon, due to implementation via the system ``sqrt`` function. @@ -2026,23 +1664,32 @@ def complex_numbers( allow_infinity = bool(max_magnitude is None) elif allow_infinity and max_magnitude is not None: raise InvalidArgument( - "Cannot have allow_infinity=%r with max_magnitude=%r" - % (allow_infinity, max_magnitude) + f"Cannot have allow_infinity={allow_infinity!r} with " + f"max_magnitude={max_magnitude!r}" ) if allow_nan is None: allow_nan = bool(min_magnitude == 0 and max_magnitude is None) elif allow_nan and not (min_magnitude == 0 and max_magnitude is None): raise InvalidArgument( - "Cannot have allow_nan=%r, min_magnitude=%r max_magnitude=%r" - % (allow_nan, min_magnitude, max_magnitude) + f"Cannot have allow_nan={allow_nan!r}, min_magnitude={min_magnitude!r} " + f"max_magnitude={max_magnitude!r}" ) - allow_kw = {"allow_nan": allow_nan, "allow_infinity": allow_infinity} + + check_type(bool, allow_subnormal, "allow_subnormal") + allow_kw = { + "allow_nan": allow_nan, + "allow_infinity": allow_infinity, + # If we have a nonzero normal min_magnitude and draw a zero imaginary part, + # then allow_subnormal=True would be an error with the min_value to the floats() + # strategy for the real part. We therefore replace True with None. + "allow_subnormal": None if allow_subnormal else allow_subnormal, + } if min_magnitude == 0 and max_magnitude is None: # In this simple but common case, there are no constraints on the # magnitude and therefore no relationship between the real and # imaginary parts. - return builds(complex, floats(**allow_kw), floats(**allow_kw)) + return builds(complex, floats(**allow_kw), floats(**allow_kw)) # type: ignore @composite def constrained_complex(draw): @@ -2069,6 +1716,7 @@ def constrained_complex(draw): return constrained_complex() +@defines_strategy(never_lazy=True) def shared( base: SearchStrategy[Ex], *, @@ -2093,32 +1741,47 @@ def shared( return SharedStrategy(base, key) +@composite +def _maybe_nil_uuids(draw, uuid): + # Equivalent to `random_uuids | just(...)`, with a stronger bias to the former. + if draw(data()).conjecture_data.draw_bits(6) == 63: + return UUID("00000000-0000-0000-0000-000000000000") + return uuid + + @cacheable @defines_strategy(force_reusable_values=True) -def uuids(*, version: Optional[int] = None) -> SearchStrategy[UUID]: +def uuids( + *, version: Optional[int] = None, allow_nil: bool = False +) -> SearchStrategy[UUID]: """Returns a strategy that generates :class:`UUIDs `. If the optional version argument is given, value is passed through to :class:`~python:uuid.UUID` and only UUIDs of that version will be generated. - All returned values from this will be unique, so e.g. if you do + If ``allow_nil`` is True, generate the nil UUID much more often. + Otherwise, all returned values from this will be unique, so e.g. if you do ``lists(uuids())`` the resulting list will never contain duplicates. Examples from this strategy don't have any meaningful shrink order. """ + check_type(bool, allow_nil, "allow_nil") if version not in (None, 1, 2, 3, 4, 5): raise InvalidArgument( - ( - "version=%r, but version must be in (None, 1, 2, 3, 4, 5) " - "to pass to the uuid.UUID constructor." - ) - % (version,) + f"version={version!r}, but version must be in " + "(None, 1, 2, 3, 4, 5) to pass to the uuid.UUID constructor." ) - return shared( + random_uuids = shared( randoms(use_true_random=True), key="hypothesis.strategies.uuids.generator" ).map(lambda r: UUID(version=version, int=r.getrandbits(128))) + if allow_nil: + if version is not None: + raise InvalidArgument("The nil UUID is not of any version") + return random_uuids.flatmap(_maybe_nil_uuids) + return random_uuids + class RunnerStrategy(SearchStrategy): def __init__(self, default): @@ -2203,12 +1866,13 @@ def example(self): def __not_a_first_class_strategy(self, name): raise InvalidArgument( - "Cannot call %s on a DataStrategy. You should probably be using " - "@composite for whatever it is you're trying to do." % (name,) + f"Cannot call {name} on a DataStrategy. You should probably " + "be using @composite for whatever it is you're trying to do." ) @cacheable +@defines_strategy(never_lazy=True) def data() -> SearchStrategy[DataObject]: """This isn't really a normal strategy, but instead gives you an object which can be used to draw data interactively from other strategies. @@ -2251,7 +1915,12 @@ def register_type_strategy( from hypothesis.strategies._internal import types if not types.is_a_type(custom_type): - raise InvalidArgument("custom_type=%r must be a type") + raise InvalidArgument(f"custom_type={custom_type!r} must be a type") + if custom_type in types.NON_RUNTIME_TYPES: + raise InvalidArgument( + f"custom_type={custom_type!r} is not allowed to be registered, " + f"because there is no such thing as a runtime instance of {custom_type!r}" + ) elif not (isinstance(strategy, SearchStrategy) or callable(strategy)): raise InvalidArgument( "strategy=%r must be a SearchStrategy, or a function that takes " @@ -2267,12 +1936,26 @@ def register_type_strategy( f"for {origin!r} which can inspect specific type objects and return a " "strategy." ) + if ( + "pydantic.generics" in sys.modules + and issubclass(custom_type, sys.modules["pydantic.generics"].GenericModel) + and not re.search(r"[A-Za-z_]+\[.+\]", repr(custom_type)) + and callable(strategy) + ): # pragma: no cover + # See https://github.com/HypothesisWorks/hypothesis/issues/2940 + raise InvalidArgument( + f"Cannot register a function for {custom_type!r}, because parametrized " + "`pydantic.generics.GenericModel` subclasses aren't actually generic " + "types at runtime. In this case, you should register a strategy " + "directly for each parametrized form that you anticipate using." + ) types._global_type_lookup[custom_type] = strategy from_type.__clear_cache() # type: ignore @cacheable +@defines_strategy(never_lazy=True) def deferred(definition: Callable[[], SearchStrategy[Ex]]) -> SearchStrategy[Ex]: """A deferred strategy allows you to write a strategy that references other strategies that have not yet been defined. This allows for the easy @@ -2325,49 +2008,114 @@ def emails() -> SearchStrategy[str]: ) -@defines_strategy() -def functions( - *, - like: Callable[..., Any] = lambda: None, - returns: Optional[SearchStrategy[Any]] = None, - pure: bool = False, -) -> SearchStrategy[Callable[..., Any]]: - # The proper type signature of `functions()` would have T instead of Any, but mypy - # disallows default args for generics: https://github.com/python/mypy/issues/3737 - """functions(*, like=lambda: None, returns=none(), pure=False) - - A strategy for functions, which can be used in callbacks. - - The generated functions will mimic the interface of ``like``, which must - be a callable (including a class, method, or function). The return value - for the function is drawn from the ``returns`` argument, which must be a - strategy. - - If ``pure=True``, all arguments passed to the generated function must be - hashable, and if passed identical arguments the original return value will - be returned again - *not* regenerated, so beware mutable values. - - If ``pure=False``, generated functions do not validate their arguments, and - may return a different value if called again with the same arguments. - - Generated functions can only be called within the scope of the ``@given`` - which created them. This strategy does not support ``.example()``. - """ +def _functions(*, like, returns, pure): + # Wrapped up to use ParamSpec below check_type(bool, pure, "pure") if not callable(like): raise InvalidArgument( "The first argument to functions() must be a callable to imitate, " - "but got non-callable like=%r" % (nicerepr(like),) + f"but got non-callable like={nicerepr(like)!r}" ) - - if returns is None: + if returns in (None, ...): + # Passing `None` has never been *documented* as working, but it still + # did from May 2020 to Jan 2022 so we'll avoid breaking it without cause. hints = get_type_hints(like) returns = from_type(hints.get("return", type(None))) - check_strategy(returns, "returns") return FunctionStrategy(like, returns, pure) +if typing.TYPE_CHECKING or ParamSpec is not None: + + @overload + def functions( + *, pure: bool = ... + ) -> SearchStrategy[Callable[[], None]]: # pragma: no cover + ... + + @overload + def functions( + *, + like: Callable[P, T], + pure: bool = ..., + ) -> SearchStrategy[Callable[P, T]]: # pragma: no cover + ... + + @overload + def functions( + *, + returns: SearchStrategy[T], + pure: bool = ..., + ) -> SearchStrategy[Callable[[], T]]: # pragma: no cover + ... + + @overload + def functions( + *, + like: Callable[P, Any], + returns: SearchStrategy[T], + pure: bool = ..., + ) -> SearchStrategy[Callable[P, T]]: # pragma: no cover + ... + + @defines_strategy() + def functions(*, like=lambda: None, returns=..., pure=False): + # We shouldn't need overloads here, but mypy disallows default args for + # generics: https://github.com/python/mypy/issues/3737 + """functions(*, like=lambda: None, returns=..., pure=False) + + A strategy for functions, which can be used in callbacks. + + The generated functions will mimic the interface of ``like``, which must + be a callable (including a class, method, or function). The return value + for the function is drawn from the ``returns`` argument, which must be a + strategy. If ``returns`` is not passed, we attempt to infer a strategy + from the return-type annotation if present, falling back to :func:`~none`. + + If ``pure=True``, all arguments passed to the generated function must be + hashable, and if passed identical arguments the original return value will + be returned again - *not* regenerated, so beware mutable values. + + If ``pure=False``, generated functions do not validate their arguments, and + may return a different value if called again with the same arguments. + + Generated functions can only be called within the scope of the ``@given`` + which created them. This strategy does not support ``.example()``. + """ + return _functions(like=like, returns=returns, pure=pure) + +else: # pragma: no cover + + @defines_strategy() + def functions( + *, + like: Callable[..., Any] = lambda: None, + returns: Union[SearchStrategy[Any], EllipsisType] = ..., + pure: bool = False, + ) -> SearchStrategy[Callable[..., Any]]: + """functions(*, like=lambda: None, returns=..., pure=False) + + A strategy for functions, which can be used in callbacks. + + The generated functions will mimic the interface of ``like``, which must + be a callable (including a class, method, or function). The return value + for the function is drawn from the ``returns`` argument, which must be a + strategy. If ``returns`` is not passed, we attempt to infer a strategy + from the return-type annotation if present, falling back to :func:`~none`. + + If ``pure=True``, all arguments passed to the generated function must be + hashable, and if passed identical arguments the original return value will + be returned again - *not* regenerated, so beware mutable values. + + If ``pure=False``, generated functions do not validate their arguments, and + may return a different value if called again with the same arguments. + + Generated functions can only be called within the scope of the ``@given`` + which created them. This strategy does not support ``.example()``. + """ + return _functions(like=like, returns=returns, pure=pure) + + @composite def slices(draw: Any, size: int) -> slice: """Generates slices that will select indices up to the supplied size @@ -2382,13 +2130,9 @@ def slices(draw: Any, size: int) -> slice: if size == 0: step = draw(none() | integers().filter(bool)) return slice(None, None, step) - - min_start = min_stop = 0 - max_start = max_stop = size - min_step = 1 # For slices start is inclusive and stop is exclusive - start = draw(integers(min_start, max_start) | none()) - stop = draw(integers(min_stop, max_stop) | none()) + start = draw(integers(0, size - 1) | none()) + stop = draw(integers(0, size) | none()) # Limit step size to be reasonable if start is None and stop is None: @@ -2400,14 +2144,16 @@ def slices(draw: Any, size: int) -> slice: else: max_step = abs(start - stop) - step = draw(integers(min_step, max_step or 1)) + step = draw(integers(1, max_step or 1)) - if (stop or 0) < (start or 0): + if (draw(booleans()) and start == stop) or (stop or 0) < (start or 0): step *= -1 if draw(booleans()) and start is not None: start -= size if draw(booleans()) and stop is not None: stop -= size + if (not draw(booleans())) and step == 1: + step = None return slice(start, stop, step) diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/datetime.py b/hypothesis-python/src/hypothesis/strategies/_internal/datetime.py index a4f65a6408..5ec85ec858 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/datetime.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/datetime.py @@ -1,54 +1,39 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import datetime as dt import os.path from calendar import monthrange from functools import lru_cache +from importlib import resources from typing import Optional from hypothesis.errors import InvalidArgument from hypothesis.internal.conjecture import utils from hypothesis.internal.validation import check_type, check_valid_interval -from hypothesis.strategies._internal.core import ( - defines_strategy, - just, - none, - sampled_from, -) +from hypothesis.strategies._internal.core import sampled_from +from hypothesis.strategies._internal.misc import just, none from hypothesis.strategies._internal.strategies import SearchStrategy +from hypothesis.strategies._internal.utils import defines_strategy -# These standard-library modules are required for the timezones() and -# timezone_keys() strategies, but not present in older versions of Python. -# We therefore try to import them here, but only raise errors recommending -# `pip install hypothesis[zoneinfo]` to install the backports (if needed) -# when those strategies are actually used. -try: - import importlib.resources as importlib_resources -except ImportError: - try: - import importlib_resources # type: ignore - except ImportError: - importlib_resources = None # type: ignore +# The zoneinfo module, required for the timezones() and timezone_keys() +# strategies, is new in Python 3.9 and the backport might be missing. try: import zoneinfo except ImportError: try: from backports import zoneinfo # type: ignore except ImportError: - zoneinfo = None + # We raise an error recommending `pip install hypothesis[zoneinfo]` + # when timezones() or timezone_keys() strategies are actually used. + zoneinfo = None # type: ignore DATENAMES = ("year", "month", "day") TIMENAMES = ("hour", "minute", "second", "microsecond") @@ -100,6 +85,16 @@ def datetime_does_not_exist(value): # meaningless before ~1900 and subject to a lot of change by # 9999, so it should be a very small fraction of possible values. return True + + if ( + value.tzinfo is not roundtrip.tzinfo + and value.utcoffset() != roundtrip.utcoffset() + ): + # This only ever occurs during imaginary (i.e. nonexistent) datetimes, + # and only for pytz timezones which do not follow PEP-495 semantics. + # (may exclude a few other edge cases, but you should use zoneinfo anyway) + return True + assert value.tzinfo is roundtrip.tzinfo, "so only the naive portions are compared" return value != roundtrip @@ -191,17 +186,19 @@ def datetimes( ``timezones`` must be a strategy that generates either ``None``, for naive datetimes, or :class:`~python:datetime.tzinfo` objects for 'aware' datetimes. - You can construct your own, though we recommend using the :pypi:`dateutil - ` package and :func:`hypothesis.extra.dateutil.timezones` - strategy, and also provide :func:`hypothesis.extra.pytz.timezones`. + You can construct your own, though we recommend using one of these built-in + strategies: + + * with Python 3.9 or newer or :pypi:`backports.zoneinfo`: + :func:`hypothesis.strategies.timezones`; + * with :pypi:`dateutil `: + :func:`hypothesis.extra.dateutil.timezones`; or + * with :pypi:`pytz`: :func:`hypothesis.extra.pytz.timezones`. You may pass ``allow_imaginary=False`` to filter out "imaginary" datetimes which did not (or will not) occur due to daylight savings, leap seconds, timezone and calendar adjustments, etc. Imaginary datetimes are allowed by default, because malformed timestamps are a common source of bugs. - Note that because :pypi:`pytz` predates :pep:`495`, this does not work - correctly with timezones that use a negative DST offset (such as - ``"Europe/Dublin"``). Examples from this strategy shrink towards midnight on January 1st 2000, local time. @@ -225,8 +222,8 @@ def datetimes( check_valid_interval(min_value, max_value, "min_value", "max_value") if not isinstance(timezones, SearchStrategy): raise InvalidArgument( - "timezones=%r must be a SearchStrategy that can provide tzinfo " - "for datetimes (either None or dt.tzinfo objects)" % (timezones,) + f"timezones={timezones!r} must be a SearchStrategy that can " + "provide tzinfo for datetimes (either None or dt.tzinfo objects)" ) return DatetimeStrategy(min_value, max_value, timezones, allow_imaginary) @@ -262,9 +259,9 @@ def times( check_type(dt.time, min_value, "min_value") check_type(dt.time, max_value, "max_value") if min_value.tzinfo is not None: - raise InvalidArgument("min_value=%r must not have tzinfo" % min_value) + raise InvalidArgument(f"min_value={min_value!r} must not have tzinfo") if max_value.tzinfo is not None: - raise InvalidArgument("max_value=%r must not have tzinfo" % max_value) + raise InvalidArgument(f"max_value={max_value!r} must not have tzinfo") check_valid_interval(min_value, max_value, "min_value", "max_value") return TimeStrategy(min_value, max_value, timezones) @@ -353,15 +350,15 @@ def _valid_key_cacheable(tzpath, key): # This branch is only taken for names which are known to zoneinfo # but not present on the filesystem, i.e. on Windows with tzdata, # and so is never executed by our coverage tests. - if importlib_resources is None: - raise ImportError( - "The importlib_resources module is required, but could not be " - "imported. Run `pip install hypothesis[zoneinfo]` and try again." - ) *package_loc, resource_name = key.split("/") package = "tzdata.zoneinfo." + ".".join(package_loc) try: - return importlib_resources.is_resource(package, resource_name) + try: + traversable = resources.files(package) / resource_name + return traversable.exists() + except (AttributeError, ValueError): + # .files() was added in Python 3.9 + return resources.is_resource(package, resource_name) except ModuleNotFoundError: return False @@ -392,8 +389,10 @@ def timezone_keys( .. note:: The :mod:`python:zoneinfo` module is new in Python 3.9, so you will need - to install the :pypi:`backports.zoneinfo` module on earlier versions, and - the :pypi:`importlib_resources` backport on Python 3.6. + to install the :pypi:`backports.zoneinfo` module on earlier versions. + + `On Windows, you will also need to install the tzdata package + `__. ``pip install hypothesis[zoneinfo]`` will install these conditional dependencies if and only if they are needed. @@ -401,7 +400,8 @@ def timezone_keys( On Windows, you may need to access IANA timezone data via the :pypi:`tzdata` package. For non-IANA timezones, such as Windows-native names or GNU TZ strings, we recommend using :func:`~hypothesis.strategies.sampled_from` with - the :pypi:`dateutil` package, e.g. :meth:`dateutil:dateutil.tz.tzwin.list`. + the :pypi:`dateutil ` package, e.g. + :meth:`dateutil:dateutil.tz.tzwin.list`. """ # check_type(bool, allow_alias, "allow_alias") # check_type(bool, allow_deprecated, "allow_deprecated") @@ -449,8 +449,10 @@ def timezones(*, no_cache: bool = False) -> SearchStrategy["zoneinfo.ZoneInfo"]: .. note:: The :mod:`python:zoneinfo` module is new in Python 3.9, so you will need - to install the :pypi:`backports.zoneinfo` module on earlier versions, and - the :pypi:`importlib_resources` backport on Python 3.6. + to install the :pypi:`backports.zoneinfo` module on earlier versions. + + `On Windows, you will also need to install the tzdata package + `__. ``pip install hypothesis[zoneinfo]`` will install these conditional dependencies if and only if they are needed. diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/deferred.py b/hypothesis-python/src/hypothesis/strategies/_internal/deferred.py index dd637ee109..489b4d7b7a 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/deferred.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/deferred.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import inspect @@ -24,7 +19,7 @@ class DeferredStrategy(SearchStrategy): """A strategy which may be used before it is fully defined.""" def __init__(self, definition): - SearchStrategy.__init__(self) + super().__init__() self.__wrapped_strategy = None self.__in_repr = False self.__definition = definition diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/featureflags.py b/hypothesis-python/src/hypothesis/strategies/_internal/featureflags.py index 43c9c49cce..5976168212 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/featureflags.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/featureflags.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis.internal.conjecture import utils as cu from hypothesis.strategies._internal.strategies import SearchStrategy diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/flatmapped.py b/hypothesis-python/src/hypothesis/strategies/_internal/flatmapped.py index baaaffa01a..b1fca8f883 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/flatmapped.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/flatmapped.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis.internal.reflection import get_pretty_function_description from hypothesis.strategies._internal.strategies import SearchStrategy, check_strategy diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/functions.py b/hypothesis-python/src/hypothesis/strategies/_internal/functions.py index 5a319e8b8c..9eb6fcb755 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/functions.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/functions.py @@ -1,27 +1,23 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER + +from weakref import WeakKeyDictionary from hypothesis.control import note from hypothesis.errors import InvalidState from hypothesis.internal.reflection import ( - arg_string, convert_positional_arguments, nicerepr, proxies, + repr_call, ) -from hypothesis.strategies._internal.shared import SharedStrategy from hypothesis.strategies._internal.strategies import SearchStrategy @@ -33,6 +29,9 @@ def __init__(self, like, returns, pure): self.like = like self.returns = returns self.pure = pure + # Using wekrefs-to-generated-functions means that the cache can be + # garbage-collected at the end of each example, reducing memory use. + self._cache = WeakKeyDictionary() def calc_is_empty(self, recur): return recur(self.returns) @@ -42,19 +41,22 @@ def do_draw(self, data): def inner(*args, **kwargs): if data.frozen: raise InvalidState( - "This generated %s function can only be called within the " - "scope of the @given that created it." % (nicerepr(self.like),) + f"This generated {nicerepr(self.like)} function can only " + "be called within the scope of the @given that created it." ) if self.pure: args, kwargs = convert_positional_arguments(self.like, args, kwargs) - key = (inner, args, frozenset(kwargs.items())) - val = data.draw(SharedStrategy(base=self.returns, key=key)) + key = (args, frozenset(kwargs.items())) + cache = self._cache.setdefault(inner, {}) + if key not in cache: + cache[key] = data.draw(self.returns) + rep = repr_call(self.like, args, kwargs, reorder=False) + note(f"Called function: {rep} -> {cache[key]!r}") + return cache[key] else: val = data.draw(self.returns) - note( - "Called function: %s(%s) -> %r" - % (nicerepr(self.like), arg_string(self.like, args, kwargs), val) - ) - return val + rep = repr_call(self.like, args, kwargs, reorder=False) + note(f"Called function: {rep} -> {val!r}") + return val return inner diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/ipaddress.py b/hypothesis-python/src/hypothesis/strategies/_internal/ipaddress.py index 2d3bbf83f8..717943f29e 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/ipaddress.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/ipaddress.py @@ -1,30 +1,22 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from ipaddress import IPv4Address, IPv4Network, IPv6Address, IPv6Network, ip_network from typing import Optional, Union from hypothesis.errors import InvalidArgument from hypothesis.internal.validation import check_type -from hypothesis.strategies._internal.core import ( - SearchStrategy, - binary, - defines_strategy, - integers, - sampled_from, -) +from hypothesis.strategies._internal.core import binary, sampled_from +from hypothesis.strategies._internal.numbers import integers +from hypothesis.strategies._internal.strategies import SearchStrategy +from hypothesis.strategies._internal.utils import defines_strategy # See https://www.iana.org/assignments/iana-ipv4-special-registry/ SPECIAL_IPv4_RANGES = ( @@ -100,7 +92,7 @@ def ip_addresses( """ if v is not None: check_type(int, v, "v") - if v != 4 and v != 6: + if v not in (4, 6): raise InvalidArgument(f"v={v!r}, but only v=4 or v=6 are valid") if network is None: # We use the reserved-address registries to boost the chance diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/lazy.py b/hypothesis-python/src/hypothesis/strategies/_internal/lazy.py index d828df88ce..18292c044c 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/lazy.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/lazy.py @@ -1,29 +1,26 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER -from inspect import getfullargspec -from typing import Dict +from inspect import signature +from typing import MutableMapping +from weakref import WeakKeyDictionary from hypothesis.internal.reflection import ( - arg_string, convert_keyword_arguments, convert_positional_arguments, + get_pretty_function_description, + repr_call, ) from hypothesis.strategies._internal.strategies import SearchStrategy -unwrap_cache = {} # type: Dict[SearchStrategy, SearchStrategy] +unwrap_cache: MutableMapping[SearchStrategy, SearchStrategy] = WeakKeyDictionary() unwrap_depth = 0 @@ -63,6 +60,10 @@ def unwrap_strategies(s): assert unwrap_depth >= 0 +def _repr_filter(condition): + return f".filter({get_pretty_function_description(condition)})" + + class LazyStrategy(SearchStrategy): """A strategy which is defined purely by conversion to and from another strategy. @@ -70,13 +71,14 @@ class LazyStrategy(SearchStrategy): Its parameter and distribution come from that other strategy. """ - def __init__(self, function, args, kwargs, *, force_repr=None): - SearchStrategy.__init__(self) + def __init__(self, function, args, kwargs, filters=(), *, force_repr=None): + super().__init__() self.__wrapped_strategy = None self.__representation = force_repr self.function = function self.__args = args self.__kwargs = kwargs + self.__filters = filters @property def supports_find(self): @@ -110,8 +112,19 @@ def wrapped_strategy(self): self.__wrapped_strategy = self.function( *unwrapped_args, **unwrapped_kwargs ) + for f in self.__filters: + self.__wrapped_strategy = self.__wrapped_strategy.filter(f) return self.__wrapped_strategy + def filter(self, condition): + return LazyStrategy( + self.function, + self.__args, + self.__kwargs, + self.__filters + (condition,), + force_repr=f"{self!r}{_repr_filter(condition)}", + ) + def do_validate(self): w = self.wrapped_strategy assert isinstance(w, SearchStrategy), f"{self!r} returned non-strategy {w!r}" @@ -119,41 +132,29 @@ def do_validate(self): def __repr__(self): if self.__representation is None: - _args = self.__args - _kwargs = self.__kwargs - argspec = getfullargspec(self.function) - defaults = dict(argspec.kwonlydefaults or {}) - if argspec.defaults is not None: - for name, value in zip( - reversed(argspec.args), reversed(argspec.defaults) - ): - defaults[name] = value - if len(argspec.args) > 1 or argspec.defaults: + sig = signature(self.function) + pos = [p for p in sig.parameters.values() if "POSITIONAL" in p.kind.name] + if len(pos) > 1 or any(p.default is not sig.empty for p in pos): _args, _kwargs = convert_positional_arguments( - self.function, _args, _kwargs + self.function, self.__args, self.__kwargs ) else: _args, _kwargs = convert_keyword_arguments( - self.function, _args, _kwargs + self.function, self.__args, self.__kwargs ) - kwargs_for_repr = dict(_kwargs) - for k, v in defaults.items(): - if k in kwargs_for_repr and kwargs_for_repr[k] is v: - del kwargs_for_repr[k] - self.__representation = "{}({})".format( - self.function.__name__, - arg_string(self.function, _args, kwargs_for_repr, reorder=False), - ) + kwargs_for_repr = { + k: v + for k, v in _kwargs.items() + if k not in sig.parameters or v is not sig.parameters[k].default + } + self.__representation = repr_call( + self.function, _args, kwargs_for_repr, reorder=False + ) + "".join(map(_repr_filter, self.__filters)) return self.__representation def do_draw(self, data): return data.draw(self.wrapped_strategy) - def do_filtered_draw(self, data, filter_strategy): - return self.wrapped_strategy.do_filtered_draw( - data=data, filter_strategy=filter_strategy - ) - @property def label(self): return self.wrapped_strategy.label diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/misc.py b/hypothesis-python/src/hypothesis/strategies/_internal/misc.py index c39824260a..ad37107f73 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/misc.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/misc.py @@ -1,24 +1,21 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER +from hypothesis.internal.reflection import get_pretty_function_description from hypothesis.strategies._internal.strategies import ( - FilteredStrategy, SampledFromStrategy, - filter_not_satisfied, + SearchStrategy, + T, is_simple_data, ) +from hypothesis.strategies._internal.utils import cacheable, defines_strategy class JustStrategy(SampledFromStrategy): @@ -39,24 +36,83 @@ def value(self): return self.elements[0] def __repr__(self): + suffix = "".join( + f".{name}({get_pretty_function_description(f)})" + for name, f in self._transformations + ) if self.value is None: - return "none()" - return f"just({self.value!r})" - - def calc_has_reusable_values(self, recur): - return True + return "none()" + suffix + return f"just({get_pretty_function_description(self.value)}){suffix}" def calc_is_cacheable(self, recur): return is_simple_data(self.value) - def do_draw(self, data): - result = self._transform(self.value) - if result is filter_not_satisfied: - data.note_event(f"Aborted test because unable to satisfy {self!r}") - data.mark_invalid() - return result - - def do_filtered_draw(self, data, filter_strategy): - if isinstance(filter_strategy, FilteredStrategy): - return self._transform(self.value, filter_strategy.flat_conditions) + def do_filtered_draw(self, data): + # The parent class's `do_draw` implementation delegates directly to + # `do_filtered_draw`, which we can greatly simplify in this case since + # we have exactly one value. (This also avoids drawing any data.) return self._transform(self.value) + + +@defines_strategy(never_lazy=True) +def just(value: T) -> SearchStrategy[T]: + """Return a strategy which only generates ``value``. + + Note: ``value`` is not copied. Be wary of using mutable values. + + If ``value`` is the result of a callable, you can use + :func:`builds(callable) ` instead + of ``just(callable())`` to get a fresh value each time. + + Examples from this strategy do not shrink (because there is only one). + """ + return JustStrategy([value]) + + +@defines_strategy(force_reusable_values=True) +def none() -> SearchStrategy[None]: + """Return a strategy which only generates None. + + Examples from this strategy do not shrink (because there is only + one). + """ + return just(None) + + +class Nothing(SearchStrategy): + def calc_is_empty(self, recur): + return True + + def do_draw(self, data): + # This method should never be called because draw() will mark the + # data as invalid immediately because is_empty is True. + raise NotImplementedError("This should never happen") + + def calc_has_reusable_values(self, recur): + return True + + def __repr__(self): + return "nothing()" + + def map(self, f): + return self + + def filter(self, f): + return self + + def flatmap(self, f): + return self + + +NOTHING = Nothing() + + +@cacheable +@defines_strategy(never_lazy=True) +def nothing() -> SearchStrategy: + """This strategy never successfully draws a value and will always reject on + an attempt to draw. + + Examples from this strategy do not shrink (because there are none). + """ + return NOTHING diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/numbers.py b/hypothesis-python/src/hypothesis/strategies/_internal/numbers.py index fe9928af87..380c8732cf 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/numbers.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/numbers.py @@ -1,61 +1,174 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import math +from decimal import Decimal +from fractions import Fraction +from sys import float_info +from typing import Optional, Union -from hypothesis.control import assume, reject +from hypothesis.control import reject +from hypothesis.errors import InvalidArgument from hypothesis.internal.conjecture import floats as flt, utils as d from hypothesis.internal.conjecture.utils import calc_label_from_name -from hypothesis.internal.floats import float_of -from hypothesis.strategies._internal.strategies import SearchStrategy - +from hypothesis.internal.filtering import ( + get_float_predicate_bounds, + get_integer_predicate_bounds, +) +from hypothesis.internal.floats import ( + float_of, + float_to_int, + int_to_float, + is_negative, + make_float_clamper, + next_down, + next_down_normal, + next_up, + next_up_normal, + width_smallest_normals, +) +from hypothesis.internal.validation import ( + check_type, + check_valid_bound, + check_valid_interval, +) +from hypothesis.strategies._internal.misc import nothing +from hypothesis.strategies._internal.strategies import ( + SampledFromStrategy, + SearchStrategy, +) +from hypothesis.strategies._internal.utils import cacheable, defines_strategy -class WideRangeIntStrategy(SearchStrategy): +# See https://github.com/python/mypy/issues/3186 - numbers.Real is wrong! +Real = Union[int, float, Fraction, Decimal] +ONE_BOUND_INTEGERS_LABEL = d.calc_label_from_name("trying a one-bound int allowing 0") - distribution = d.Sampler([4.0, 8.0, 1.0, 1.0, 0.5]) - sizes = [8, 16, 32, 64, 128] +class IntegersStrategy(SearchStrategy): + def __init__(self, start, end): + assert isinstance(start, int) or start is None + assert isinstance(end, int) or end is None + assert start is None or end is None or start <= end + self.start = start + self.end = end def __repr__(self): - return "WideRangeIntStrategy()" + if self.start is None and self.end is None: + return "integers()" + if self.end is None: + return f"integers(min_value={self.start})" + if self.start is None: + return f"integers(max_value={self.end})" + return f"integers({self.start}, {self.end})" def do_draw(self, data): - size = self.sizes[self.distribution.sample(data)] - r = data.draw_bits(size) - sign = r & 1 - r >>= 1 - if sign: - r = -r - return int(r) + if self.start is None and self.end is None: + return d.unbounded_integers(data) + if self.start is None: + if self.end <= 0: + return self.end - abs(d.unbounded_integers(data)) + else: + probe = self.end + 1 + while self.end < probe: + data.start_example(ONE_BOUND_INTEGERS_LABEL) + probe = d.unbounded_integers(data) + data.stop_example(discard=self.end < probe) + return probe -class BoundedIntStrategy(SearchStrategy): - """A strategy for providing integers in some interval with inclusive - endpoints.""" + if self.end is None: + if self.start >= 0: + return self.start + abs(d.unbounded_integers(data)) + else: + probe = self.start - 1 + while probe < self.start: + data.start_example(ONE_BOUND_INTEGERS_LABEL) + probe = d.unbounded_integers(data) + data.stop_example(discard=probe < self.start) + return probe - def __init__(self, start, end): - SearchStrategy.__init__(self) - self.start = start - self.end = end + # For bounded integers, make the bounds and near-bounds more likely. + forced = None + if self.end - self.start > 127: + forced = { + 122: self.start, + 123: self.start, + 124: self.end, + 125: self.end, + 126: self.start + 1, + 127: self.end - 1, + }.get(data.draw_bits(7)) - def __repr__(self): - return f"BoundedIntStrategy({self.start}, {self.end})" + return d.integer_range(data, self.start, self.end, center=0, forced=forced) + + def filter(self, condition): + if condition is math.isfinite: + return self + if condition in [math.isinf, math.isnan]: + return nothing() + kwargs, pred = get_integer_predicate_bounds(condition) + + start, end = self.start, self.end + if "min_value" in kwargs: + start = max(kwargs["min_value"], -math.inf if start is None else start) + if "max_value" in kwargs: + end = min(kwargs["max_value"], math.inf if end is None else end) + + if start != self.start or end != self.end: + if start is not None and end is not None and start > end: + return nothing() + self = type(self)(start, end) + if pred is None: + return self + return super().filter(pred) - def do_draw(self, data): - return d.integer_range(data, self.start, self.end) +@cacheable +@defines_strategy(force_reusable_values=True) +def integers( + min_value: Optional[int] = None, + max_value: Optional[int] = None, +) -> SearchStrategy[int]: + """Returns a strategy which generates integers. + + If min_value is not None then all values will be >= min_value. If + max_value is not None then all values will be <= max_value + + Examples from this strategy will shrink towards zero, and negative values + will also shrink towards positive (i.e. -n may be replaced by +n). + """ + check_valid_bound(min_value, "min_value") + check_valid_bound(max_value, "max_value") + check_valid_interval(min_value, max_value, "min_value", "max_value") + + if min_value is not None: + if min_value != int(min_value): + raise InvalidArgument( + "min_value=%r of type %r cannot be exactly represented as an integer." + % (min_value, type(min_value)) + ) + min_value = int(min_value) + if max_value is not None: + if max_value != int(max_value): + raise InvalidArgument( + "max_value=%r of type %r cannot be exactly represented as an integer." + % (max_value, type(max_value)) + ) + max_value = int(max_value) + + return IntegersStrategy(min_value, max_value) + + +SMALLEST_SUBNORMAL = next_up(0.0) +SIGNALING_NAN = int_to_float(0x7FF8_0000_0000_0001) # nonzero mantissa +assert math.isnan(SIGNALING_NAN) and math.copysign(1, SIGNALING_NAN) == 1 NASTY_FLOATS = sorted( [ @@ -68,8 +181,9 @@ def do_draw(self, data): 10e6, 10e-6, 1.175494351e-38, - 2.2250738585072014e-308, - 1.7976931348623157e308, + next_up(0.0), + float_info.min, + float_info.max, 3.402823466e38, 9007199254740992, 1 - 10e-6, @@ -77,7 +191,10 @@ def do_draw(self, data): 1.192092896e-07, 2.2204460492503131e-016, ] - + [math.inf, math.nan] * 5, + + [2.0**-n for n in (24, 14, 149, 126)] # minimum (sub)normals for float16,32 + + [float_info.min / n for n in (2, 10, 1000, 100_000)] # subnormal in float64 + + [math.inf, math.nan] * 5 + + [SIGNALING_NAN], key=flt.float_to_lex, ) NASTY_FLOATS = list(map(float, NASTY_FLOATS)) @@ -88,91 +205,437 @@ def do_draw(self, data): ) +def _sign_aware_lte(x: float, y: float) -> bool: + """Less-than-or-equals, but strictly orders -0.0 and 0.0""" + if x == 0.0 == y: + return math.copysign(1.0, x) <= math.copysign(1.0, y) + else: + return x <= y + + class FloatStrategy(SearchStrategy): - """Generic superclass for strategies which produce floats.""" + """A strategy for floating point numbers.""" - def __init__(self, allow_infinity, allow_nan, width): - SearchStrategy.__init__(self) - assert isinstance(allow_infinity, bool) + def __init__( + self, + *, + min_value: float, + max_value: float, + allow_nan: bool, + # The smallest nonzero number we can represent is usually a subnormal, but may + # be the smallest normal if we're running in unsafe denormals-are-zero mode. + # While that's usually an explicit error, we do need to handle the case where + # the user passes allow_subnormal=False. + smallest_nonzero_magnitude: float = SMALLEST_SUBNORMAL, + ): + super().__init__() assert isinstance(allow_nan, bool) - assert width in (16, 32, 64) - self.allow_infinity = allow_infinity + assert smallest_nonzero_magnitude >= 0.0, "programmer error if this is negative" + if smallest_nonzero_magnitude == 0.0: # pragma: no cover + raise FloatingPointError( + "Got allow_subnormal=True, but we can't represent subnormal floats " + "right now, in violation of the IEEE-754 floating-point " + "specification. This is usually because something was compiled with " + "-ffast-math or a similar option, which sets global processor state. " + "See https://simonbyrne.github.io/notes/fastmath/ for a more detailed " + "writeup - and good luck!" + ) + self.min_value = min_value + self.max_value = max_value self.allow_nan = allow_nan - self.width = width + self.smallest_nonzero_magnitude = smallest_nonzero_magnitude + boundary_values = [ + min_value, + next_up(min_value), + min_value + 1, + max_value - 1, + next_down(max_value), + max_value, + ] self.nasty_floats = [ - float_of(f, self.width) for f in NASTY_FLOATS if self.permitted(f) + f for f in NASTY_FLOATS + boundary_values if self.permitted(f) ] weights = [0.2 * len(self.nasty_floats)] + [0.8] * len(self.nasty_floats) - self.sampler = d.Sampler(weights) + self.sampler = d.Sampler(weights) if self.nasty_floats else None + + self.pos_clamper = self.neg_clamper = None + if _sign_aware_lte(0.0, max_value): + pos_min = max(min_value, smallest_nonzero_magnitude) + allow_zero = _sign_aware_lte(min_value, 0.0) + self.pos_clamper = make_float_clamper(pos_min, max_value, allow_zero) + if _sign_aware_lte(min_value, -0.0): + neg_max = min(max_value, -smallest_nonzero_magnitude) + allow_zero = _sign_aware_lte(-0.0, max_value) + self.neg_clamper = make_float_clamper(-neg_max, -min_value, allow_zero) + + self.forced_sign_bit: Optional[int] = None + if (self.pos_clamper is None) != (self.neg_clamper is None): + self.forced_sign_bit = 1 if self.neg_clamper else 0 def __repr__(self): - return "{}(allow_infinity={}, allow_nan={}, width={})".format( - self.__class__.__name__, self.allow_infinity, self.allow_nan, self.width + return "{}(min_value={}, max_value={}, allow_nan={}, smallest_nonzero_magnitude={})".format( + self.__class__.__name__, + self.min_value, + self.max_value, + self.allow_nan, + self.smallest_nonzero_magnitude, ) def permitted(self, f): assert isinstance(f, float) - if not self.allow_infinity and math.isinf(f): + if math.isnan(f): + return self.allow_nan + if 0 < abs(f) < self.smallest_nonzero_magnitude: return False - if not self.allow_nan and math.isnan(f): - return False - if self.width < 64: - try: - float_of(f, self.width) - return True - except OverflowError: # pragma: no cover - return False - return True + return _sign_aware_lte(self.min_value, f) and _sign_aware_lte(f, self.max_value) def do_draw(self, data): while True: data.start_example(FLOAT_STRATEGY_DO_DRAW_LABEL) - i = self.sampler.sample(data) + i = self.sampler.sample(data) if self.sampler else 0 + data.start_example(flt.DRAW_FLOAT_LABEL) if i == 0: - result = flt.draw_float(data) + result = flt.draw_float(data, forced_sign_bit=self.forced_sign_bit) + is_negative = flt.float_to_int(result) >> 63 + if is_negative: + clamped = -self.neg_clamper(-result) + else: + clamped = self.pos_clamper(result) + if clamped != result: + data.stop_example(discard=True) + data.start_example(flt.DRAW_FLOAT_LABEL) + flt.write_float(data, clamped) + result = clamped else: result = self.nasty_floats[i - 1] + flt.write_float(data, result) - if self.permitted(result): - data.stop_example() - if self.width < 64: - return float_of(result, self.width) - return result - data.stop_example(discard=True) + data.stop_example() # (DRAW_FLOAT_LABEL) + data.stop_example() # (FLOAT_STRATEGY_DO_DRAW_LABEL) + return result + + def filter(self, condition): + # Handle a few specific weird cases. + if condition is math.isfinite: + return FloatStrategy( + min_value=max(self.min_value, next_up(float("-inf"))), + max_value=min(self.max_value, next_down(float("inf"))), + allow_nan=False, + smallest_nonzero_magnitude=self.smallest_nonzero_magnitude, + ) + if condition is math.isinf: + permitted_infs = [x for x in (-math.inf, math.inf) if self.permitted(x)] + if not permitted_infs: + return nothing() + return SampledFromStrategy(permitted_infs) + if condition is math.isnan: + if not self.allow_nan: + return nothing() + return NanStrategy() + + kwargs, pred = get_float_predicate_bounds(condition) + if not kwargs: + return super().filter(pred) + min_bound = max(kwargs.get("min_value", -math.inf), self.min_value) + max_bound = min(kwargs.get("max_value", math.inf), self.max_value) -class FixedBoundedFloatStrategy(SearchStrategy): - """A strategy for floats distributed between two endpoints. + # Adjustments for allow_subnormal=False, if any need to be made + if -self.smallest_nonzero_magnitude < min_bound < 0: + min_bound = -0.0 + elif 0 < min_bound < self.smallest_nonzero_magnitude: + min_bound = self.smallest_nonzero_magnitude + if -self.smallest_nonzero_magnitude < max_bound < 0: + max_bound = -self.smallest_nonzero_magnitude + elif 0 < max_bound < self.smallest_nonzero_magnitude: + max_bound = 0.0 - The conditional distribution tries to produce values clustered - closer to one of the ends. + if min_bound > max_bound: + return nothing() + if ( + min_bound > self.min_value + or self.max_value > max_bound + or (self.allow_nan and (-math.inf < min_bound or max_bound < math.inf)) + ): + self = type(self)( + min_value=min_bound, + max_value=max_bound, + allow_nan=False, + smallest_nonzero_magnitude=self.smallest_nonzero_magnitude, + ) + if pred is None: + return self + return super().filter(pred) + + +@cacheable +@defines_strategy(force_reusable_values=True) +def floats( + min_value: Optional[Real] = None, + max_value: Optional[Real] = None, + *, + allow_nan: Optional[bool] = None, + allow_infinity: Optional[bool] = None, + allow_subnormal: Optional[bool] = None, + width: int = 64, + exclude_min: bool = False, + exclude_max: bool = False, +) -> SearchStrategy[float]: + """Returns a strategy which generates floats. + + - If min_value is not None, all values will be ``>= min_value`` + (or ``> min_value`` if ``exclude_min``). + - If max_value is not None, all values will be ``<= max_value`` + (or ``< max_value`` if ``exclude_max``). + - If min_value or max_value is not None, it is an error to enable + allow_nan. + - If both min_value and max_value are not None, it is an error to enable + allow_infinity. + - If inferred values range does not include subnormal values, it is an error + to enable allow_subnormal. + + Where not explicitly ruled out by the bounds, + :wikipedia:`subnormals `, infinities, and NaNs are possible + values generated by this strategy. + + The width argument specifies the maximum number of bits of precision + required to represent the generated float. Valid values are 16, 32, or 64. + Passing ``width=32`` will still use the builtin 64-bit ``float`` class, + but always for values which can be exactly represented as a 32-bit float. + + The exclude_min and exclude_max argument can be used to generate numbers + from open or half-open intervals, by excluding the respective endpoints. + Excluding either signed zero will also exclude the other. + Attempting to exclude an endpoint which is None will raise an error; + use ``allow_infinity=False`` to generate finite floats. You can however + use e.g. ``min_value=-math.inf, exclude_min=True`` to exclude only + one infinite endpoint. + + Examples from this strategy have a complicated and hard to explain + shrinking behaviour, but it tries to improve "human readability". Finite + numbers will be preferred to infinity and infinity will be preferred to + NaN. """ + check_type(bool, exclude_min, "exclude_min") + check_type(bool, exclude_max, "exclude_max") - def __init__(self, lower_bound, upper_bound, width): - SearchStrategy.__init__(self) - assert isinstance(lower_bound, float) - assert isinstance(upper_bound, float) - assert 0 <= lower_bound < upper_bound - assert math.copysign(1, lower_bound) == 1, "lower bound may not be -0.0" - assert width in (16, 32, 64) - self.lower_bound = lower_bound - self.upper_bound = upper_bound - self.width = width + if allow_nan is None: + allow_nan = bool(min_value is None and max_value is None) + elif allow_nan and (min_value is not None or max_value is not None): + raise InvalidArgument( + f"Cannot have allow_nan={allow_nan!r}, with min_value or max_value" + ) - def __repr__(self): - return "FixedBoundedFloatStrategy({}, {}, {})".format( - self.lower_bound, self.upper_bound, self.width + if width not in (16, 32, 64): + raise InvalidArgument( + f"Got width={width!r}, but the only valid values " + "are the integers 16, 32, and 64." ) - def do_draw(self, data): - f = self.lower_bound + ( - self.upper_bound - self.lower_bound - ) * d.fractional_float(data) - if self.width < 64: + check_valid_bound(min_value, "min_value") + check_valid_bound(max_value, "max_value") + + if math.copysign(1.0, -0.0) == 1.0: # pragma: no cover + raise FloatingPointError( + "You Python install can't represent -0.0, which is required by the " + "IEEE-754 floating-point specification. This is probably because it was " + "compiled with an unsafe option like -ffast-math; for a more detailed " + "explanation see https://simonbyrne.github.io/notes/fastmath/" + ) + if allow_subnormal and next_up(0.0, width=width) == 0: # pragma: no cover + # Not worth having separate CI envs and dependencies just to cover this branch; + # discussion in https://github.com/HypothesisWorks/hypothesis/issues/3092 + # + # Erroring out here ensures that the database contents are interpreted + # consistently - which matters for such a foundational strategy, even if it's + # not always true for all user-composed strategies further up the stack. + from _hypothesis_ftz_detector import identify_ftz_culprits + + try: + ftz_pkg = identify_ftz_culprits() + except Exception: + ftz_pkg = None + if ftz_pkg: + ftz_msg = ( + f"This seems to be because the `{ftz_pkg}` package was compiled with " + f"-ffast-math or a similar option, which sets global processor state " + f"- see https://simonbyrne.github.io/notes/fastmath/ for details. " + f"If you don't know why {ftz_pkg} is installed, `pipdeptree -rp " + f"{ftz_pkg}` will show which packages depend on it." + ) + else: + ftz_msg = ( + "This is usually because something was compiled with -ffast-math " + "or a similar option, which sets global processor state. See " + "https://simonbyrne.github.io/notes/fastmath/ for a more detailed " + "writeup - and good luck!" + ) + raise FloatingPointError( + f"Got allow_subnormal={allow_subnormal!r}, but we can't represent " + f"subnormal floats right now, in violation of the IEEE-754 floating-point " + f"specification. {ftz_msg}" + ) + + min_arg, max_arg = min_value, max_value + if min_value is not None: + min_value = float_of(min_value, width) + assert isinstance(min_value, float) + if max_value is not None: + max_value = float_of(max_value, width) + assert isinstance(max_value, float) + + if min_value != min_arg: + raise InvalidArgument( + f"min_value={min_arg!r} cannot be exactly represented as a float " + f"of width {width} - use min_value={min_value!r} instead." + ) + if max_value != max_arg: + raise InvalidArgument( + f"max_value={max_arg!r} cannot be exactly represented as a float " + f"of width {width} - use max_value={max_value!r} instead." + ) + + if exclude_min and (min_value is None or min_value == math.inf): + raise InvalidArgument(f"Cannot exclude min_value={min_value!r}") + if exclude_max and (max_value is None or max_value == -math.inf): + raise InvalidArgument(f"Cannot exclude max_value={max_value!r}") + + assumed_allow_subnormal = allow_subnormal is None or allow_subnormal + if min_value is not None and ( + exclude_min or (min_arg is not None and min_value < min_arg) + ): + min_value = next_up_normal(min_value, width, assumed_allow_subnormal) + if min_value == min_arg: + assert min_value == min_arg == 0 + assert is_negative(min_arg) and not is_negative(min_value) + min_value = next_up_normal(min_value, width, assumed_allow_subnormal) + assert min_value > min_arg # type: ignore + if max_value is not None and ( + exclude_max or (max_arg is not None and max_value > max_arg) + ): + max_value = next_down_normal(max_value, width, assumed_allow_subnormal) + if max_value == max_arg: + assert max_value == max_arg == 0 + assert is_negative(max_value) and not is_negative(max_arg) + max_value = next_down_normal(max_value, width, assumed_allow_subnormal) + assert max_value < max_arg # type: ignore + + if min_value == -math.inf: + min_value = None + if max_value == math.inf: + max_value = None + + bad_zero_bounds = ( + min_value == max_value == 0 + and is_negative(max_value) + and not is_negative(min_value) + ) + if ( + min_value is not None + and max_value is not None + and (min_value > max_value or bad_zero_bounds) + ): + # This is a custom alternative to check_valid_interval, because we want + # to include the bit-width and exclusion information in the message. + msg = ( + "There are no %s-bit floating-point values between min_value=%r " + "and max_value=%r" % (width, min_arg, max_arg) + ) + if exclude_min or exclude_max: + msg += f", exclude_min={exclude_min!r} and exclude_max={exclude_max!r}" + raise InvalidArgument(msg) + + if allow_infinity is None: + allow_infinity = bool(min_value is None or max_value is None) + elif allow_infinity: + if min_value is not None and max_value is not None: + raise InvalidArgument( + f"Cannot have allow_infinity={allow_infinity!r}, " + "with both min_value and max_value" + ) + elif min_value == math.inf: + if min_arg == math.inf: + raise InvalidArgument("allow_infinity=False excludes min_value=inf") + raise InvalidArgument( + f"exclude_min=True turns min_value={min_arg!r} into inf, " + "but allow_infinity=False" + ) + elif max_value == -math.inf: + if max_arg == -math.inf: + raise InvalidArgument("allow_infinity=False excludes max_value=-inf") + raise InvalidArgument( + f"exclude_max=True turns max_value={max_arg!r} into -inf, " + "but allow_infinity=False" + ) + + smallest_normal = width_smallest_normals[width] + if allow_subnormal is None: + if min_value is not None and max_value is not None: + if min_value == max_value: + allow_subnormal = -smallest_normal < min_value < smallest_normal + else: + allow_subnormal = ( + min_value < smallest_normal and max_value > -smallest_normal + ) + elif min_value is not None: + allow_subnormal = min_value < smallest_normal + elif max_value is not None: + allow_subnormal = max_value > -smallest_normal + else: + allow_subnormal = True + if allow_subnormal: + if min_value is not None and min_value >= smallest_normal: + raise InvalidArgument( + f"allow_subnormal=True, but minimum value {min_value} " + f"excludes values below float{width}'s " + f"smallest positive normal {smallest_normal}" + ) + if max_value is not None and max_value <= -smallest_normal: + raise InvalidArgument( + f"allow_subnormal=True, but maximum value {max_value} " + f"excludes values above float{width}'s " + f"smallest negative normal {-smallest_normal}" + ) + + if min_value is None: + min_value = float("-inf") + if max_value is None: + max_value = float("inf") + if not allow_infinity: + min_value = max(min_value, next_up(float("-inf"))) + max_value = min(max_value, next_down(float("inf"))) + assert isinstance(min_value, float) + assert isinstance(max_value, float) + smallest_nonzero_magnitude = ( + SMALLEST_SUBNORMAL if allow_subnormal else smallest_normal + ) + result: SearchStrategy = FloatStrategy( + min_value=min_value, + max_value=max_value, + allow_nan=allow_nan, + smallest_nonzero_magnitude=smallest_nonzero_magnitude, + ) + + if width < 64: + + def downcast(x): try: - f = float_of(f, self.width) + return float_of(x, width) except OverflowError: # pragma: no cover reject() - assume(self.lower_bound <= f <= self.upper_bound) - return f + + result = result.map(downcast) + return result + + +class NanStrategy(SearchStrategy): + """Strategy for sampling the space of nan float values.""" + + def do_draw(self, data): + # Nans must have all exponent bits and the first mantissa bit set, so + # we generate by taking 64 random bits and setting the required ones. + sign_bit = data.draw_bits(1) << 63 + nan_bits = float_to_int(math.nan) + mantissa_bits = data.draw_bits(52) + return int_to_float(sign_bit | nan_bits | mantissa_bits) diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/random.py b/hypothesis-python/src/hypothesis/strategies/_internal/random.py index 266ec28f55..2c8997e523 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/random.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/random.py @@ -1,21 +1,17 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import inspect import math from random import Random +from typing import Dict import attr @@ -23,7 +19,13 @@ from hypothesis.internal.conjecture import utils as cu from hypothesis.internal.reflection import define_function_signature from hypothesis.reporting import report -from hypothesis.strategies._internal import core as st +from hypothesis.strategies._internal.core import ( + binary, + lists, + permutations, + sampled_from, +) +from hypothesis.strategies._internal.numbers import floats, integers from hypothesis.strategies._internal.strategies import SearchStrategy @@ -93,22 +95,22 @@ def _hypothesis_do_random(self, method, kwargs): # Fake shims to get a good signature -def getrandbits(self, n: int) -> int: +def getrandbits(self, n: int) -> int: # type: ignore raise NotImplementedError() -def random(self) -> float: +def random(self) -> float: # type: ignore raise NotImplementedError() -def _randbelow(self, n: int) -> int: +def _randbelow(self, n: int) -> int: # type: ignore raise NotImplementedError() STUBS = {f.__name__: f for f in [getrandbits, random, _randbelow]} -SIGNATURES = {} +SIGNATURES: Dict[str, inspect.Signature] = {} def sig_of(name): @@ -131,9 +133,9 @@ def implementation(self, **kwargs): self._hypothesis_log_random(name, kwargs, result) return result - spec = inspect.getfullargspec(STUBS.get(name, target)) + sig = inspect.signature(STUBS.get(name, target)) - result = define_function_signature(target.__name__, target.__doc__, spec)( + result = define_function_signature(target.__name__, target.__doc__, sig)( implementation ) @@ -169,10 +171,10 @@ def state_for_seed(data, seed): return state -UNIFORM = st.floats(0, 1) +UNIFORM = floats(0, 1) -def normalize_zero(f): +def normalize_zero(f: float) -> float: if f == 0.0: return 0.0 else: @@ -180,10 +182,10 @@ def normalize_zero(f): class ArtificialRandom(HypothesisRandom): - VERSION = 10 ** 6 + VERSION = 10**6 def __init__(self, note_method_calls, data): - HypothesisRandom.__init__(self, note_method_calls=note_method_calls) + super().__init__(note_method_calls=note_method_calls) self.__data = data self.__state = RandomState() @@ -236,16 +238,16 @@ def _hypothesis_do_random(self, method, kwargs): elif method == "uniform": a = normalize_zero(kwargs["a"]) b = normalize_zero(kwargs["b"]) - result = self.__data.draw(st.floats(a, b)) + result = self.__data.draw(floats(a, b)) elif method in ("weibullvariate", "gammavariate"): - result = self.__data.draw(st.floats(min_value=0.0, allow_infinity=False)) + result = self.__data.draw(floats(min_value=0.0, allow_infinity=False)) elif method in ("gauss", "normalvariate"): mu = kwargs["mu"] result = mu + self.__data.draw( - st.floats(allow_nan=False, allow_infinity=False) + floats(allow_nan=False, allow_infinity=False) ) elif method == "vonmisesvariate": - result = self.__data.draw(st.floats(0, 2 * math.pi)) + result = self.__data.draw(floats(0, 2 * math.pi)) elif method == "randrange": if kwargs["stop"] is None: stop = kwargs["start"] @@ -275,8 +277,8 @@ def _hypothesis_do_random(self, method, kwargs): elif method == "choices": k = kwargs["k"] result = self.__data.draw( - st.lists( - st.integers(0, len(kwargs["population"]) - 1), + lists( + integers(0, len(kwargs["population"]) - 1), min_size=k, max_size=k, ) @@ -291,8 +293,8 @@ def _hypothesis_do_random(self, method, kwargs): ) result = self.__data.draw( - st.lists( - st.sampled_from(range(len(seq))), + lists( + sampled_from(range(len(seq))), min_size=k, max_size=k, unique=True, @@ -306,19 +308,19 @@ def _hypothesis_do_random(self, method, kwargs): high = normalize_zero(kwargs["high"]) mode = normalize_zero(kwargs["mode"]) if mode is None: - result = self.__data.draw(st.floats(low, high)) + result = self.__data.draw(floats(low, high)) elif self.__data.draw_bits(1): - result = self.__data.draw(st.floats(mode, high)) + result = self.__data.draw(floats(mode, high)) else: - result = self.__data.draw(st.floats(low, mode)) + result = self.__data.draw(floats(low, mode)) elif method in ("paretovariate", "expovariate", "lognormvariate"): - result = self.__data.draw(st.floats(min_value=0.0)) + result = self.__data.draw(floats(min_value=0.0)) elif method == "shuffle": - result = self.__data.draw(st.permutations(range(len(kwargs["x"])))) + result = self.__data.draw(permutations(range(len(kwargs["x"])))) # This is tested for but only appears in 3.9 so doesn't appear in coverage. elif method == "randbytes": # pragma: no cover n = kwargs["n"] - result = self.__data.draw(st.binary(min_size=n, max_size=n)) + result = self.__data.draw(binary(min_size=n, max_size=n)) else: raise NotImplementedError(method) @@ -390,7 +392,7 @@ def convert_kwargs(name, kwargs): class TrueRandom(HypothesisRandom): def __init__(self, seed, note_method_calls): - HypothesisRandom.__init__(self, note_method_calls) + super().__init__(note_method_calls) self.__seed = seed self.__random = Random(seed) diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/recursive.py b/hypothesis-python/src/hypothesis/strategies/_internal/recursive.py index 6cbb71175d..afc845de5d 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/recursive.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/recursive.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import threading from contextlib import contextmanager @@ -66,13 +61,13 @@ def do_draw(self, data): @contextmanager def capped(self, max_templates): - assert not self.currently_capped try: + was_capped = self.currently_capped self.currently_capped = True self.marker = max_templates yield finally: - self.currently_capped = False + self.currently_capped = was_capped class RecursiveStrategy(SearchStrategy): diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/regex.py b/hypothesis-python/src/hypothesis/strategies/_internal/regex.py index f66a91cdeb..b93e8d192b 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/regex.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/regex.py @@ -1,22 +1,22 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import operator import re -import sre_constants as sre -import sre_parse + +try: # pragma: no cover + import re._constants as sre + import re._parser as sre_parse +except ImportError: # Python < 3.11 + import sre_constants as sre # type: ignore + import sre_parse # type: ignore from hypothesis import reject, strategies as st from hypothesis.internal.charmap import as_general_categories, categories @@ -161,7 +161,7 @@ def add_category(self, category): self._categories |= UNICODE_CATEGORIES - UNICODE_WORD_CATEGORIES self._blacklist_chars.add("_") else: - raise NotImplementedError("Unknown character category: %s" % category) + raise NotImplementedError(f"Unknown character category: {category}") def add_char(self, char): """Add given char to the whitelist.""" @@ -218,7 +218,7 @@ def base_regex_strategy(regex, parsed=None): ) -def regex_strategy(regex, fullmatch): +def regex_strategy(regex, fullmatch, *, _temp_jsonschema_hack_no_end_newline=False): if not hasattr(regex, "pattern"): regex = re.compile(regex) @@ -226,6 +226,11 @@ def regex_strategy(regex, fullmatch): parsed = sre_parse.parse(regex.pattern, flags=regex.flags) + if fullmatch: + if not parsed: + return st.just("" if is_unicode else b"") + return base_regex_strategy(regex, parsed).filter(regex.fullmatch) + if not parsed: if is_unicode: return st.text() @@ -244,9 +249,7 @@ def regex_strategy(regex, fullmatch): right_pad = base_padding_strategy left_pad = base_padding_strategy - if fullmatch: - right_pad = empty - elif parsed[-1][0] == sre.AT: + if parsed[-1][0] == sre.AT: if parsed[-1][1] == sre.AT_END_STRING: right_pad = empty elif parsed[-1][1] == sre.AT_END: @@ -256,9 +259,16 @@ def regex_strategy(regex, fullmatch): ) else: right_pad = st.one_of(empty, newline) - if fullmatch: - left_pad = empty - elif parsed[0][0] == sre.AT: + + # This will be removed when a regex-syntax-translation library exists. + # It's a pretty nasty hack, but means that we can match the semantics + # of JSONschema's compatible subset of ECMA regex, which is important + # for hypothesis-jsonschema and Schemathesis. See e.g. + # https://github.com/schemathesis/schemathesis/issues/1241 + if _temp_jsonschema_hack_no_end_newline: + right_pad = empty + + if parsed[0][0] == sre.AT: if parsed[0][1] == sre.AT_BEGINNING_STRING: left_pad = empty elif parsed[0][1] == sre.AT_BEGINNING: @@ -414,7 +424,7 @@ def recurse(codes): else: # Currently there are no known code points other than # handled here. This code is just future proofing - raise NotImplementedError("Unknown charset code: %s" % charset_code) + raise NotImplementedError(f"Unknown charset code: {charset_code}") return builder.strategy elif code == sre.ANY: @@ -488,4 +498,4 @@ def recurse(codes): else: # Currently there are no known code points other than handled here. # This code is just future proofing - raise NotImplementedError("Unknown code point: %s" % repr(code)) + raise NotImplementedError(f"Unknown code point: {code!r}") diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/shared.py b/hypothesis-python/src/hypothesis/strategies/_internal/shared.py index 39b96673a0..fe495db9d6 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/shared.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/shared.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis.strategies._internal import SearchStrategy diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py b/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py index cdbd45f9c2..edc7191136 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py @@ -1,28 +1,35 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import sys import warnings -from collections import defaultdict -from random import choice as random_choice -from typing import Any, Callable, Generic, List, TypeVar, Union +from collections import abc, defaultdict +from random import shuffle +from typing import ( + Any, + Callable, + Dict, + Generic, + List, + Sequence, + TypeVar, + Union, + cast, + overload, +) from hypothesis._settings import HealthCheck, Phase, Verbosity, settings from hypothesis.control import _current_build_context, assume from hypothesis.errors import ( HypothesisException, + HypothesisWarning, InvalidArgument, NonInteractiveExampleWarning, UnsatisfiedAssumption, @@ -36,11 +43,19 @@ ) from hypothesis.internal.coverage import check_function from hypothesis.internal.lazyformat import lazyformat -from hypothesis.internal.reflection import get_pretty_function_description +from hypothesis.internal.reflection import ( + get_pretty_function_description, + is_identity_function, +) +from hypothesis.strategies._internal.utils import defines_strategy from hypothesis.utils.conventions import UniqueIdentifier Ex = TypeVar("Ex", covariant=True) +Ex_Inv = TypeVar("Ex_Inv") T = TypeVar("T") +T3 = TypeVar("T3") +T4 = TypeVar("T4") +T5 = TypeVar("T5") calculating = UniqueIdentifier("calculating") @@ -167,7 +182,7 @@ def recur_inner(other): # Note: This is actually covered, by test_very_deep_deferral # in tests/cover/test_deferred_strategies.py. Unfortunately it # runs into a coverage bug. See - # https://bitbucket.org/ned/coveragepy/issues/605/ + # https://github.com/nedbat/coveragepy/issues/605 # for details. if count > 50: # pragma: no cover key = frozenset(mapping.items()) @@ -229,6 +244,12 @@ def available(self, data): # Returns True if values from this strategy can safely be reused without # this causing unexpected behaviour. + + # True if values from this strategy can be implicitly reused (e.g. as + # background values in a numpy array) without causing surprising + # user-visible behaviour. Should be false for built-in strategies that + # produce mutable values, and for strategies that have been mapped/filtered + # by arbitrary user-provided functions. has_reusable_values = recursive_property("has_reusable_values", True) # Whether this strategy is suitable for holding onto in a cache. @@ -268,6 +289,7 @@ def example(self) -> Ex: "it performs better, saves and replays failures to avoid flakiness, " "and reports minimal examples. (strategy: %r)" % (self,), NonInteractiveExampleWarning, + stacklevel=2, ) context = _current_build_context.value @@ -291,6 +313,11 @@ def example(self) -> Ex: "#drawing-interactively-in-tests for more details." ) + try: + return self.__examples.pop() + except (AttributeError, IndexError): + self.__examples: List[Ex] = [] + from hypothesis.core import given # Note: this function has a weird name because it might appear in @@ -298,18 +325,18 @@ def example(self) -> Ex: @given(self) @settings( database=None, - max_examples=10, + max_examples=100, deadline=None, verbosity=Verbosity.quiet, phases=(Phase.generate,), suppress_health_check=HealthCheck.all(), ) def example_generating_inner_function(ex): - examples.append(ex) + self.__examples.append(ex) - examples = [] # type: List[Ex] example_generating_inner_function() - return random_choice(examples) + shuffle(self.__examples) + return self.__examples.pop() def map(self, pack: Callable[[Ex], T]) -> "SearchStrategy[T]": """Returns a new strategy that generates values by generating a value @@ -317,6 +344,8 @@ def map(self, pack: Callable[[Ex], T]) -> "SearchStrategy[T]": This method is part of the public API. """ + if is_identity_function(pack): + return self # type: ignore # Mypy has no way to know that `Ex == T` return MappedSearchStrategy(pack=pack, strategy=self) def flatmap( @@ -342,11 +371,17 @@ def filter(self, condition: Callable[[Ex], Any]) -> "SearchStrategy[Ex]": """ return FilteredStrategy(conditions=(condition,), strategy=self) - def do_filtered_draw(self, data, filter_strategy): - # Hook for strategies that want to override the behaviour of - # FilteredStrategy. Most strategies don't, so by default we delegate - # straight back to the default filtered-draw implementation. - return filter_strategy.default_do_filtered_draw(data) + def _filter_for_filtered_draw(self, condition): + # Hook for parent strategies that want to perform fallible filtering + # on one of their internal strategies (e.g. UniqueListStrategy). + # The returned object must have a `.do_filtered_draw(data)` method + # that behaves like `do_draw`, but returns the sentinel object + # `filter_not_satisfied` if the condition could not be satisfied. + + # This is separate from the main `filter` method so that strategies + # can override `filter` without having to also guarantee a + # `do_filtered_draw` method. + return FilteredStrategy(conditions=(condition,), strategy=self) @property def branches(self) -> List["SearchStrategy[Ex]"]: @@ -362,6 +397,14 @@ def __or__(self, other: "SearchStrategy[T]") -> "SearchStrategy[Union[Ex, T]]": raise ValueError(f"Cannot | a SearchStrategy with {other!r}") return OneOfStrategy((self, other)) + def __bool__(self) -> bool: + warnings.warn( + f"bool({self!r}) is always True, did you mean to draw a value?", + HypothesisWarning, + stacklevel=2, + ) + return True + def validate(self) -> None: """Throw an exception if the strategy is not valid. @@ -378,7 +421,7 @@ def validate(self) -> None: self.validate_called = False raise - LABELS = {} # type: dict + LABELS: Dict[type, int] = {} @property def class_label(self): @@ -392,13 +435,13 @@ def class_label(self): return result @property - def label(self): + def label(self) -> int: if self.__label is calculating: return 0 if self.__label is None: self.__label = calculating self.__label = self.calc_label() - return self.__label + return cast(int, self.__label) def calc_label(self): return self.class_label @@ -431,7 +474,7 @@ class SampledFromStrategy(SearchStrategy): """ def __init__(self, elements, repr_=None, transformations=()): - SearchStrategy.__init__(self) + super().__init__() self.elements = cu.check_sample(elements, "sampled_from") assert self.elements self.repr_ = repr_ @@ -452,20 +495,29 @@ def filter(self, condition): ) def __repr__(self): - return (self.repr_ or f"sampled_from({list(self.elements)!r})") + "".join( + return ( + self.repr_ + or "sampled_from([" + + ", ".join(map(get_pretty_function_description, self.elements)) + + "])" + ) + "".join( f".{name}({get_pretty_function_description(f)})" for name, f in self._transformations ) def calc_has_reusable_values(self, recur): - return True + # Because our custom .map/.filter implementations skip the normal + # wrapper strategies (which would automatically return False for us), + # we need to manually return False here if any transformations have + # been applied. + return not self._transformations def calc_is_cacheable(self, recur): return is_simple_data(self.elements) - def _transform(self, element, conditions=()): + def _transform(self, element): # Used in UniqueSampledListStrategy - for name, f in self._transformations + tuple(("filter", c) for c in conditions): + for name, f in self._transformations: if name == "map": element = f(element) else: @@ -475,33 +527,26 @@ def _transform(self, element, conditions=()): return element def do_draw(self, data): - result = self.do_filtered_draw(data, self) + result = self.do_filtered_draw(data) if result is filter_not_satisfied: data.note_event(f"Aborted test because unable to satisfy {self!r}") data.mark_invalid() return result - def get_element(self, i, conditions=()): - return self._transform(self.elements[i], conditions=conditions) + def get_element(self, i): + return self._transform(self.elements[i]) - def do_filtered_draw(self, data, filter_strategy): + def do_filtered_draw(self, data): # Set of indices that have been tried so far, so that we never test # the same element twice during a draw. known_bad_indices = set() - # If we're being called via FilteredStrategy, the filter_strategy argument - # might have additional conditions we have to fulfill. - if isinstance(filter_strategy, FilteredStrategy): - conditions = filter_strategy.flat_conditions - else: - conditions = () - # Start with ordinary rejection sampling. It's fast if it works, and # if it doesn't work then it was only a small amount of overhead. for _ in range(3): i = cu.integer_range(data, 0, len(self.elements) - 1) if i not in known_bad_indices: - element = self.get_element(i, conditions=conditions) + element = self.get_element(i) if element is not filter_not_satisfied: return element if not known_bad_indices: @@ -534,7 +579,7 @@ def do_filtered_draw(self, data, filter_strategy): allowed = [] for i in range(min(len(self.elements), cutoff)): if i not in known_bad_indices: - element = self.get_element(i, conditions=conditions) + element = self.get_element(i) if element is not filter_not_satisfied: allowed.append((i, element)) if len(allowed) > speculative_index: @@ -553,7 +598,7 @@ def do_filtered_draw(self, data, filter_strategy): return filter_not_satisfied -class OneOfStrategy(SearchStrategy): +class OneOfStrategy(SearchStrategy[Ex]): """Implements a union of strategies. Given a number of strategies this generates values which could have come from any of them. @@ -563,7 +608,7 @@ class OneOfStrategy(SearchStrategy): """ def __init__(self, strategies): - SearchStrategy.__init__(self) + super().__init__() strategies = tuple(strategies) self.original_strategies = list(strategies) self.__element_strategies = None @@ -610,7 +655,7 @@ def element_strategies(self): def calc_label(self): return combine_labels( - self.class_label, *[p.label for p in self.original_strategies] + self.class_label, *(p.label for p in self.original_strategies) ) def do_draw(self, data: ConjectureData) -> Ex: @@ -639,8 +684,113 @@ def branches(self): else: return [self] + def filter(self, condition): + return FilteredStrategy( + OneOfStrategy([s.filter(condition) for s in self.original_strategies]), + conditions=(), + ) + + +@overload +def one_of( + __args: Sequence[SearchStrategy[Any]], +) -> SearchStrategy[Any]: # pragma: no cover + ... + + +@overload # noqa: F811 +def one_of(__a1: SearchStrategy[Ex]) -> SearchStrategy[Ex]: # pragma: no cover + ... + + +@overload # noqa: F811 +def one_of( + __a1: SearchStrategy[Ex], __a2: SearchStrategy[T] +) -> SearchStrategy[Union[Ex, T]]: # pragma: no cover + ... + + +@overload # noqa: F811 +def one_of( + __a1: SearchStrategy[Ex], __a2: SearchStrategy[T], __a3: SearchStrategy[T3] +) -> SearchStrategy[Union[Ex, T, T3]]: # pragma: no cover + ... + + +@overload # noqa: F811 +def one_of( + __a1: SearchStrategy[Ex], + __a2: SearchStrategy[T], + __a3: SearchStrategy[T3], + __a4: SearchStrategy[T4], +) -> SearchStrategy[Union[Ex, T, T3, T4]]: # pragma: no cover + ... + + +@overload # noqa: F811 +def one_of( + __a1: SearchStrategy[Ex], + __a2: SearchStrategy[T], + __a3: SearchStrategy[T3], + __a4: SearchStrategy[T4], + __a5: SearchStrategy[T5], +) -> SearchStrategy[Union[Ex, T, T3, T4, T5]]: # pragma: no cover + ... -class MappedSearchStrategy(SearchStrategy): + +@overload # noqa: F811 +def one_of(*args: SearchStrategy[Any]) -> SearchStrategy[Any]: # pragma: no cover + ... + + +@defines_strategy(never_lazy=True) +def one_of( + *args: Union[Sequence[SearchStrategy[Any]], SearchStrategy[Any]] +) -> SearchStrategy[Any]: # noqa: F811 + # Mypy workaround alert: Any is too loose above; the return parameter + # should be the union of the input parameters. Unfortunately, Mypy <=0.600 + # raises errors due to incompatible inputs instead. See #1270 for links. + # v0.610 doesn't error; it gets inference wrong for 2+ arguments instead. + """Return a strategy which generates values from any of the argument + strategies. + + This may be called with one iterable argument instead of multiple + strategy arguments, in which case ``one_of(x)`` and ``one_of(*x)`` are + equivalent. + + Examples from this strategy will generally shrink to ones that come from + strategies earlier in the list, then shrink according to behaviour of the + strategy that produced them. In order to get good shrinking behaviour, + try to put simpler strategies first. e.g. ``one_of(none(), text())`` is + better than ``one_of(text(), none())``. + + This is especially important when using recursive strategies. e.g. + ``x = st.deferred(lambda: st.none() | st.tuples(x, x))`` will shrink well, + but ``x = st.deferred(lambda: st.tuples(x, x) | st.none())`` will shrink + very badly indeed. + """ + if len(args) == 1 and not isinstance(args[0], SearchStrategy): + try: + args = tuple(args[0]) + except TypeError: + pass + if len(args) == 1 and isinstance(args[0], SearchStrategy): + # This special-case means that we can one_of over lists of any size + # without incurring any performance overhead when there is only one + # strategy, and keeps our reprs simple. + return args[0] + if args and not any(isinstance(a, SearchStrategy) for a in args): + # And this special case is to give a more-specific error message if it + # seems that the user has confused `one_of()` for `sampled_from()`; + # the remaining validation is left to OneOfStrategy. See PR #2627. + raise InvalidArgument( + f"Did you mean st.sampled_from({list(args)!r})? st.one_of() is used " + "to combine strategies, but all of the arguments were of other types." + ) + return OneOfStrategy(args) + + +class MappedSearchStrategy(SearchStrategy[Ex]): """A strategy which is defined purely by conversion to and from another strategy. @@ -648,7 +798,7 @@ class MappedSearchStrategy(SearchStrategy): """ def __init__(self, strategy, pack=None): - SearchStrategy.__init__(self) + super().__init__() self.mapped_strategy = strategy if pack is not None: self.pack = pack @@ -673,20 +823,25 @@ def do_validate(self): def pack(self, x): """Take a value produced by the underlying mapped_strategy and turn it into a value suitable for outputting from this strategy.""" - raise NotImplementedError("%s.pack()" % (self.__class__.__name__)) - - def do_draw(self, data: ConjectureData) -> Ex: - for _ in range(3): - i = data.index - try: - data.start_example(MAPPED_SEARCH_STRATEGY_DO_DRAW_LABEL) - result = self.pack(data.draw(self.mapped_strategy)) - data.stop_example() - return result - except UnsatisfiedAssumption: - data.stop_example(discard=True) - if data.index == i: - raise + raise NotImplementedError(f"{self.__class__.__name__}.pack()") + + def do_draw(self, data: ConjectureData) -> Any: + with warnings.catch_warnings(): + if isinstance(self.pack, type) and issubclass( + self.pack, (abc.Mapping, abc.Set) + ): + warnings.simplefilter("ignore", BytesWarning) + for _ in range(3): + i = data.index + try: + data.start_example(MAPPED_SEARCH_STRATEGY_DO_DRAW_LABEL) + result = self.pack(data.draw(self.mapped_strategy)) # type: ignore + data.stop_example() + return result + except UnsatisfiedAssumption: + data.stop_example(discard=True) + if data.index == i: + raise raise UnsatisfiedAssumption() @property @@ -700,19 +855,17 @@ def branches(self) -> List[SearchStrategy[Ex]]: filter_not_satisfied = UniqueIdentifier("filter not satisfied") -class FilteredStrategy(SearchStrategy): +class FilteredStrategy(SearchStrategy[Ex]): def __init__(self, strategy, conditions): super().__init__() if isinstance(strategy, FilteredStrategy): - # Flatten chained filters into a single filter with multiple - # conditions. + # Flatten chained filters into a single filter with multiple conditions. self.flat_conditions = strategy.flat_conditions + conditions self.filtered_strategy = strategy.filtered_strategy else: self.flat_conditions = conditions self.filtered_strategy = strategy - assert self.flat_conditions assert isinstance(self.flat_conditions, tuple) assert not isinstance(self.filtered_strategy, FilteredStrategy) @@ -729,23 +882,58 @@ def __repr__(self): self._cached_repr = "{!r}{}".format( self.filtered_strategy, "".join( - ".filter(%s)" % get_pretty_function_description(cond) + f".filter({get_pretty_function_description(cond)})" for cond in self.flat_conditions ), ) return self._cached_repr def do_validate(self): + # Start by validating our inner filtered_strategy. If this was a LazyStrategy, + # validation also reifies it so that subsequent calls to e.g. `.filter()` will + # be passed through. self.filtered_strategy.validate() + # So now we have a reified inner strategy, we'll replay all our saved + # predicates in case some or all of them can be rewritten. Note that this + # replaces the `fresh` strategy too! + fresh = self.filtered_strategy + for cond in self.flat_conditions: + fresh = fresh.filter(cond) + if isinstance(fresh, FilteredStrategy): + # In this case we have at least some non-rewritten filter predicates, + # so we just re-initialize the strategy. + FilteredStrategy.__init__( + self, fresh.filtered_strategy, fresh.flat_conditions + ) + else: + # But if *all* the predicates were rewritten... well, do_validate() is + # an in-place method so we still just re-initialize the strategy! + FilteredStrategy.__init__(self, fresh, ()) + + def filter(self, condition): + # If we can, it's more efficient to rewrite our strategy to satisfy the + # condition. We therefore exploit the fact that the order of predicates + # doesn't matter (`f(x) and g(x) == g(x) and f(x)`) by attempting to apply + # condition directly to our filtered strategy as the inner-most filter. + out = self.filtered_strategy.filter(condition) + # If it couldn't be rewritten, we'll get a new FilteredStrategy - and then + # combine the conditions of each in our expected newest=last order. + if isinstance(out, FilteredStrategy): + return FilteredStrategy( + out.filtered_strategy, self.flat_conditions + out.flat_conditions + ) + # But if it *could* be rewritten, we can return the more efficient form! + return FilteredStrategy(out, self.flat_conditions) @property def condition(self): if self.__condition is None: - assert self.flat_conditions if len(self.flat_conditions) == 1: - # Avoid an extra indirection in the common case of only one - # condition. + # Avoid an extra indirection in the common case of only one condition. self.__condition = self.flat_conditions[0] + elif len(self.flat_conditions) == 0: + # Possible, if unlikely, due to filter predicate rewriting + self.__condition = lambda _: True else: self.__condition = lambda x: all( cond(x) for cond in self.flat_conditions @@ -753,9 +941,7 @@ def condition(self): return self.__condition def do_draw(self, data: ConjectureData) -> Ex: - result = self.filtered_strategy.do_filtered_draw( - data=data, filter_strategy=self - ) + result = self.do_filtered_draw(data) if result is not filter_not_satisfied: return result @@ -766,7 +952,7 @@ def do_draw(self, data: ConjectureData) -> Ex: def note_retried(self, data): data.note_event(lazyformat("Retried draw from %r to satisfy filter", self)) - def default_do_filtered_draw(self, data): + def do_filtered_draw(self, data): for i in range(3): start_index = data.index data.start_example(FILTERED_SEARCH_STRATEGY_DO_DRAW_LABEL) diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/strings.py b/hypothesis-python/src/hypothesis/strategies/_internal/strings.py index 291d9da026..e1dc1f4da5 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/strings.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/strings.py @@ -1,22 +1,21 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER -from hypothesis.errors import InvalidArgument +import copy +import warnings + +from hypothesis.errors import HypothesisWarning, InvalidArgument from hypothesis.internal import charmap from hypothesis.internal.conjecture.utils import biased_coin, integer_range from hypothesis.internal.intervalsets import IntervalSet +from hypothesis.strategies._internal.collections import ListStrategy from hypothesis.strategies._internal.strategies import SearchStrategy @@ -42,8 +41,9 @@ def __init__( include_characters=whitelist_characters, exclude_characters=blacklist_characters, ) - if not intervals: - arguments = [ + self._arg_repr = ", ".join( + f"{k}={v!r}" + for k, v in [ ("whitelist_categories", whitelist_categories), ("blacklist_categories", blacklist_categories), ("whitelist_characters", whitelist_characters), @@ -51,10 +51,12 @@ def __init__( ("min_codepoint", min_codepoint), ("max_codepoint", max_codepoint), ] + if not (v in (None, "") or (k == "blacklist_categories" and v == ("Cs",))) + ) + if not intervals: raise InvalidArgument( "No characters are allowed to be generated by this " - "combination of arguments: " - + ", ".join("%s=%r" % arg for arg in arguments if arg[1] is not None) + f"combination of arguments: {self._arg_repr}" ) self.intervals = IntervalSet(intervals) self.zero_point = self.intervals.index_above(ord("0")) @@ -62,6 +64,9 @@ def __init__( self.intervals.index_above(ord("Z")), len(self.intervals) - 1 ) + def __repr__(self): + return f"characters({self._arg_repr})" + def do_draw(self, data): if len(self.intervals) > 256: if biased_coin(data, 0.2): @@ -99,6 +104,73 @@ def rewrite_integer(self, i): return i +class TextStrategy(ListStrategy): + def do_draw(self, data): + return "".join(super().do_draw(data)) + + def __repr__(self): + args = [] + if repr(self.element_strategy) != "characters()": + args.append(repr(self.element_strategy)) + if self.min_size: + args.append(f"min_size={self.min_size}") + if self.max_size < float("inf"): + args.append(f"max_size={self.max_size}") + return f"text({', '.join(args)})" + + # See https://docs.python.org/3/library/stdtypes.html#string-methods + # These methods always return Truthy values for any nonempty string. + _nonempty_filters = ListStrategy._nonempty_filters + ( + str, + str.capitalize, + str.casefold, + str.encode, + str.expandtabs, + str.join, + str.lower, + str.rsplit, + str.split, + str.splitlines, + str.swapcase, + str.title, + str.upper, + ) + _nonempty_and_content_filters = ( + str.isidentifier, + str.islower, + str.isupper, + str.isalnum, + str.isalpha, + str.isascii, + str.isdecimal, + str.isdigit, + str.isnumeric, + str.isspace, + str.istitle, + str.lstrip, + str.rstrip, + str.strip, + ) + + def filter(self, condition): + if condition in (str.lower, str.title, str.upper): + warnings.warn( + f"You applied str.{condition.__name__} as a filter, but this allows " + f"all nonempty strings! Did you mean str.is{condition.__name__}?", + HypothesisWarning, + ) + # We use ListStrategy filter logic for the conditions that *only* imply + # the string is nonempty. Here, we increment the min_size but still apply + # the filter for conditions that imply nonempty *and specific contents*. + if condition in self._nonempty_and_content_filters: + assert self.max_size >= 1, "Always-empty is special cased in st.text()" + self = copy.copy(self) + self.min_size = max(1, self.min_size) + return ListStrategy.filter(self, condition) + + return super().filter(condition) + + class FixedSizeBytes(SearchStrategy): def __init__(self, size): self.size = size diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/types.py b/hypothesis-python/src/hypothesis/strategies/_internal/types.py index 0b89a3e2ab..4e410bcf58 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/types.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/types.py @@ -1,18 +1,14 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER +import builtins import collections import datetime import decimal @@ -32,9 +28,9 @@ from hypothesis import strategies as st from hypothesis.errors import InvalidArgument, ResolutionFailed -from hypothesis.internal.compat import ForwardRef, typing_root_type +from hypothesis.internal.compat import PYPY, BaseExceptionGroup, ExceptionGroup from hypothesis.internal.conjecture.utils import many as conjecture_utils_many -from hypothesis.strategies._internal.datetime import zoneinfo +from hypothesis.strategies._internal.datetime import zoneinfo # type: ignore from hypothesis.strategies._internal.ipaddress import ( SPECIAL_IPv4_RANGES, SPECIAL_IPv6_RANGES, @@ -43,54 +39,220 @@ from hypothesis.strategies._internal.lazy import unwrap_strategies from hypothesis.strategies._internal.strategies import OneOfStrategy +GenericAlias: typing.Any +UnionType: typing.Any +try: + # The type of PEP-604 unions (`int | str`), added in Python 3.10 + from types import GenericAlias, UnionType +except ImportError: + GenericAlias = () + UnionType = () + try: import typing_extensions except ImportError: typing_extensions = None # type: ignore try: - from typing import GenericMeta as _GenericMeta # python < 3.7 + from typing import _AnnotatedAlias # type: ignore except ImportError: - _GenericMeta = () # type: ignore + try: + from typing_extensions import _AnnotatedAlias + except ImportError: + _AnnotatedAlias = () +TypeAliasTypes: tuple = () try: - from typing import _GenericAlias # type: ignore # python >= 3.7 -except ImportError: - _GenericAlias = () + TypeAliasTypes += (typing.TypeAlias,) +except AttributeError: + pass # Is missing for `python<3.10` +try: + TypeAliasTypes += (typing_extensions.TypeAlias,) +except AttributeError: # pragma: no cover + pass # Is missing for `typing_extensions<3.10` + +ClassVarTypes: tuple = (typing.ClassVar,) +try: + ClassVarTypes += (typing_extensions.ClassVar,) +except AttributeError: # pragma: no cover + pass # `typing_extensions` might not be installed + +FinalTypes: tuple = () +try: + FinalTypes += (typing.Final,) +except AttributeError: # pragma: no cover + pass # Is missing for `python<3.8` +try: + FinalTypes += (typing_extensions.Final,) +except AttributeError: # pragma: no cover + pass # `typing_extensions` might not be installed + +ConcatenateTypes: tuple = () +try: + ConcatenateTypes += (typing.Concatenate,) +except AttributeError: # pragma: no cover + pass # Is missing for `python<3.8` +try: + ConcatenateTypes += (typing_extensions.Concatenate,) +except AttributeError: # pragma: no cover + pass # `typing_extensions` might not be installed + +ParamSpecTypes: tuple = () +try: + ParamSpecTypes += (typing.ParamSpec,) +except AttributeError: # pragma: no cover + pass # Is missing for `python<3.8` +try: + ParamSpecTypes += (typing_extensions.ParamSpec,) +except AttributeError: # pragma: no cover + pass # `typing_extensions` might not be installed + +TypeGuardTypes: tuple = () +try: + TypeGuardTypes += (typing.TypeGuard,) +except AttributeError: # pragma: no cover + pass # Is missing for `python<3.8` +try: + TypeGuardTypes += (typing_extensions.TypeGuard,) +except AttributeError: # pragma: no cover + pass # `typing_extensions` might not be installed + + +RequiredTypes: tuple = () +try: + RequiredTypes += (typing.Required,) # type: ignore +except AttributeError: # pragma: no cover + pass # Is missing for `python<3.11` +try: + RequiredTypes += (typing_extensions.Required,) +except AttributeError: # pragma: no cover + pass # `typing_extensions` might not be installed + + +NotRequiredTypes: tuple = () +try: + NotRequiredTypes += (typing.NotRequired,) # type: ignore +except AttributeError: # pragma: no cover + pass # Is missing for `python<3.11` +try: + NotRequiredTypes += (typing_extensions.NotRequired,) +except AttributeError: # pragma: no cover + pass # `typing_extensions` might not be installed + + +# We use this variable to be sure that we are working with a type from `typing`: +typing_root_type = (typing._Final, typing._GenericAlias) # type: ignore + +# We use this to disallow all non-runtime types from being registered and resolved. +# By "non-runtime" we mean: types that do not really exist in python's +# and are just added for more fancy type annotations. +# `Final` is a great example: it just indicates +# that this value can't be reassigned. +NON_RUNTIME_TYPES = ( + typing.Any, + *ClassVarTypes, + *TypeAliasTypes, + *FinalTypes, + *ConcatenateTypes, + *ParamSpecTypes, + *TypeGuardTypes, +) +for name in ( + "Annotated", + "NoReturn", + "Self", + "Required", + "NotRequired", + "Never", + "TypeVarTuple", + "Unpack", + "LiteralString", +): + try: + NON_RUNTIME_TYPES += (getattr(typing, name),) + except AttributeError: + pass + try: + NON_RUNTIME_TYPES += (getattr(typing_extensions, name),) + except AttributeError: # pragma: no cover + pass # typing_extensions might not be installed def type_sorting_key(t): """Minimise to None, then non-container types, then container types.""" - if not is_a_type(t): # This branch is for Python < 3.8 + if not (is_a_type(t) or is_typing_literal(t)): # This branch is for Python < 3.8 raise InvalidArgument(f"thing={t} must be a type") # pragma: no cover if t is None or t is type(None): # noqa: E721 return (-1, repr(t)) + t = getattr(t, "__origin__", t) if not isinstance(t, type): # pragma: no cover # Some generics in the typing module are not actually types in 3.7 return (2, repr(t)) return (int(issubclass(t, collections.abc.Container)), repr(t)) +def _compatible_args(args, superclass_args): + """Check that the args of two generic types are compatible for try_issubclass.""" + assert superclass_args is not None + if args is None: + return True + return len(args) == len(superclass_args) and all( + # "a==b or either is a typevar" is a hacky approximation, but it's + # good enough for all the cases that I've seen so far and has the + # substantial virtue of (relative) simplicity. + a == b or isinstance(a, typing.TypeVar) or isinstance(b, typing.TypeVar) + for a, b in zip(args, superclass_args) + ) + + def try_issubclass(thing, superclass): - thing = getattr(thing, "__origin__", None) or thing - superclass = getattr(superclass, "__origin__", None) or superclass try: - return issubclass(thing, superclass) + # In this case we're looking at two distinct classes - which might be generics. + # That brings in some complications: + if issubclass( + getattr(thing, "__origin__", None) or thing, + getattr(superclass, "__origin__", None) or superclass, + ): + superclass_args = getattr(superclass, "__args__", None) + if not superclass_args: + # The superclass is not generic, so we're definitely a subclass. + return True + # Sadly this is just some really fiddly logic to handle all the cases + # of user-defined generic types, types inheriting from parametrised + # generics, and so on. If you need to change this code, read PEP-560 + # and Hypothesis issue #2951 closely first, and good luck. The tests + # will help you, I hope - good luck. + if getattr(thing, "__args__", None) is not None: + return True + for orig_base in getattr(thing, "__orig_bases__", None) or [None]: + args = getattr(orig_base, "__args__", None) + if _compatible_args(args, superclass_args): + return True + return False except (AttributeError, TypeError): # Some types can't be the subject or object of an instance or subclass check return False def is_a_new_type(thing): - # At runtime, `typing.NewType` returns an identity function rather - # than an actual type, but we can check whether that thing matches. - return ( - hasattr(thing, "__supertype__") - and ( - getattr(thing, "__module__", None) == "typing" - or getattr(thing, "__module__", None) == "typing_extensions" + if not isinstance(typing.NewType, type): + # At runtime, `typing.NewType` returns an identity function rather + # than an actual type, but we can check whether that thing matches. + return ( + hasattr(thing, "__supertype__") + and getattr(thing, "__module__", None) in ("typing", "typing_extensions") + and inspect.isfunction(thing) ) - and inspect.isfunction(thing) + # In 3.10 and later, NewType is actually a class - which simplifies things. + # See https://bugs.python.org/issue44353 for links to the various patches. + return isinstance(thing, typing.NewType) # pragma: no cover # on 3.8, anyway + + +def is_a_union(thing): + """Return True if thing is a typing.Union or types.UnionType (in py310).""" + return ( + isinstance(thing, UnionType) + or getattr(thing, "__origin__", None) is typing.Union ) @@ -104,18 +266,36 @@ def is_typing_literal(thing): hasattr(typing, "Literal") and getattr(thing, "__origin__", None) == typing.Literal or hasattr(typing_extensions, "Literal") - and ( - getattr(thing, "__origin__", None) == typing_extensions.Literal - or sys.version_info[:2] == (3, 6) - and isinstance(thing, type(typing_extensions.Literal[None])) - ) + and getattr(thing, "__origin__", None) == typing_extensions.Literal + ) + + +def is_annotated_type(thing): + return ( + isinstance(thing, _AnnotatedAlias) + and getattr(thing, "__args__", None) is not None + ) + + +def find_annotated_strategy(annotated_type): # pragma: no cover + flattened_meta = [] + + all_args = ( + *getattr(annotated_type, "__args__", ()), + *getattr(annotated_type, "__metadata__", ()), ) + for arg in all_args: + if is_annotated_type(arg): + flattened_meta.append(find_annotated_strategy(arg)) + if isinstance(arg, st.SearchStrategy): + flattened_meta.append(arg) + return flattened_meta[-1] if flattened_meta else None def has_type_arguments(type_): """Decides whethere or not this type has applied type arguments.""" args = getattr(type_, "__args__", None) - if args and isinstance(type_, (_GenericAlias, _GenericMeta)): + if args and isinstance(type_, (typing._GenericAlias, GenericAlias)): # There are some cases when declared types do already have type arguments # Like `Sequence`, that is `_GenericAlias(abc.Sequence[T])[T]` parameters = getattr(type_, "__parameters__", None) @@ -125,13 +305,11 @@ def has_type_arguments(type_): def is_generic_type(type_): - """Decides whethere a given type is generic or not.""" + """Decides whether a given type is generic or not.""" # The ugly truth is that `MyClass`, `MyClass[T]`, and `MyClass[int]` are very different. - # In different python versions the might have the same type (3.6) - # or it can be regular type vs `_GenericAlias` (3.7+) # We check for `MyClass[T]` and `MyClass[int]` with the first condition, - # while the second condition is for `MyClass` in `python3.7+`. - return isinstance(type_, typing_root_type) or ( + # while the second condition is for `MyClass`. + return isinstance(type_, typing_root_type + (GenericAlias,)) or ( isinstance(type_, type) and typing.Generic in type_.__mro__ ) @@ -147,14 +325,6 @@ def _try_import_forward_ref(thing, bound): # pragma: no cover try: return typing._eval_type(bound, vars(sys.modules[thing.__module__]), None) except (KeyError, AttributeError, NameError): - if ( - isinstance(thing, typing.TypeVar) - and getattr(thing, "__module__", None) == "typing" - ): - raise ResolutionFailed( - "It looks like you're using a TypeVar bound to a ForwardRef on Python " - "3.6, which is not supported - try ugrading to Python 3.7 or later." - ) from None # We fallback to `ForwardRef` instance, you can register it as a type as well: # >>> from typing import ForwardRef # >>> from hypothesis import strategies as st @@ -163,9 +333,8 @@ def _try_import_forward_ref(thing, bound): # pragma: no cover def from_typing_type(thing): - # We start with special-case support for Union and Tuple - the latter - # isn't actually a generic type. Then we handle Literal since it doesn't - # support `isinstance`. + # We start with special-case support for Tuple, which isn't actually a generic + # type; then Final, Literal, and Annotated since they don't support `isinstance`. # # We then explicitly error on non-Generic types, which don't carry enough # information to sensibly resolve to strategies at runtime. @@ -197,6 +366,15 @@ def from_typing_type(thing): else: literals.append(arg) return st.sampled_from(literals) + if is_annotated_type(thing): # pragma: no cover + # This requires Python 3.9+ or the typing_extensions package + annotated_strategy = find_annotated_strategy(thing) + if annotated_strategy is not None: + return annotated_strategy + args = thing.__args__ + assert args, "it's impossible to make an annotated type with no args" + annotated_type = args[0] + return st.from_type(annotated_type) # Now, confirm that we're dealing with a generic type as we expected if sys.version_info[:2] < (3, 9) and not isinstance( thing, typing_root_type @@ -205,11 +383,9 @@ def from_typing_type(thing): # Some "generic" classes are not generic *in* anything - for example both # Hashable and Sized have `__args__ == ()` on Python 3.7 or later. - # (In 3.6 they're just aliases for the collections.abc classes) origin = getattr(thing, "__origin__", thing) if ( - typing.Hashable is not collections.abc.Hashable - and origin in vars(collections.abc).values() + origin in vars(collections.abc).values() and len(getattr(thing, "__args__", None) or []) == 0 ): return st.from_type(origin) @@ -226,9 +402,6 @@ def from_typing_type(thing): # ItemsView can cause test_lookup.py::test_specialised_collection_types # to fail, due to weird isinstance behaviour around the elements. mapping.pop(typing.ItemsView, None) - if sys.version_info[:2] == (3, 6): # pragma: no cover - # `isinstance(dict().values(), Container) is False` on py36 only -_- - mapping.pop(typing.ValuesView, None) if typing.Deque in mapping and len(mapping) > 1: # Resolving generic sequences to include a deque is more trouble for e.g. # the ghostwriter than it's worth, via undefined names in the repr. @@ -244,7 +417,7 @@ def from_typing_type(thing): # if there is more than one allowed type, and the element type is # not either `int` or a Union with `int` as one of its elements. elem_type = (getattr(thing, "__args__", None) or ["not int"])[0] - if getattr(elem_type, "__origin__", None) is typing.Union: + if is_a_union(elem_type): union_elems = elem_type.__args__ else: union_elems = () @@ -253,6 +426,12 @@ def from_typing_type(thing): for T in list(union_elems) + [elem_type] ): mapping.pop(typing.ByteString, None) + elif ( + (not mapping) + and isinstance(thing, typing.ForwardRef) + and thing.__forward_arg__ in vars(builtins) + ): + return st.from_type(getattr(builtins, thing.__forward_arg__)) strategies = [ v if isinstance(v, st.SearchStrategy) else v(thing) for k, v in mapping.items() @@ -261,8 +440,8 @@ def from_typing_type(thing): empty = ", ".join(repr(s) for s in strategies if s.is_empty) if empty or not strategies: raise ResolutionFailed( - "Could not resolve %s to a strategy; consider using " - "register_type_strategy" % (empty or thing,) + f"Could not resolve {empty or thing} to a strategy; " + "consider using register_type_strategy" ) return st.one_of(strategies) @@ -277,7 +456,7 @@ def can_cast(type, value): def _networks(bits): - return st.tuples(st.integers(0, 2 ** bits - 1), st.integers(-bits, 0).map(abs)) + return st.tuples(st.integers(0, 2**bits - 1), st.integers(-bits, 0).map(abs)) utc_offsets = st.builds( @@ -294,7 +473,9 @@ def _networks(bits): # As a general rule, we try to limit this to scalars because from_type() # would have to decide on arbitrary collection elements, and we'd rather # not (with typing module generic types and some builtins as exceptions). -_global_type_lookup = { +_global_type_lookup: typing.Dict[ + type, typing.Union[st.SearchStrategy, typing.Callable[[type], st.SearchStrategy]] +] = { type(None): st.none(), bool: st.booleans(), int: st.integers(), @@ -334,7 +515,7 @@ def _networks(bits): st.none() | st.integers(), ), range: st.one_of( - st.integers(min_value=0).map(range), + st.builds(range, st.integers(min_value=0)), st.builds(range, st.integers(), st.integers()), st.builds(range, st.integers(), st.integers(), st.integers().filter(bool)), ), @@ -351,20 +532,55 @@ def _networks(bits): st.sampled_from(SPECIAL_IPv6_RANGES).map(ipaddress.IPv6Network), ), os.PathLike: st.builds(PurePath, st.text()), + UnicodeDecodeError: st.builds( + UnicodeDecodeError, + st.just("unknown encoding"), + st.just(b""), + st.just(0), + st.just(0), + st.just("reason"), + ), + UnicodeEncodeError: st.builds( + UnicodeEncodeError, + st.just("unknown encoding"), + st.text(), + st.just(0), + st.just(0), + st.just("reason"), + ), + UnicodeTranslateError: st.builds( + UnicodeTranslateError, st.text(), st.just(0), st.just(0), st.just("reason") + ), + BaseExceptionGroup: st.builds( + BaseExceptionGroup, + st.text(), + st.lists(st.from_type(BaseException), min_size=1), + ), + ExceptionGroup: st.builds( + ExceptionGroup, + st.text(), + st.lists(st.from_type(Exception), min_size=1), + ), + enumerate: st.builds(enumerate, st.just(())), + filter: st.builds(filter, st.just(lambda _: None), st.just(())), + map: st.builds(map, st.just(lambda _: None), st.just(())), + reversed: st.builds(reversed, st.just(())), + classmethod: st.builds(classmethod, st.just(lambda self: self)), + staticmethod: st.builds(staticmethod, st.just(lambda self: self)), + super: st.builds(super, st.from_type(type)), + re.Match: st.text().map(lambda c: re.match(".", c, flags=re.DOTALL)).filter(bool), + re.Pattern: st.builds(re.compile, st.sampled_from(["", b""])), # Pull requests with more types welcome! } if zoneinfo is not None: # pragma: no branch _global_type_lookup[zoneinfo.ZoneInfo] = st.timezones() +if PYPY: + _global_type_lookup[builtins.sequenceiterator] = st.builds(iter, st.tuples()) # type: ignore + _global_type_lookup[type] = st.sampled_from( [type(None)] + sorted(_global_type_lookup, key=str) ) - -if sys.version_info[:2] >= (3, 7): # pragma: no branch - _global_type_lookup[re.Match] = ( - st.text().map(lambda c: re.match(".", c, flags=re.DOTALL)).filter(bool) - ) - _global_type_lookup[re.Pattern] = st.builds(re.compile, st.sampled_from(["", b""])) if sys.version_info[:2] >= (3, 9): # pragma: no cover # subclass of MutableMapping, and in Python 3.9 we resolve to a union # which includes this... but we don't actually ever want to build one. @@ -441,7 +657,7 @@ def _networks(bits): } ) if hasattr(typing, "SupportsIndex"): # pragma: no branch # new in Python 3.8 - _global_type_lookup[typing.SupportsIndex] = st.integers() | st.booleans() # type: ignore + _global_type_lookup[typing.SupportsIndex] = st.integers() | st.booleans() def register(type_, fallback=None, *, module=typing): @@ -477,16 +693,19 @@ def resolve_Type(thing): # This branch is for Python < 3.8, when __args__ was not always tracked return st.just(type) # pragma: no cover args = (thing.__args__[0],) - if getattr(args[0], "__origin__", None) is typing.Union: + if is_a_union(args[0]): args = args[0].__args__ # Duplicate check from from_type here - only paying when needed. - for a in args: # pragma: no cover # only on Python 3.6 - if type(a) == ForwardRef: - raise ResolutionFailed( - "thing=%s cannot be resolved. Upgrading to " - "python>=3.6 may fix this problem via improvements " - "to the typing module." % (thing,) - ) + args = list(args) + for i, a in enumerate(args): + if type(a) == typing.ForwardRef: + try: + args[i] = getattr(builtins, a.__forward_arg__) + except AttributeError: + raise ResolutionFailed( + f"Cannot find the type referenced by {thing} - try using " + f"st.register_type_strategy({thing}, st.from_type(...))" + ) from None return st.sampled_from(sorted(args, key=type_sorting_key)) @@ -643,13 +862,33 @@ def resolve_Callable(thing): # use of keyword arguments and we'd rather not force positional-only. if not thing.__args__: # pragma: no cover # varies by minor version return st.functions() + + *args_types, return_type = thing.__args__ + # Note that a list can only appear in __args__ under Python 3.9 with the # collections.abc version; see https://bugs.python.org/issue42195 + if len(args_types) == 1 and isinstance(args_types[0], list): + args_types = tuple(args_types[0]) + + pep612 = ConcatenateTypes + ParamSpecTypes + for arg in args_types: + # awkward dance because you can't use Concatenate in isistance or issubclass + if getattr(arg, "__origin__", arg) in pep612 or type(arg) in pep612: + raise InvalidArgument( + "Hypothesis can't yet construct a strategy for instances of a " + f"Callable type parametrized by {arg!r}. Consider using an " + "explicit strategy, or opening an issue." + ) + if getattr(return_type, "__origin__", None) in TypeGuardTypes: + raise InvalidArgument( + "Hypothesis cannot yet construct a strategy for callables which " + f"are PEP-647 TypeGuards (got {return_type!r}). " + "Consider using an explicit strategy, or opening an issue." + ) + return st.functions( - like=(lambda: None) - if len(thing.__args__) == 1 or thing.__args__[0] == [] - else (lambda *a, **k: None), - returns=st.from_type(thing.__args__[-1]), + like=(lambda *a, **k: None) if args_types else (lambda: None), + returns=st.from_type(return_type), ) @@ -659,7 +898,7 @@ def resolve_TypeVar(thing): if getattr(thing, "__bound__", None) is not None: bound = thing.__bound__ - if isinstance(bound, ForwardRef): + if isinstance(bound, typing.ForwardRef): bound = _try_import_forward_ref(thing, bound) strat = unwrap_strategies(st.from_type(bound)) if not isinstance(strat, OneOfStrategy): diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/utils.py b/hypothesis-python/src/hypothesis/strategies/_internal/utils.py new file mode 100644 index 0000000000..b0e0746314 --- /dev/null +++ b/hypothesis-python/src/hypothesis/strategies/_internal/utils.py @@ -0,0 +1,146 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import threading +from inspect import signature +from typing import TYPE_CHECKING, Callable, Dict + +from hypothesis.internal.cache import LRUReusedCache +from hypothesis.internal.floats import float_to_int +from hypothesis.internal.reflection import proxies + +if TYPE_CHECKING: + from hypothesis.strategies._internal.strategies import SearchStrategy, T + +_strategies: Dict[str, Callable[..., "SearchStrategy"]] = {} + + +class FloatKey: + def __init__(self, f): + self.value = float_to_int(f) + + def __eq__(self, other): + return isinstance(other, FloatKey) and (other.value == self.value) + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.value) + + +def convert_value(v): + if isinstance(v, float): + return FloatKey(v) + return (type(v), v) + + +_CACHE = threading.local() + + +def get_cache() -> LRUReusedCache: + try: + return _CACHE.STRATEGY_CACHE + except AttributeError: + _CACHE.STRATEGY_CACHE = LRUReusedCache(1024) + return _CACHE.STRATEGY_CACHE + + +def clear_cache() -> None: + cache = get_cache() + cache.clear() + + +def cacheable(fn: "T") -> "T": + from hypothesis.strategies._internal.strategies import SearchStrategy + + @proxies(fn) + def cached_strategy(*args, **kwargs): + try: + kwargs_cache_key = {(k, convert_value(v)) for k, v in kwargs.items()} + except TypeError: + return fn(*args, **kwargs) + cache_key = (fn, tuple(map(convert_value, args)), frozenset(kwargs_cache_key)) + cache = get_cache() + try: + if cache_key in cache: + return cache[cache_key] + except TypeError: + return fn(*args, **kwargs) + else: + result = fn(*args, **kwargs) + if not isinstance(result, SearchStrategy) or result.is_cacheable: + cache[cache_key] = result + return result + + cached_strategy.__clear_cache = clear_cache # type: ignore + return cached_strategy + + +def defines_strategy( + *, + force_reusable_values: bool = False, + try_non_lazy: bool = False, + never_lazy: bool = False, +) -> Callable[["T"], "T"]: + """Returns a decorator for strategy functions. + + If ``force_reusable_values`` is True, the returned strategy will be marked + with ``.has_reusable_values == True`` even if it uses maps/filters or + non-reusable strategies internally. This tells our numpy/pandas strategies + that they can implicitly use such strategies as background values. + + If ``try_non_lazy`` is True, attempt to execute the strategy definition + function immediately, so that a LazyStrategy is only returned if this + raises an exception. + + If ``never_lazy`` is True, the decorator performs no lazy-wrapping at all, + and instead returns the original function. + """ + + def decorator(strategy_definition): + """A decorator that registers the function as a strategy and makes it + lazily evaluated.""" + _strategies[strategy_definition.__name__] = signature(strategy_definition) + + if never_lazy: + assert not try_non_lazy + # We could potentially support never_lazy + force_reusable_values + # with a suitable wrapper, but currently there are no callers that + # request this combination. + assert not force_reusable_values + return strategy_definition + + from hypothesis.strategies._internal.lazy import LazyStrategy + + @proxies(strategy_definition) + def accept(*args, **kwargs): + if try_non_lazy: + # Why not try this unconditionally? Because we'd end up with very + # deep nesting of recursive strategies - better to be lazy unless we + # *know* that eager evaluation is the right choice. + try: + return strategy_definition(*args, **kwargs) + except Exception: + # If invoking the strategy definition raises an exception, + # wrap that up in a LazyStrategy so it happens again later. + pass + result = LazyStrategy(strategy_definition, args, kwargs) + if force_reusable_values: + # Setting `force_has_reusable_values` here causes the recursive + # property code to set `.has_reusable_values == True`. + result.force_has_reusable_values = True + assert result.has_reusable_values + return result + + accept.is_hypothesis_strategy_function = True + return accept + + return decorator diff --git a/hypothesis-python/src/hypothesis/utils/__init__.py b/hypothesis-python/src/hypothesis/utils/__init__.py index 60ea6de55e..ad785ffccd 100644 --- a/hypothesis-python/src/hypothesis/utils/__init__.py +++ b/hypothesis-python/src/hypothesis/utils/__init__.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """hypothesis.utils is a package for things that you can consider part of the semi-public Hypothesis API but aren't really the core point.""" diff --git a/hypothesis-python/src/hypothesis/utils/conventions.py b/hypothesis-python/src/hypothesis/utils/conventions.py index 1b591d9845..ec01326b49 100644 --- a/hypothesis-python/src/hypothesis/utils/conventions.py +++ b/hypothesis-python/src/hypothesis/utils/conventions.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER class UniqueIdentifier: @@ -24,9 +19,5 @@ def __repr__(self): return self.identifier -class InferType(UniqueIdentifier): - """We have a subclass for `infer` so we can type-hint public APIs.""" - - -infer = InferType("infer") +infer = ... not_set = UniqueIdentifier("not_set") diff --git a/hypothesis-python/src/hypothesis/utils/dynamicvariables.py b/hypothesis-python/src/hypothesis/utils/dynamicvariables.py index d976391681..ad00cc8b48 100644 --- a/hypothesis-python/src/hypothesis/utils/dynamicvariables.py +++ b/hypothesis-python/src/hypothesis/utils/dynamicvariables.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import threading from contextlib import contextmanager diff --git a/hypothesis-python/src/hypothesis/utils/terminal.py b/hypothesis-python/src/hypothesis/utils/terminal.py new file mode 100644 index 0000000000..6a6f410d6a --- /dev/null +++ b/hypothesis-python/src/hypothesis/utils/terminal.py @@ -0,0 +1,36 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import os + + +def guess_background_color(): + """Returns one of "dark", "light", or "unknown". + + This is basically just guessing, but better than always guessing "dark"! + See also https://stackoverflow.com/questions/2507337/ and + https://unix.stackexchange.com/questions/245378/ + """ + django_colors = os.getenv("DJANGO_COLORS") + if django_colors in ("light", "dark"): + return django_colors + # Guessing based on the $COLORFGBG environment variable + try: + fg, *_, bg = os.getenv("COLORFGBG").split(";") + except Exception: + pass + else: + # 0=black, 7=light-grey, 15=white ; we don't interpret other colors + if fg in ("7", "15") and bg == "0": + return "dark" + elif fg == "0" and bg in ("7", "15"): + return "light" + # TODO: Guessing based on the xterm control sequence + return "unknown" diff --git a/hypothesis-python/src/hypothesis/vendor/__init__.py b/hypothesis-python/src/hypothesis/vendor/__init__.py index e61e3c2872..fcb1ac6538 100644 --- a/hypothesis-python/src/hypothesis/vendor/__init__.py +++ b/hypothesis-python/src/hypothesis/vendor/__init__.py @@ -1,14 +1,9 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER diff --git a/hypothesis-python/src/hypothesis/vendor/pretty.py b/hypothesis-python/src/hypothesis/vendor/pretty.py index bda9eac072..2f52ecf642 100644 --- a/hypothesis-python/src/hypothesis/vendor/pretty.py +++ b/hypothesis-python/src/hypothesis/vendor/pretty.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """ Python advanced pretty printer. This pretty printer is intended to @@ -20,9 +15,6 @@ This module is based on ruby's `prettyprint.rb` library by `Tanaka Akira`. Example Usage ------------- -To directly print the representation of an object use `pprint`:: - from pretty import pprint - pprint(complex_object) To get a string of the output use `pretty`:: from pretty import pretty string = pretty(complex_object) @@ -73,15 +65,15 @@ def _repr_pretty_(self, p, cycle): import datetime import platform import re -import sys +import struct import types from collections import deque from contextlib import contextmanager from io import StringIO +from math import copysign, isnan __all__ = [ "pretty", - "pprint", "PrettyPrinter", "RepresentationPrinter", "for_type_by_name", @@ -120,19 +112,6 @@ def pretty( return stream.getvalue() -def pprint( - obj, verbose=False, max_width=79, newline="\n", max_seq_length=MAX_SEQ_LENGTH -): - """Like `pretty` but print to stdout.""" - printer = RepresentationPrinter( - sys.stdout, verbose, max_width, newline, max_seq_length=max_seq_length - ) - printer.pretty(obj) - printer.flush() - sys.stdout.write(newline) - sys.stdout.flush() - - class _PrettyPrinterBase: @contextmanager def indent(self, indent): @@ -179,6 +158,8 @@ def __init__( self.group_queue = GroupQueue(root_group) self.indentation = 0 + self.snans = 0 + def _break_outer_groups(self): while self.max_width < self.output_width + self.buffer_width: group = self.group_queue.deq() @@ -284,6 +265,12 @@ def end_group(self, dedent=0, close=""): def flush(self): """Flush data that is left in the buffer.""" + if self.snans: + # Reset self.snans *before* calling breakable(), which might flush() + snans = self.snans + self.snans = 0 + self.breakable(" ") + self.text(f"# Saw {snans} signaling NaN" + "s" * (snans > 1)) for data in self.buffer: self.output_width += data.output(self.output, self.output_width) self.buffer.clear() @@ -333,9 +320,7 @@ def __init__( max_seq_length=MAX_SEQ_LENGTH, ): - PrettyPrinter.__init__( - self, output, max_width, newline, max_seq_length=max_seq_length - ) + super().__init__(output, max_width, newline, max_seq_length=max_seq_length) self.verbose = verbose self.stack = [] if singleton_pprinters is None: @@ -500,7 +485,7 @@ def _default_pprint(obj, p, cycle): return p.begin_group(1, "<") p.pretty(klass) - p.text(" at 0x%x" % id(obj)) + p.text(f" at 0x{id(obj):x}") if cycle: p.text(" ...") elif p.verbose: @@ -576,7 +561,7 @@ def inner(obj, p, cycle): if cycle: return p.text(start + "..." + end) - if len(obj) == 0: + if not obj: # Special case. p.text(basetype.__name__ + "()") else: @@ -618,16 +603,7 @@ def inner(obj, p, cycle): if cycle: return p.text("{...}") p.begin_group(1, start) - keys = obj.keys() - # if dict isn't large enough to be truncated, sort keys before - # displaying - if not (p.max_seq_length and len(obj) >= p.max_seq_length): - try: - keys = sorted(keys) - except Exception: - # Sometimes the keys don't sort. - pass - for idx, key in p._enumerate(keys): + for idx, key in p._enumerate(obj): if idx: p.text(",") p.breakable() @@ -684,7 +660,7 @@ def _re_pattern_pprint(obj, p, cycle): "VERBOSE", "DEBUG", ): - if obj.flags & getattr(re, flag): + if obj.flags & getattr(re, flag, 0): if done_one: p.text("|") p.text("re." + flag) @@ -733,11 +709,9 @@ def _repr_pprint(obj, p, cycle): def _function_pprint(obj, p, cycle): """Base pprint for all functions and builtin functions.""" - name = _safe_getattr(obj, "__qualname__", obj.__name__) - mod = obj.__module__ - if mod and mod not in ("__builtin__", "builtins", "exceptions"): - name = mod + "." + name - p.text("" % name) + from hypothesis.internal.reflection import get_pretty_function_description + + p.text(get_pretty_function_description(obj)) def _exception_pprint(obj, p, cycle): @@ -755,10 +729,20 @@ def _exception_pprint(obj, p, cycle): p.end_group(step, ")") +def _repr_float_counting_nans(obj, p, cycle): + if isnan(obj) and hasattr(p, "snans"): + if struct.pack("!d", abs(obj)) != struct.pack("!d", float("nan")): + p.snans += 1 + if copysign(1.0, obj) == -1.0: + p.text("-nan") + return + p.text(repr(obj)) + + #: printers for builtin types _type_pprinters = { int: _repr_pprint, - float: _repr_pprint, + float: _repr_float_counting_nans, str: _repr_pprint, tuple: _seq_pprinter_factory("(", ")", tuple), list: _seq_pprinter_factory("[", "]", list), @@ -815,7 +799,7 @@ def _ordereddict_pprint(obj, p, cycle): with p.group(len(name) + 1, name + "(", ")"): if cycle: p.text("...") - elif len(obj): + elif obj: p.pretty(list(obj.items())) @@ -833,18 +817,20 @@ def _counter_pprint(obj, p, cycle): with p.group(len(name) + 1, name + "(", ")"): if cycle: p.text("...") - elif len(obj): + elif obj: p.pretty(dict(obj)) +def _repr_dataframe(obj, p, cycle): # pragma: no cover + with p.indent(4): + p.break_() + _repr_pprint(obj, p, cycle) + p.break_() + + for_type_by_name("collections", "defaultdict", _defaultdict_pprint) for_type_by_name("collections", "OrderedDict", _ordereddict_pprint) for_type_by_name("ordereddict", "OrderedDict", _ordereddict_pprint) for_type_by_name("collections", "deque", _deque_pprint) for_type_by_name("collections", "Counter", _counter_pprint) -for_type_by_name("counter", "Counter", _counter_pprint) - -for_type_by_name("_collections", "defaultdict", _defaultdict_pprint) -for_type_by_name("_collections", "OrderedDict", _ordereddict_pprint) -for_type_by_name("_collections", "deque", _deque_pprint) -for_type_by_name("_collections", "Counter", _counter_pprint) +for_type_by_name("pandas.core.frame", "DataFrame", _repr_dataframe) diff --git a/hypothesis-python/src/hypothesis/vendor/tlds-alpha-by-domain.txt b/hypothesis-python/src/hypothesis/vendor/tlds-alpha-by-domain.txt index e0e4747e14..7a0bdfdb64 100644 --- a/hypothesis-python/src/hypothesis/vendor/tlds-alpha-by-domain.txt +++ b/hypothesis-python/src/hypothesis/vendor/tlds-alpha-by-domain.txt @@ -1,4 +1,4 @@ -# Version 2019080400, Last Updated Sun Aug 4 07:07:02 2019 UTC +# Version 2022100800, Last Updated Sat Oct 8 07:07:01 2022 UTC AAA AARP ABARTH @@ -25,7 +25,6 @@ AEG AERO AETNA AF -AFAMILYCOMPANY AFL AFRICA AG @@ -33,7 +32,6 @@ AGAKHAN AGENCY AI AIG -AIGO AIRBUS AIRFORCE AIRTEL @@ -48,6 +46,7 @@ ALLY ALSACE ALSTOM AM +AMAZON AMERICANEXPRESS AMERICANFAMILY AMEX @@ -177,8 +176,6 @@ BROTHER BRUSSELS BS BT -BUDAPEST -BUGATTI BUILD BUILDERS BUSINESS @@ -198,7 +195,6 @@ CALVINKLEIN CAM CAMERA CAMP -CANCERRESEARCH CANON CAPETOWN CAPITAL @@ -210,10 +206,8 @@ CARE CAREER CAREERS CARS -CARTIER CASA CASE -CASEIH CASH CASINO CAT @@ -225,7 +219,6 @@ CBRE CBS CC CD -CEB CENTER CEO CERN @@ -243,7 +236,6 @@ CHEAP CHINTAI CHRISTMAS CHROME -CHRYSLER CHURCH CI CIPRIANI @@ -295,6 +287,7 @@ COUNTRY COUPON COUPONS COURSES +CPA CR CREDIT CREDITCARD @@ -304,7 +297,6 @@ CROWN CRS CRUISE CRUISES -CSC CU CUISINELLA CV @@ -356,7 +348,6 @@ DNP DO DOCS DOCTOR -DODGE DOG DOMAINS DOT @@ -364,9 +355,7 @@ DOWNLOAD DRIVE DTV DUBAI -DUCK DUNLOP -DUNS DUPONT DURBAN DVAG @@ -395,14 +384,12 @@ ERNI ES ESQ ESTATE -ESURANCE ET ETISALAT EU EUROVISION EUS EVENTS -EVERBANK EXCHANGE EXPERT EXPOSED @@ -467,7 +454,6 @@ FRONTDOOR FRONTIER FTR FUJITSU -FUJIXEROX FUN FUND FURNITURE @@ -482,6 +468,7 @@ GAME GAMES GAP GARDEN +GAY GB GBIZ GD @@ -501,7 +488,6 @@ GIFTS GIVES GIVING GL -GLADE GLASS GLE GLOBAL @@ -617,7 +603,6 @@ INSTITUTE INSURANCE INSURE INT -INTEL INTERNATIONAL INTUIT INVESTMENTS @@ -627,18 +612,15 @@ IQ IR IRISH IS -ISELECT ISMAILI IST ISTANBUL IT ITAU ITV -IVECO JAGUAR JAVA JCB -JCP JE JEEP JETZT @@ -669,6 +651,7 @@ KG KH KI KIA +KIDS KIM KINDER KINDLE @@ -692,12 +675,10 @@ KYOTO KZ LA LACAIXA -LADBROKES LAMBORGHINI LAMER LANCASTER LANCIA -LANCOME LAND LANDROVER LANXESS @@ -718,7 +699,6 @@ LEGO LEXUS LGBT LI -LIAISON LIDL LIFE LIFEINSURANCE @@ -734,9 +714,9 @@ LINK LIPSY LIVE LIVING -LIXIL LK LLC +LLP LOAN LOANS LOCKER @@ -756,7 +736,6 @@ LTD LTDA LU LUNDBECK -LUPIN LUXE LUXURY LV @@ -792,7 +771,6 @@ MEMORIAL MEN MENU MERCKMSD -METLIFE MG MH MIAMI @@ -812,7 +790,6 @@ MN MO MOBI MOBILE -MOBILY MODA MOE MOI @@ -820,7 +797,6 @@ MOM MONASH MONEY MONSTER -MOPAR MORMON MORTGAGE MOSCOW @@ -828,7 +804,6 @@ MOTO MOTORCYCLES MOV MOVIE -MOVISTAR MP MQ MR @@ -839,6 +814,7 @@ MTN MTR MU MUSEUM +MUSIC MUTUAL MV MW @@ -847,10 +823,8 @@ MY MZ NA NAB -NADEX NAGOYA NAME -NATIONWIDE NATURA NAVY NBA @@ -863,7 +837,6 @@ NETFLIX NETWORK NEUSTAR NEW -NEWHOLLAND NEWS NEXT NEXTDIRECT @@ -898,7 +871,6 @@ NYC NZ OBI OBSERVER -OFF OFFICE OKINAWA OLAYAN @@ -911,7 +883,6 @@ ONE ONG ONL ONLINE -ONYOURSIDE OOO OPEN ORACLE @@ -948,7 +919,6 @@ PHOTO PHOTOGRAPHY PHOTOS PHYSIO -PIAGET PICS PICTET PICTURES @@ -999,10 +969,8 @@ QA QPON QUEBEC QUEST -QVC RACING RADIO -RAID RE READ REALESTATE @@ -1031,11 +999,9 @@ REXROTH RICH RICHARDLI RICOH -RIGHTATHOME RIL RIO RIP -RMIT RO ROCHER ROCKS @@ -1081,8 +1047,6 @@ SCHOOL SCHULE SCHWARZ SCIENCE -SCJOHNSON -SCOR SCOT SD SE @@ -1114,7 +1078,6 @@ SHOPPING SHOUJI SHOW SHOWTIME -SHRIRAM SI SILK SINA @@ -1144,13 +1107,12 @@ SOLUTIONS SONG SONY SOY +SPA SPACE SPORT SPOT -SPREADBETTING SR SRL -SRT SS ST STADA @@ -1177,12 +1139,10 @@ SURGERY SUZUKI SV SWATCH -SWIFTCOVER SWISS SX SY SYDNEY -SYMANTEC SYSTEMS SZ TAB @@ -1203,7 +1163,6 @@ TEAM TECH TECHNOLOGY TEL -TELEFONICA TEMASEK TENNIS TEVA @@ -1263,7 +1222,6 @@ TZ UA UBANK UBS -UCONNECT UG UK UNICOM @@ -1297,7 +1255,6 @@ VIP VIRGIN VISA VISION -VISTAPRINT VIVA VIVO VLAANDEREN @@ -1316,7 +1273,6 @@ WALMART WALTER WANG WANGGOU -WARMAN WATCH WATCHES WEATHER @@ -1361,12 +1317,12 @@ XN--3BST00M XN--3DS443G XN--3E0B707E XN--3HCRJ9C -XN--3OQ18VL8PN36A XN--3PXU8K XN--42C2D9A XN--45BR5CYL XN--45BRJ9C XN--45Q11C +XN--4DBRK0CE XN--4GBRIM XN--54B7FTA0CC XN--55QW42G @@ -1392,6 +1348,7 @@ XN--BCK1B9A5DRE4C XN--C1AVG XN--C2BR7G XN--CCK2B3B +XN--CCKWCXETD XN--CG4BKI XN--CLCHC0EA0B2G2A9GCD XN--CZR694B @@ -1402,7 +1359,6 @@ XN--D1ALF XN--E1A4C XN--ECKVDTC9D XN--EFVY88H -XN--ESTV75G XN--FCT429K XN--FHBEI XN--FIQ228C5HS @@ -1428,12 +1384,12 @@ XN--IO0A7I XN--J1AEF XN--J1AMH XN--J6W193G +XN--JLQ480N2RG XN--JLQ61U9W7B XN--JVR189M XN--KCRX77D1X4A XN--KPRW13D XN--KPRY57D -XN--KPU716F XN--KPUT3I XN--L1ACC XN--LGBBAT1AD8J @@ -1447,11 +1403,11 @@ XN--MGBAB2BD XN--MGBAH1A3HJKRD XN--MGBAI9AZGQP6J XN--MGBAYH7GPA -XN--MGBB9FBPOB XN--MGBBH1A XN--MGBBH1A71E XN--MGBC0A9AZCG XN--MGBCA7DZDO +XN--MGBCPQ6GPA1A XN--MGBERP4A5D4AR XN--MGBGU82A XN--MGBI4ECEXP @@ -1474,11 +1430,12 @@ XN--OGBPF8FL XN--OTU796D XN--P1ACF XN--P1AI -XN--PBT977C XN--PGBS0DH XN--PSSY2U +XN--Q7CE6A XN--Q9JYB4C XN--QCKA1PMC +XN--QXA6A XN--QXAM XN--RHQV96G XN--ROVU88B diff --git a/hypothesis-python/src/hypothesis/version.py b/hypothesis-python/src/hypothesis/version.py index de9034bfe3..80cdac2103 100644 --- a/hypothesis-python/src/hypothesis/version.py +++ b/hypothesis-python/src/hypothesis/version.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER -__version_info__ = (6, 3, 0) +__version_info__ = (6, 58, 0) __version__ = ".".join(map(str, __version_info__)) diff --git a/hypothesis-python/tests/__init__.py b/hypothesis-python/tests/__init__.py index e61e3c2872..fcb1ac6538 100644 --- a/hypothesis-python/tests/__init__.py +++ b/hypothesis-python/tests/__init__.py @@ -1,14 +1,9 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER diff --git a/hypothesis-python/tests/array_api/README.md b/hypothesis-python/tests/array_api/README.md new file mode 100644 index 0000000000..bc2f4d1aa2 --- /dev/null +++ b/hypothesis-python/tests/array_api/README.md @@ -0,0 +1,50 @@ +This folder contains tests for `hypothesis.extra.array_api`. + +## Mocked array module + +A mock of the Array API namespace exists as `mock_xp` in `extra.array_api`. This +wraps NumPy-proper to conform it to the *draft* spec, where `numpy.array_api` +might not. This is not a fully compliant wrapper, but conforms enough for the +purposes of testing. + +## Running against different array modules + +You can test other array modules which adopt the Array API via the +`HYPOTHESIS_TEST_ARRAY_API` environment variable. There are two recognized +options: + +* `"default"`: uses the mock. +* `"all"`: uses all array modules found via entry points, _and_ the mock. + +If neither of these, the test suite will then try resolve the variable like so: + +1. If the variable matches a name of an available entry point, load said entry point. +2. If the variables matches a valid import path, import said path. + +For example, to specify NumPy's Array API implementation, you could use its +entry point (**1.**), + + HYPOTHESIS_TEST_ARRAY_API=numpy pytest tests/array_api + +or use the import path (**2.**), + + HYPOTHESIS_TEST_ARRAY_API=numpy.array_api pytest tests/array_api + +The former method is more ergonomic, but as entry points are optional for +adopting the Array API, you will need to use the latter method for libraries +that opt-out. + +## Running against different API versions + +You can specify the `api_version` to use when testing array modules via the +`HYPOTHESIS_TEST_ARRAY_API_VERSION` environment variable. There is one +recognized option: + +* `"default"`: infers the latest API version for each array module. + +Otherwise the test suite will use the variable as the `api_version` argument for +`make_strategies_namespace()`. + +In the future we intend to support running tests against multiple API versioned +namespaces, likely with an additional recognized option that infers all +supported versions. diff --git a/hypothesis-python/tests/array_api/__init__.py b/hypothesis-python/tests/array_api/__init__.py new file mode 100644 index 0000000000..fcb1ac6538 --- /dev/null +++ b/hypothesis-python/tests/array_api/__init__.py @@ -0,0 +1,9 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. diff --git a/hypothesis-python/tests/array_api/common.py b/hypothesis-python/tests/array_api/common.py new file mode 100644 index 0000000000..cf706d970a --- /dev/null +++ b/hypothesis-python/tests/array_api/common.py @@ -0,0 +1,72 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +from importlib.metadata import EntryPoint, entry_points # type: ignore +from typing import Dict + +import pytest + +from hypothesis.extra.array_api import ( + COMPLEX_NAMES, + REAL_NAMES, + RELEASED_VERSIONS, + NominalVersion, +) +from hypothesis.internal.floats import next_up + +__all__ = [ + "MIN_VER_FOR_COMPLEX:", + "installed_array_modules", + "flushes_to_zero", + "dtype_name_params", +] + + +# This should be updated to the next spec release, which should include complex numbers +MIN_VER_FOR_COMPLEX: NominalVersion = "draft" +if len(RELEASED_VERSIONS) > 1: + assert MIN_VER_FOR_COMPLEX == RELEASED_VERSIONS[1] + + +def installed_array_modules() -> Dict[str, EntryPoint]: + """Returns a dictionary of array module names paired to their entry points + + A convenience wrapper for importlib.metadata.entry_points(). It has the + added benefit of working with both the original dict interface and the new + select interface, so this can be used warning-free in all modern Python + versions. + """ + try: + eps = entry_points(group="array_api") + except TypeError: + # The select interface for entry_points was introduced in py3.10, + # supplanting its dict interface. We fallback to the dict interface so + # we can still find entry points in py3.8 and py3.9. + eps = entry_points().get("array_api", []) + return {ep.name: ep for ep in eps} + + +def flushes_to_zero(xp, width: int) -> bool: + """Infer whether build of array module has its float dtype of the specified + width flush subnormals to zero + + We do this per-width because compilers might FTZ for one dtype but allow + subnormals in the other. + """ + if width not in [32, 64]: + raise ValueError(f"{width=}, but should be either 32 or 64") + dtype = getattr(xp, f"float{width}") + return bool(xp.asarray(next_up(0.0, width=width), dtype=dtype) == 0) + + +dtype_name_params = ["bool"] + list(REAL_NAMES) +for name in COMPLEX_NAMES: + param = pytest.param(name, marks=pytest.mark.xp_min_version(MIN_VER_FOR_COMPLEX)) + dtype_name_params.append(param) diff --git a/hypothesis-python/tests/array_api/conftest.py b/hypothesis-python/tests/array_api/conftest.py new file mode 100644 index 0000000000..7ee6a1f4eb --- /dev/null +++ b/hypothesis-python/tests/array_api/conftest.py @@ -0,0 +1,119 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import warnings +from importlib import import_module +from os import getenv +from types import ModuleType, SimpleNamespace +from typing import Tuple + +import pytest + +from hypothesis.errors import HypothesisWarning, InvalidArgument +from hypothesis.extra.array_api import ( + NOMINAL_VERSIONS, + NominalVersion, + make_strategies_namespace, + mock_xp, +) + +from tests.array_api.common import installed_array_modules + +# See README.md in regards to the env variables +test_xp_option = getenv("HYPOTHESIS_TEST_ARRAY_API", "default") + +test_version_option = getenv("HYPOTHESIS_TEST_ARRAY_API_VERSION", "default") +if test_version_option != "default" and test_version_option not in NOMINAL_VERSIONS: + raise ValueError( + f"HYPOTHESIS_TEST_ARRAY_API_VERSION='{test_version_option}' is not " + f"'default' or a valid api_version {NOMINAL_VERSIONS}." + ) +with pytest.warns(HypothesisWarning): + mock_version = "draft" if test_version_option == "default" else test_version_option + mock_xps = make_strategies_namespace(mock_xp, api_version=mock_version) +api_version = None if test_version_option == "default" else test_version_option + + +class InvalidArgumentWarning(UserWarning): + """Custom warning so we can bypass our global capturing""" + + +name_to_entry_point = installed_array_modules() +xp_and_xps_pairs: Tuple[ModuleType, SimpleNamespace] = [] +with warnings.catch_warnings(): + # We ignore all warnings here as many array modules warn on import. Ideally + # we would just ignore ImportWarning, but no one seems to use it! + warnings.simplefilter("ignore") + warnings.simplefilter("default", category=InvalidArgumentWarning) + # We go through the steps described in README.md to define `xp_xps_pairs`, + # which contains the array module(s) to be run against the test suite, along + # with their respective strategy namespaces. + if test_xp_option == "default": + xp_and_xps_pairs = [(mock_xp, mock_xps)] + elif test_xp_option == "all": + if len(name_to_entry_point) == 0: + raise ValueError( + "HYPOTHESIS_TEST_ARRAY_API='all', but no entry points where found" + ) + xp_and_xps_pairs = [(mock_xp, mock_xps)] + for name, ep in name_to_entry_point.items(): + xp = ep.load() + try: + xps = make_strategies_namespace(xp, api_version=api_version) + except InvalidArgument as e: + warnings.warn(str(e), InvalidArgumentWarning) + else: + xp_and_xps_pairs.append((xp, xps)) + elif test_xp_option in name_to_entry_point.keys(): + ep = name_to_entry_point[test_xp_option] + xp = ep.load() + xps = make_strategies_namespace(xp, api_version=api_version) + xp_and_xps_pairs = [(xp, xps)] + else: + try: + xp = import_module(test_xp_option) + except ImportError as e: + raise ValueError( + f"HYPOTHESIS_TEST_ARRAY_API='{test_xp_option}' is not a valid " + "option ('default' or 'all'), name of an available entry point, " + "or a valid import path." + ) from e + else: + xps = make_strategies_namespace(xp, api_version=api_version) + xp_and_xps_pairs = [(xp, xps)] + + +def pytest_generate_tests(metafunc): + xp_params = [] + xp_and_xps_params = [] + for xp, xps in xp_and_xps_pairs: + xp_params.append(pytest.param(xp, id=xp.__name__)) + xp_and_xps_params.append( + pytest.param(xp, xps, id=f"{xp.__name__}-{xps.api_version}") + ) + if "xp" in metafunc.fixturenames: + if "xps" in metafunc.fixturenames: + metafunc.parametrize("xp, xps", xp_and_xps_params) + else: + metafunc.parametrize("xp", xp_params) + + +def pytest_collection_modifyitems(config, items): + for item in items: + if "xps" in item.fixturenames: + markers = [m for m in item.own_markers if m.name == "xp_min_version"] + if markers: + assert len(markers) == 1 # sanity check + min_version: NominalVersion = markers[0].args[0] + xps_version: NominalVersion = item.callspec.params["xps"].api_version + if xps_version < min_version: + item.add_marker( + pytest.mark.skip(reason=f"requires api_version=>{min_version}") + ) diff --git a/hypothesis-python/tests/array_api/test_argument_validation.py b/hypothesis-python/tests/array_api/test_argument_validation.py new file mode 100644 index 0000000000..3a14ee9987 --- /dev/null +++ b/hypothesis-python/tests/array_api/test_argument_validation.py @@ -0,0 +1,235 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +from typing import Optional + +import pytest + +from hypothesis.errors import InvalidArgument +from hypothesis.extra.array_api import NominalVersion, make_strategies_namespace + +from tests.array_api.common import MIN_VER_FOR_COMPLEX + + +def e(name, *, _min_version: Optional[NominalVersion] = None, **kwargs): + kw = ", ".join(f"{k}={v!r}" for k, v in kwargs.items()) + id_ = f"{name}({kw})" + if _min_version is None: + marks = () + else: + marks = pytest.mark.xp_min_version(_min_version) + return pytest.param(name, kwargs, id=id_, marks=marks) + + +@pytest.mark.parametrize( + ("strat_name", "kwargs"), + [ + e("arrays", dtype=1, shape=5), + e("arrays", dtype=None, shape=5), + e("arrays", dtype="int8", shape=(0.5,)), + e("arrays", dtype="int8", shape=1, fill=3), + e("arrays", dtype="int8", shape=1, elements="not a strategy"), + e("arrays", dtype="int8", shape="not a shape or strategy"), + e("array_shapes", min_side=2, max_side=1), + e("array_shapes", min_dims=3, max_dims=2), + e("array_shapes", min_dims=-1), + e("array_shapes", min_side=-1), + e("array_shapes", min_side="not an int"), + e("array_shapes", max_side="not an int"), + e("array_shapes", min_dims="not an int"), + e("array_shapes", max_dims="not an int"), + e("from_dtype", dtype=1), + e("from_dtype", dtype=None), + e("from_dtype", dtype="int8", min_value="not an int"), + e("from_dtype", dtype="int8", max_value="not an int"), + e("from_dtype", dtype="float32", min_value="not a float"), + e("from_dtype", dtype="float32", max_value="not a float"), + e("from_dtype", dtype="int8", min_value=10, max_value=5), + e("from_dtype", dtype="float32", min_value=10, max_value=5), + e("from_dtype", dtype="int8", min_value=-999), + e("from_dtype", dtype="int8", max_value=-999), + e("from_dtype", dtype="int8", min_value=999), + e("from_dtype", dtype="int8", max_value=999), + e("from_dtype", dtype="uint8", min_value=-999), + e("from_dtype", dtype="uint8", max_value=-999), + e("from_dtype", dtype="uint8", min_value=999), + e("from_dtype", dtype="uint8", max_value=999), + e("from_dtype", dtype="float32", min_value=-4e38), + e("from_dtype", dtype="float32", max_value=-4e38), + e("from_dtype", dtype="float32", min_value=4e38), + e("from_dtype", dtype="float32", max_value=4e38), + e("integer_dtypes", sizes=()), + e("integer_dtypes", sizes=(3,)), + e("unsigned_integer_dtypes", sizes=()), + e("unsigned_integer_dtypes", sizes=(3,)), + e("floating_dtypes", sizes=()), + e("floating_dtypes", sizes=(3,)), + e("complex_dtypes", _min_version=MIN_VER_FOR_COMPLEX, sizes=()), + e("complex_dtypes", _min_version=MIN_VER_FOR_COMPLEX, sizes=(3,)), + e("valid_tuple_axes", ndim=-1), + e("valid_tuple_axes", ndim=2, min_size=-1), + e("valid_tuple_axes", ndim=2, min_size=3, max_size=10), + e("valid_tuple_axes", ndim=2, min_size=2, max_size=1), + e("valid_tuple_axes", ndim=2.0, min_size=2, max_size=1), + e("valid_tuple_axes", ndim=2, min_size=1.0, max_size=2), + e("valid_tuple_axes", ndim=2, min_size=1, max_size=2.0), + e("valid_tuple_axes", ndim=2, min_size=1, max_size=3), + e("broadcastable_shapes", shape="a"), + e("broadcastable_shapes", shape=(2, 2), min_side="a"), + e("broadcastable_shapes", shape=(2, 2), min_dims="a"), + e("broadcastable_shapes", shape=(2, 2), max_side="a"), + e("broadcastable_shapes", shape=(2, 2), max_dims="a"), + e("broadcastable_shapes", shape=(2, 2), min_side=-1), + e("broadcastable_shapes", shape=(2, 2), min_dims=-1), + e("broadcastable_shapes", shape=(2, 2), min_side=1, max_side=0), + e("broadcastable_shapes", shape=(2, 2), min_dims=1, max_dims=0), + e( + "broadcastable_shapes", # max_side too small + shape=(5, 1), + min_dims=2, + max_dims=4, + min_side=2, + max_side=3, + ), + e( + "broadcastable_shapes", # min_side too large + shape=(0, 1), + min_dims=2, + max_dims=4, + min_side=2, + max_side=3, + ), + e( + "broadcastable_shapes", # default max_dims unsatisfiable + shape=(5, 3, 2, 1), + min_dims=3, + max_dims=None, + min_side=2, + max_side=3, + ), + e( + "broadcastable_shapes", # default max_dims unsatisfiable + shape=(0, 3, 2, 1), + min_dims=3, + max_dims=None, + min_side=2, + max_side=3, + ), + e("mutually_broadcastable_shapes", num_shapes=0), + e("mutually_broadcastable_shapes", num_shapes="a"), + e("mutually_broadcastable_shapes", num_shapes=2, base_shape="a"), + e( + "mutually_broadcastable_shapes", # min_side is invalid type + num_shapes=2, + min_side="a", + ), + e( + "mutually_broadcastable_shapes", # min_dims is invalid type + num_shapes=2, + min_dims="a", + ), + e( + "mutually_broadcastable_shapes", # max_side is invalid type + num_shapes=2, + max_side="a", + ), + e( + "mutually_broadcastable_shapes", # max_side is invalid type + num_shapes=2, + max_dims="a", + ), + e( + "mutually_broadcastable_shapes", # min_side is out of domain + num_shapes=2, + min_side=-1, + ), + e( + "mutually_broadcastable_shapes", # min_dims is out of domain + num_shapes=2, + min_dims=-1, + ), + e( + "mutually_broadcastable_shapes", # max_side < min_side + num_shapes=2, + min_side=1, + max_side=0, + ), + e( + "mutually_broadcastable_shapes", # max_dims < min_dims + num_shapes=2, + min_dims=1, + max_dims=0, + ), + e( + "mutually_broadcastable_shapes", # max_side too small + num_shapes=2, + base_shape=(5, 1), + min_dims=2, + max_dims=4, + min_side=2, + max_side=3, + ), + e( + "mutually_broadcastable_shapes", # min_side too large + num_shapes=2, + base_shape=(0, 1), + min_dims=2, + max_dims=4, + min_side=2, + max_side=3, + ), + e( + "mutually_broadcastable_shapes", # user-specified max_dims unsatisfiable + num_shapes=1, + base_shape=(5, 3, 2, 1), + min_dims=3, + max_dims=4, + min_side=2, + max_side=3, + ), + e( + "mutually_broadcastable_shapes", # user-specified max_dims unsatisfiable + num_shapes=2, + base_shape=(0, 3, 2, 1), + min_dims=3, + max_dims=4, + min_side=2, + max_side=3, + ), + e("indices", shape=0), + e("indices", shape=("1", "2")), + e("indices", shape=(0, -1)), + e("indices", shape=(0, 0), allow_newaxis=None), + e("indices", shape=(0, 0), allow_ellipsis=None), + e("indices", shape=(0, 0), min_dims=-1), + e("indices", shape=(0, 0), min_dims=1.0), + e("indices", shape=(0, 0), max_dims=-1), + e("indices", shape=(0, 0), max_dims=1.0), + e("indices", shape=(0, 0), min_dims=2, max_dims=1), + e("indices", shape=(3, 3, 3), min_dims=4), + e("indices", shape=(3, 3, 3), max_dims=5), + e("indices", shape=5, min_dims=0), + e("indices", shape=(5,), min_dims=2), + e("indices", shape=(5,), max_dims=2), + ], +) +def test_raise_invalid_argument(xp, xps, strat_name, kwargs): + """Strategies raise helpful error with invalid arguments.""" + strat_func = getattr(xps, strat_name) + strat = strat_func(**kwargs) + with pytest.raises(InvalidArgument): + strat.example() + + +@pytest.mark.parametrize("api_version", [..., "latest", "1970.01", 42]) +def test_make_strategies_namespace_raise_invalid_argument(xp, api_version): + """Function raises helpful error with invalid arguments.""" + with pytest.raises(InvalidArgument): + make_strategies_namespace(xp, api_version=api_version) diff --git a/hypothesis-python/tests/array_api/test_arrays.py b/hypothesis-python/tests/array_api/test_arrays.py new file mode 100644 index 0000000000..b36520688d --- /dev/null +++ b/hypothesis-python/tests/array_api/test_arrays.py @@ -0,0 +1,511 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import pytest + +from hypothesis import given, strategies as st +from hypothesis.errors import InvalidArgument +from hypothesis.extra.array_api import COMPLEX_NAMES, REAL_NAMES +from hypothesis.internal.floats import width_smallest_normals + +from tests.array_api.common import ( + MIN_VER_FOR_COMPLEX, + dtype_name_params, + flushes_to_zero, +) +from tests.common.debug import assert_all_examples, find_any, minimal +from tests.common.utils import flaky + + +def skip_on_missing_unique_values(xp): + if not hasattr(xp, "unique_values"): + pytest.mark.skip("xp.unique_values() is not required to exist") + + +def xfail_on_indistinct_nans(xp): + """ + xp.unique_value() should return distinct NaNs - if not, tests that (rightly) + assume such behaviour will likely fail. For example, NumPy 1.22 treats NaNs + as indistinct, so tests that use this function will be marked as xfail. + See https://mail.python.org/pipermail/numpy-discussion/2021-August/081995.html + """ + skip_on_missing_unique_values(xp) + two_nans = xp.asarray([float("nan"), float("nan")]) + if xp.unique_values(two_nans).size != 2: + pytest.xfail("NaNs not distinct") + + +@pytest.mark.parametrize("dtype_name", dtype_name_params) +def test_draw_arrays_from_dtype(xp, xps, dtype_name): + """Draw arrays from dtypes.""" + dtype = getattr(xp, dtype_name) + assert_all_examples(xps.arrays(dtype, ()), lambda x: x.dtype == dtype) + + +@pytest.mark.parametrize("dtype_name", dtype_name_params) +def test_draw_arrays_from_scalar_names(xp, xps, dtype_name): + """Draw arrays from dtype names.""" + dtype = getattr(xp, dtype_name) + assert_all_examples(xps.arrays(dtype_name, ()), lambda x: x.dtype == dtype) + + +@given(data=st.data()) +def test_draw_arrays_from_shapes(xp, xps, data): + """Draw arrays from shapes.""" + shape = data.draw(xps.array_shapes()) + x = data.draw(xps.arrays(xp.int8, shape)) + assert x.ndim == len(shape) + assert x.shape == shape + + +@given(data=st.data()) +def test_draw_arrays_from_int_shapes(xp, xps, data): + """Draw arrays from integers as shapes.""" + size = data.draw(st.integers(0, 10)) + x = data.draw(xps.arrays(xp.int8, size)) + assert x.shape == (size,) + + +@pytest.mark.parametrize( + "strat_name", + [ + "scalar_dtypes", + "boolean_dtypes", + "integer_dtypes", + "unsigned_integer_dtypes", + "floating_dtypes", + "real_dtypes", + pytest.param( + "complex_dtypes", marks=pytest.mark.xp_min_version(MIN_VER_FOR_COMPLEX) + ), + ], +) +def test_draw_arrays_from_dtype_strategies(xp, xps, strat_name): + """Draw arrays from dtype strategies.""" + strat_func = getattr(xps, strat_name) + strat = strat_func() + find_any(xps.arrays(strat, ())) + + +@given(data=st.data()) +def test_draw_arrays_from_dtype_name_strategies(xp, xps, data): + """Draw arrays from dtype name strategies.""" + all_names = ("bool",) + REAL_NAMES + if xps.api_version > "2021.12": + all_names += COMPLEX_NAMES + sample_names = data.draw( + st.lists(st.sampled_from(all_names), min_size=1, unique=True) + ) + find_any(xps.arrays(st.sampled_from(sample_names), ())) + + +def test_generate_arrays_from_shapes_strategy(xp, xps): + """Generate arrays from shapes strategy.""" + find_any(xps.arrays(xp.int8, xps.array_shapes())) + + +def test_generate_arrays_from_integers_strategy_as_shape(xp, xps): + """Generate arrays from integers strategy as shapes strategy.""" + find_any(xps.arrays(xp.int8, st.integers(0, 100))) + + +def test_generate_arrays_from_zero_dimensions(xp, xps): + """Generate arrays from empty shape.""" + assert_all_examples(xps.arrays(xp.int8, ()), lambda x: x.shape == ()) + + +@given(data=st.data()) +def test_generate_arrays_from_zero_sided_shapes(xp, xps, data): + """Generate arrays from shapes with at least one 0-sized dimension.""" + shape = data.draw(xps.array_shapes(min_side=0).filter(lambda s: 0 in s)) + assert_all_examples(xps.arrays(xp.int8, shape), lambda x: x.shape == shape) + + +def test_generate_arrays_from_unsigned_ints(xp, xps): + """Generate arrays from unsigned integer dtype.""" + assert_all_examples(xps.arrays(xp.uint32, (5, 5)), lambda x: xp.all(x >= 0)) + # Ensure we're not just picking non-negative signed integers + signed_max = xp.iinfo(xp.int32).max + find_any(xps.arrays(xp.uint32, (5, 5)), lambda x: xp.any(x > signed_max)) + + +def test_generate_arrays_from_0d_arrays(xp, xps): + """Generate arrays from 0d array elements.""" + assert_all_examples( + xps.arrays( + dtype=xp.uint8, + shape=(5, 5), + elements=xps.from_dtype(xp.uint8).map( + lambda e: xp.asarray(e, dtype=xp.uint8) + ), + ), + lambda x: x.shape == (5, 5), + ) + + +def test_minimize_arrays_with_default_dtype_shape_strategies(xp, xps): + """Strategy with default scalar_dtypes and array_shapes strategies minimize + to a boolean 1-dimensional array of size 1.""" + smallest = minimal(xps.arrays(xps.scalar_dtypes(), xps.array_shapes())) + assert smallest.shape == (1,) + assert smallest.dtype == xp.bool + assert not xp.any(smallest) + + +def test_minimize_arrays_with_0d_shape_strategy(xp, xps): + """Strategy with shape strategy that can generate empty tuples minimizes to + 0d arrays.""" + smallest = minimal(xps.arrays(xp.int8, xps.array_shapes(min_dims=0))) + assert smallest.shape == () + + +@pytest.mark.parametrize("dtype", dtype_name_params[1:]) +def test_minimizes_numeric_arrays(xp, xps, dtype): + """Strategies with numeric dtypes minimize to zero-filled arrays.""" + smallest = minimal(xps.arrays(dtype, (2, 2))) + assert xp.all(smallest == 0) + + +def test_minimize_large_uint_arrays(xp, xps): + """Strategy with uint dtype and largely sized shape minimizes to a good + example.""" + if not hasattr(xp, "nonzero"): + pytest.skip("optional API") + smallest = minimal( + xps.arrays(xp.uint8, 100), + lambda x: xp.any(x) and not xp.all(x), + timeout_after=60, + ) + assert xp.all(xp.logical_or(smallest == 0, smallest == 1)) + idx = xp.nonzero(smallest)[0] + assert idx.size in (1, smallest.size - 1) + + +@pytest.mark.filterwarnings("ignore::RuntimeWarning") +@flaky(max_runs=50, min_passes=1) +def test_minimize_float_arrays(xp, xps): + """Strategy with float dtype minimizes to a good example. + + We filter runtime warnings and expect flaky array generation for + specifically NumPy - this behaviour may not be required when testing + with other array libraries. + """ + smallest = minimal(xps.arrays(xp.float32, 50), lambda x: xp.sum(x) >= 1.0) + assert xp.sum(smallest) in (1, 50) + + +def test_minimizes_to_fill(xp, xps): + """Strategy with single fill value minimizes to arrays only containing said + fill value.""" + smallest = minimal(xps.arrays(xp.float32, 10, fill=st.just(3.0))) + assert xp.all(smallest == 3.0) + + +def test_generate_unique_arrays(xp, xps): + """Generates unique arrays.""" + skip_on_missing_unique_values(xp) + assert_all_examples( + xps.arrays(xp.int8, st.integers(0, 20), unique=True), + lambda x: xp.unique_values(x).size == x.size, + ) + + +def test_cannot_draw_unique_arrays_with_too_small_elements(xp, xps): + """Unique strategy with elements strategy range smaller than its size raises + helpful error.""" + with pytest.raises(InvalidArgument): + xps.arrays(xp.int8, 10, elements=st.integers(0, 5), unique=True).example() + + +def test_cannot_fill_arrays_with_non_castable_value(xp, xps): + """Strategy with fill not castable to dtype raises helpful error.""" + with pytest.raises(InvalidArgument): + xps.arrays(xp.int8, 10, fill=st.just("not a castable value")).example() + + +def test_generate_unique_arrays_with_high_collision_elements(xp, xps): + """Generates unique arrays with just elements of 0.0 and NaN fill.""" + + @given( + xps.arrays( + dtype=xp.float32, + shape=st.integers(0, 20), + elements=st.just(0.0), + fill=st.just(xp.nan), + unique=True, + ) + ) + def test(x): + zero_mask = x == 0.0 + assert xp.sum(xp.astype(zero_mask, xp.uint8)) <= 1 + + test() + + +def test_generate_unique_arrays_using_all_elements(xp, xps): + """Unique strategy with elements strategy range equal to its size will only + generate arrays with one of each possible element.""" + skip_on_missing_unique_values(xp) + assert_all_examples( + xps.arrays(xp.int8, (4,), elements=st.integers(0, 3), unique=True), + lambda x: xp.unique_values(x).size == x.size, + ) + + +def test_may_fill_unique_arrays_with_nan(xp, xps): + """Unique strategy with NaN fill can generate arrays holding NaNs.""" + find_any( + xps.arrays( + dtype=xp.float32, + shape=10, + elements={"allow_nan": False}, + unique=True, + fill=st.just(xp.nan), + ), + lambda x: xp.any(xp.isnan(x)), + ) + + +def test_may_not_fill_unique_array_with_non_nan(xp, xps): + """Unique strategy with just fill elements of 0.0 raises helpful error.""" + with pytest.raises(InvalidArgument): + strat = xps.arrays( + dtype=xp.float32, + shape=10, + elements={"allow_nan": False}, + unique=True, + fill=st.just(0.0), + ) + strat.example() + + +@pytest.mark.parametrize( + "kwargs", + [ + {"elements": st.just(300)}, + {"elements": st.nothing(), "fill": st.just(300)}, + ], +) +def test_may_not_use_overflowing_integers(xp, xps, kwargs): + """Strategy with elements strategy range outside the dtype's bounds raises + helpful error.""" + with pytest.raises(InvalidArgument): + xps.arrays(dtype=xp.int8, shape=1, **kwargs).example() + + +@pytest.mark.parametrize("fill", [False, True]) +@pytest.mark.parametrize( + "dtype, strat", + [ + ("float32", st.floats(min_value=10**40, allow_infinity=False)), + ("float64", st.floats(min_value=10**40, allow_infinity=False)), + pytest.param( + "complex64", + st.complex_numbers(min_magnitude=10**300, allow_infinity=False), + marks=pytest.mark.xp_min_version(MIN_VER_FOR_COMPLEX), + ), + ], +) +def test_may_not_use_unrepresentable_elements(xp, xps, fill, dtype, strat): + """Strategy with elements not representable by the dtype raises helpful error.""" + if fill: + kw = {"elements": st.nothing(), "fill": strat} + else: + kw = {"elements": strat} + with pytest.raises(InvalidArgument): + xps.arrays(dtype=dtype, shape=1, **kw).example() + + +def test_floats_can_be_constrained(xp, xps): + """Strategy with float dtype and specified elements strategy range + (inclusive) generates arrays with elements inside said range.""" + assert_all_examples( + xps.arrays( + dtype=xp.float32, shape=10, elements={"min_value": 0, "max_value": 1} + ), + lambda x: xp.all(x >= 0) and xp.all(x <= 1), + ) + + +def test_floats_can_be_constrained_excluding_endpoints(xp, xps): + """Strategy with float dtype and specified elements strategy range + (exclusive) generates arrays with elements inside said range.""" + assert_all_examples( + xps.arrays( + dtype=xp.float32, + shape=10, + elements={ + "min_value": 0, + "max_value": 1, + "exclude_min": True, + "exclude_max": True, + }, + ), + lambda x: xp.all(x > 0) and xp.all(x < 1), + ) + + +def test_is_still_unique_with_nan_fill(xp, xps): + """Unique strategy with NaN fill generates unique arrays.""" + skip_on_missing_unique_values(xp) + xfail_on_indistinct_nans(xp) + assert_all_examples( + xps.arrays( + dtype=xp.float32, + elements={"allow_nan": False}, + shape=10, + unique=True, + fill=st.just(xp.nan), + ), + lambda x: xp.unique_values(x).size == x.size, + ) + + +def test_unique_array_with_fill_can_use_all_elements(xp, xps): + """Unique strategy with elements range equivalent to its size and NaN fill + can generate arrays with all possible values.""" + skip_on_missing_unique_values(xp) + xfail_on_indistinct_nans(xp) + find_any( + xps.arrays( + dtype=xp.float32, + shape=10, + unique=True, + elements=st.integers(1, 9), + fill=st.just(xp.nan), + ), + lambda x: xp.unique_values(x).size == x.size, + ) + + +def test_generate_unique_arrays_without_fill(xp, xps): + """Generate arrays from unique strategy with no fill. + + Covers the collision-related branches for fully dense unique arrays. + Choosing 25 of 256 possible values means we're almost certain to see + colisions thanks to the birthday paradox, but finding unique values should + still be easy. + """ + skip_on_missing_unique_values(xp) + assert_all_examples( + xps.arrays(dtype=xp.uint8, shape=25, unique=True, fill=st.nothing()), + lambda x: xp.unique_values(x).size == x.size, + ) + + +def test_efficiently_generate_unique_arrays_using_all_elements(xp, xps): + """Unique strategy with elements strategy range equivalent to its size + generates arrays with all possible values. Generation is not too slow. + + Avoids the birthday paradox with UniqueSampledListStrategy. + """ + skip_on_missing_unique_values(xp) + assert_all_examples( + xps.arrays(dtype=xp.int8, shape=255, unique=True), + lambda x: xp.unique_values(x).size == x.size, + ) + + +@given(st.data(), st.integers(-100, 100), st.integers(1, 100)) +def test_array_element_rewriting(xp, xps, data, start, size): + """Unique strategy generates arrays with expected elements.""" + x = data.draw( + xps.arrays( + dtype=xp.int64, + shape=size, + elements=st.integers(start, start + size - 1), + unique=True, + ) + ) + x_set_expect = xp.linspace(start, start + size - 1, size, dtype=xp.int64) + x_set = xp.sort(xp.unique_values(x)) + assert xp.all(x_set == x_set_expect) + + +def test_generate_0d_arrays_with_no_fill(xp, xps): + """Generate arrays with zero-dimensions and no fill.""" + assert_all_examples( + xps.arrays(xp.bool, (), fill=st.nothing()), + lambda x: x.dtype == xp.bool and x.shape == (), + ) + + +@pytest.mark.parametrize("dtype", ["float32", "float64"]) +@pytest.mark.parametrize("low", [-2.0, -1.0, 0.0, 1.0]) +@given(st.data()) +def test_excluded_min_in_float_arrays(xp, xps, dtype, low, data): + """Strategy with elements strategy excluding min does not generate arrays + with elements less or equal to said min.""" + strat = xps.arrays( + dtype=dtype, + shape=(), + elements={ + "min_value": low, + "max_value": low + 1, + "exclude_min": True, + }, + ) + x = data.draw(strat, label="array") + assert xp.all(x > low) + + +@st.composite +def distinct_int64_integers(draw): + used = draw(st.shared(st.builds(set), key="distinct_int64_integers.used")) + i = draw(st.integers(-(2**63), 2**63 - 1).filter(lambda x: x not in used)) + used.add(i) + return i + + +def test_does_not_reuse_distinct_integers(xp, xps): + """Strategy with distinct integer elements strategy generates arrays with + distinct values.""" + skip_on_missing_unique_values(xp) + assert_all_examples( + xps.arrays(xp.int64, 10, elements=distinct_int64_integers()), + lambda x: xp.unique_values(x).size == x.size, + ) + + +def test_may_reuse_distinct_integers_if_asked(xp, xps): + """Strategy with shared elements and fill strategies of distinct integers + may generate arrays with non-distinct values.""" + skip_on_missing_unique_values(xp) + find_any( + xps.arrays( + xp.int64, + 10, + elements=distinct_int64_integers(), + fill=distinct_int64_integers(), + ), + lambda x: xp.unique_values(x).size < x.size, + ) + + +def test_subnormal_elements_validation(xp, xps): + """Strategy with subnormal elements strategy is correctly validated. + + For FTZ builds of array modules, a helpful error should raise. Conversely, + for builds of array modules which support subnormals, the strategy should + generate arrays without raising. + """ + elements = { + "min_value": 0.0, + "max_value": width_smallest_normals[32], + "exclude_min": True, + "exclude_max": True, + "allow_subnormal": True, + } + strat = xps.arrays(xp.float32, 10, elements=elements) + if flushes_to_zero(xp, width=32): + with pytest.raises(InvalidArgument, match="Generated subnormal float"): + strat.example() + else: + strat.example() diff --git a/hypothesis-python/tests/array_api/test_from_dtype.py b/hypothesis-python/tests/array_api/test_from_dtype.py new file mode 100644 index 0000000000..18f6d081f1 --- /dev/null +++ b/hypothesis-python/tests/array_api/test_from_dtype.py @@ -0,0 +1,107 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import math + +import pytest + +from hypothesis.extra.array_api import find_castable_builtin_for_dtype +from hypothesis.internal.floats import width_smallest_normals + +from tests.array_api.common import dtype_name_params, flushes_to_zero +from tests.common.debug import ( + assert_all_examples, + assert_no_examples, + find_any, + minimal, +) + + +@pytest.mark.parametrize("dtype_name", dtype_name_params) +def test_strategies_have_reusable_values(xp, xps, dtype_name): + """Inferred strategies have reusable values.""" + strat = xps.from_dtype(dtype_name) + assert strat.has_reusable_values + + +@pytest.mark.parametrize("dtype_name", dtype_name_params) +def test_produces_castable_instances_from_dtype(xp, xps, dtype_name): + """Strategies inferred by dtype generate values of a builtin type castable + to the dtype.""" + dtype = getattr(xp, dtype_name) + builtin = find_castable_builtin_for_dtype(xp, xps.api_version, dtype) + assert_all_examples(xps.from_dtype(dtype), lambda v: isinstance(v, builtin)) + + +@pytest.mark.parametrize("dtype_name", dtype_name_params) +def test_produces_castable_instances_from_name(xp, xps, dtype_name): + """Strategies inferred by dtype name generate values of a builtin type + castable to the dtype.""" + dtype = getattr(xp, dtype_name) + builtin = find_castable_builtin_for_dtype(xp, xps.api_version, dtype) + assert_all_examples(xps.from_dtype(dtype_name), lambda v: isinstance(v, builtin)) + + +@pytest.mark.parametrize("dtype_name", dtype_name_params) +def test_passing_inferred_strategies_in_arrays(xp, xps, dtype_name): + """Inferred strategies usable in arrays strategy.""" + elements = xps.from_dtype(dtype_name) + find_any(xps.arrays(dtype_name, 10, elements=elements)) + + +@pytest.mark.parametrize( + "dtype, kwargs, predicate", + [ + # Floating point: bounds, exclusive bounds, and excluding nonfinites + ("float32", {"min_value": 1, "max_value": 2}, lambda x: 1 <= x <= 2), + ( + "float32", + {"min_value": 1, "max_value": 2, "exclude_min": True, "exclude_max": True}, + lambda x: 1 < x < 2, + ), + ("float32", {"allow_nan": False}, lambda x: not math.isnan(x)), + ("float32", {"allow_infinity": False}, lambda x: not math.isinf(x)), + ("float32", {"allow_nan": False, "allow_infinity": False}, math.isfinite), + # Integer bounds, limited to the representable range + ("int8", {"min_value": -1, "max_value": 1}, lambda x: -1 <= x <= 1), + ("uint8", {"min_value": 1, "max_value": 2}, lambda x: 1 <= x <= 2), + ], +) +def test_from_dtype_with_kwargs(xp, xps, dtype, kwargs, predicate): + """Strategies inferred with kwargs generate values in bounds.""" + strat = xps.from_dtype(dtype, **kwargs) + assert_all_examples(strat, predicate) + + +def test_can_minimize_floats(xp, xps): + """Inferred float strategy minimizes to a good example.""" + smallest = minimal(xps.from_dtype(xp.float32), lambda n: n >= 1.0) + assert smallest == 1 + + +smallest_normal = width_smallest_normals[32] + + +@pytest.mark.parametrize( + "kwargs", + [ + {}, + {"min_value": -1}, + {"max_value": 1}, + {"min_value": -1, "max_value": 1}, + ], +) +def test_subnormal_generation(xp, xps, kwargs): + """Generation of subnormals is dependent on FTZ behaviour of array module.""" + strat = xps.from_dtype(xp.float32, **kwargs).filter(lambda n: n != 0) + if flushes_to_zero(xp, width=32): + assert_no_examples(strat, lambda n: -smallest_normal < n < smallest_normal) + else: + find_any(strat, lambda n: -smallest_normal < n < smallest_normal) diff --git a/hypothesis-python/tests/array_api/test_indices.py b/hypothesis-python/tests/array_api/test_indices.py new file mode 100644 index 0000000000..d89b6b3936 --- /dev/null +++ b/hypothesis-python/tests/array_api/test_indices.py @@ -0,0 +1,152 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import math + +import pytest + +from hypothesis import assume, given, note, strategies as st +from hypothesis.extra._array_helpers import NDIM_MAX + +from tests.common.debug import assert_all_examples, find_any + + +@pytest.mark.parametrize( + "condition", + [ + lambda ix: Ellipsis in ix, + lambda ix: Ellipsis not in ix, + lambda ix: None in ix, + lambda ix: None not in ix, + ], +) +def test_generate_optional_indices(xp, xps, condition): + """Strategy can generate indices with optional values.""" + strat = ( + xps.array_shapes(min_dims=1, max_dims=32) + .flatmap(lambda s: xps.indices(s, allow_newaxis=True)) + .map(lambda idx: idx if isinstance(idx, tuple) else (idx,)) + ) + find_any(strat, condition) + + +def test_cannot_generate_newaxis_when_disabled(xp, xps): + """Strategy does not generate newaxis when disabled (i.e. the default).""" + assert_all_examples( + xps.indices((3, 3, 3)), lambda idx: idx == ... or None not in idx + ) + + +def test_generate_indices_for_0d_shape(xp, xps): + """Strategy only generates empty tuples or Ellipsis as indices for an empty + shape.""" + assert_all_examples( + xps.indices(shape=(), allow_ellipsis=True), + lambda idx: idx in [(), Ellipsis, (Ellipsis,)], + ) + + +def test_generate_tuples_and_non_tuples_for_1d_shape(xp, xps): + """Strategy can generate tuple and non-tuple indices with a 1-dimensional shape.""" + strat = xps.indices(shape=(1,), allow_ellipsis=True) + find_any(strat, lambda ix: isinstance(ix, tuple)) + find_any(strat, lambda ix: not isinstance(ix, tuple)) + + +def test_generate_long_ellipsis(xp, xps): + """Strategy can replace runs of slice(None) with Ellipsis. + + We specifically test if [0,...,0] is generated alongside [0,:,:,:,0] + """ + strat = xps.indices(shape=(1, 0, 0, 0, 1), max_dims=3, allow_ellipsis=True) + find_any(strat, lambda ix: len(ix) == 3 and ix[1] == Ellipsis) + find_any( + strat, + lambda ix: len(ix) == 5 + and all(isinstance(key, slice) and key == slice(None) for key in ix[1:3]), + ) + + +def test_indices_replaces_whole_axis_slices_with_ellipsis(xp, xps): + # `slice(None)` (aka `:`) is the only valid index for an axis of size + # zero, so if all dimensions are 0 then a `...` will replace all the + # slices because we generate `...` for entire contiguous runs of `:` + assert_all_examples( + xps.indices(shape=(0, 0, 0, 0, 0), max_dims=5).filter( + lambda idx: isinstance(idx, tuple) and Ellipsis in idx + ), + lambda idx: slice(None) not in idx, + ) + + +def test_efficiently_generate_indexers(xp, xps): + """Generation is not too slow.""" + find_any(xps.indices((3, 3, 3, 3, 3))) + + +@given(allow_newaxis=st.booleans(), allow_ellipsis=st.booleans(), data=st.data()) +def test_generate_valid_indices(xp, xps, allow_newaxis, allow_ellipsis, data): + """Strategy generates valid indices.""" + shape = data.draw( + xps.array_shapes(min_dims=1, max_side=4) + | xps.array_shapes(min_dims=1, min_side=0, max_side=10), + label="shape", + ) + min_dims = data.draw( + st.integers(0, len(shape) if not allow_newaxis else len(shape) + 2), + label="min_dims", + ) + max_dims = data.draw( + st.none() + | st.integers(min_dims, len(shape) if not allow_newaxis else NDIM_MAX), + label="max_dims", + ) + indexer = data.draw( + xps.indices( + shape, + min_dims=min_dims, + max_dims=max_dims, + allow_newaxis=allow_newaxis, + allow_ellipsis=allow_ellipsis, + ), + label="indexer", + ) + + _indexer = indexer if isinstance(indexer, tuple) else (indexer,) + # Check that disallowed things are indeed absent + if not allow_ellipsis: + assert Ellipsis not in _indexer + if not allow_newaxis: + assert None not in _indexer # i.e. xp.newaxis + # Check index is composed of valid objects + for i in _indexer: + assert isinstance(i, int) or isinstance(i, slice) or i is None or i == Ellipsis + # Check indexer does not flat index + nonexpanding_indexer = [i for i in _indexer if i is not None] + if Ellipsis in _indexer: + assert sum(i == Ellipsis for i in _indexer) == 1 + # Note ellipsis can index 0 axes + assert len(nonexpanding_indexer) <= len(shape) + 1 + else: + assert len(nonexpanding_indexer) == len(shape) + + if 0 in shape: + # If there's a zero in the shape, the array will have no elements. + array = xp.zeros(shape) + assert array.size == 0 # sanity check + elif math.prod(shape) <= 10**5: + # If it's small enough to instantiate, do so with distinct elements. + array = xp.reshape(xp.arange(math.prod(shape)), shape) + else: + # We can't cheat on this one, so just try another. + assume(False) + # Finally, check that we can use our indexer without error + note(f"{array=}") + array[indexer] diff --git a/hypothesis-python/tests/array_api/test_partial_adoptors.py b/hypothesis-python/tests/array_api/test_partial_adoptors.py new file mode 100644 index 0000000000..2f0f410626 --- /dev/null +++ b/hypothesis-python/tests/array_api/test_partial_adoptors.py @@ -0,0 +1,140 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +from copy import copy +from functools import lru_cache +from types import SimpleNamespace +from typing import Tuple + +import pytest + +from hypothesis import given, strategies as st +from hypothesis.errors import HypothesisWarning, InvalidArgument +from hypothesis.extra.array_api import ( + COMPLEX_NAMES, + DTYPE_NAMES, + FLOAT_NAMES, + INT_NAMES, + UINT_NAMES, + make_strategies_namespace, + mock_xp, +) + +MOCK_WARN_MSG = f"determine.*{mock_xp.__name__}.*Array API" + + +@lru_cache() +def make_mock_xp(*, exclude: Tuple[str, ...] = ()) -> SimpleNamespace: + xp = copy(mock_xp) + assert isinstance(exclude, tuple) # sanity check + for attr in exclude: + delattr(xp, attr) + return xp + + +def test_warning_on_noncompliant_xp(): + """Using non-compliant array modules raises helpful warning""" + xp = make_mock_xp() + with pytest.warns(HypothesisWarning, match=MOCK_WARN_MSG): + make_strategies_namespace(xp, api_version="draft") + + +@pytest.mark.filterwarnings(f"ignore:.*{MOCK_WARN_MSG}.*") +@pytest.mark.parametrize( + "stratname, args, attr", + [("from_dtype", ["int8"], "iinfo"), ("arrays", ["int8", 5], "full")], +) +def test_error_on_missing_attr(stratname, args, attr): + """Strategies raise helpful error when using array modules that lack + required attributes.""" + xp = make_mock_xp(exclude=(attr,)) + xps = make_strategies_namespace(xp, api_version="draft") + func = getattr(xps, stratname) + with pytest.raises(InvalidArgument, match=f"{mock_xp.__name__}.*required.*{attr}"): + func(*args).example() + + +dtypeless_xp = make_mock_xp(exclude=tuple(DTYPE_NAMES)) +with pytest.warns(HypothesisWarning): + dtypeless_xps = make_strategies_namespace(dtypeless_xp, api_version="draft") + + +@pytest.mark.parametrize( + "stratname", + [ + "scalar_dtypes", + "boolean_dtypes", + "numeric_dtypes", + "integer_dtypes", + "unsigned_integer_dtypes", + "floating_dtypes", + "real_dtypes", + "complex_dtypes", + ], +) +def test_error_on_missing_dtypes(stratname): + """Strategies raise helpful error when using array modules that lack + required dtypes.""" + func = getattr(dtypeless_xps, stratname) + with pytest.raises(InvalidArgument, match=f"{mock_xp.__name__}.*dtype.*namespace"): + func().example() + + +@pytest.mark.filterwarnings(f"ignore:.*{MOCK_WARN_MSG}.*") +@pytest.mark.parametrize( + "stratname, keep_anys", + [ + ("scalar_dtypes", [INT_NAMES, UINT_NAMES, FLOAT_NAMES]), + ("numeric_dtypes", [INT_NAMES, UINT_NAMES, FLOAT_NAMES, COMPLEX_NAMES]), + ("integer_dtypes", [INT_NAMES]), + ("unsigned_integer_dtypes", [UINT_NAMES]), + ("floating_dtypes", [FLOAT_NAMES]), + ("real_dtypes", [INT_NAMES, UINT_NAMES, FLOAT_NAMES]), + ("complex_dtypes", [COMPLEX_NAMES]), + ], +) +@given(st.data()) +def test_warning_on_partial_dtypes(stratname, keep_anys, data): + """Strategies using array modules with at least one of a dtype in the + necessary category/categories execute with a warning.""" + exclude = [] + for keep_any in keep_anys: + exclude.extend( + data.draw( + st.lists( + st.sampled_from(keep_any), + min_size=1, + max_size=len(keep_any) - 1, + unique=True, + ) + ) + ) + xp = make_mock_xp(exclude=tuple(exclude)) + xps = make_strategies_namespace(xp, api_version="draft") + func = getattr(xps, stratname) + with pytest.warns(HypothesisWarning, match=f"{mock_xp.__name__}.*dtype.*namespace"): + data.draw(func()) + + +def test_raises_on_inferring_with_no_dunder_version(): + """When xp has no __array_api_version__, inferring api_version raises + helpful error.""" + xp = make_mock_xp(exclude=("__array_api_version__",)) + with pytest.raises(InvalidArgument, match="has no attribute"): + make_strategies_namespace(xp) + + +def test_raises_on_invalid_dunder_version(): + """When xp has invalid __array_api_version__, inferring api_version raises + helpful error.""" + xp = make_mock_xp() + xp.__array_api_version__ = None + with pytest.raises(InvalidArgument): + make_strategies_namespace(xp) diff --git a/hypothesis-python/tests/array_api/test_pretty.py b/hypothesis-python/tests/array_api/test_pretty.py new file mode 100644 index 0000000000..bb38f6427f --- /dev/null +++ b/hypothesis-python/tests/array_api/test_pretty.py @@ -0,0 +1,104 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +from inspect import signature + +import pytest + +from hypothesis.errors import InvalidArgument +from hypothesis.extra.array_api import make_strategies_namespace + +from tests.array_api.common import MIN_VER_FOR_COMPLEX + + +@pytest.mark.parametrize( + "name", + [ + "from_dtype", + "arrays", + "array_shapes", + "scalar_dtypes", + "boolean_dtypes", + "numeric_dtypes", + "integer_dtypes", + "unsigned_integer_dtypes", + "floating_dtypes", + "real_dtypes", + pytest.param( + "complex_dtypes", marks=pytest.mark.xp_min_version(MIN_VER_FOR_COMPLEX) + ), + "valid_tuple_axes", + "broadcastable_shapes", + "mutually_broadcastable_shapes", + "indices", + ], +) +def test_namespaced_methods_meta(xp, xps, name): + """Namespaced method objects have good meta attributes.""" + func = getattr(xps, name) + assert func.__name__ == name + assert func.__doc__ is not None + # The (private) top-level strategy methods may expose a xp argument in their + # function signatures. make_strategies_namespace() exists to wrap these + # top-level methods by binding the passed xp argument, and so the namespace + # it returns should not expose xp in any of its function signatures. + assert "xp" not in signature(func).parameters.keys() + + +@pytest.mark.parametrize( + "name, valid_args", + [ + ("from_dtype", ["int8"]), + ("arrays", ["int8", 5]), + ("array_shapes", []), + ("scalar_dtypes", []), + ("boolean_dtypes", []), + ("numeric_dtypes", []), + ("integer_dtypes", []), + ("unsigned_integer_dtypes", []), + ("floating_dtypes", []), + ("real_dtypes", []), + pytest.param( + "complex_dtypes", [], marks=pytest.mark.xp_min_version(MIN_VER_FOR_COMPLEX) + ), + ("valid_tuple_axes", [0]), + ("broadcastable_shapes", [()]), + ("mutually_broadcastable_shapes", [3]), + ("indices", [(5,)]), + ], +) +def test_namespaced_strategies_repr(xp, xps, name, valid_args): + """Namespaced strategies have good repr.""" + func = getattr(xps, name) + strat = func(*valid_args) + assert repr(strat).startswith(name + "("), f"{name} not in strat repr {strat!r}" + assert len(repr(strat)) < 100, "strat repr looks too long" + assert xp.__name__ not in repr(strat), f"{xp.__name__} in strat repr" + + +@pytest.mark.filterwarnings("ignore::hypothesis.errors.HypothesisWarning") +def test_inferred_version_strategies_namespace_repr(xp): + """Strategies namespace has good repr when api_version=None.""" + try: + xps = make_strategies_namespace(xp) + except InvalidArgument as e: + pytest.skip(str(e)) + expected = f"make_strategies_namespace({xp.__name__})" + assert repr(xps) == expected + assert str(xps) == expected + + +@pytest.mark.filterwarnings("ignore::hypothesis.errors.HypothesisWarning") +def test_specified_version_strategies_namespace_repr(xp): + """Strategies namespace has good repr when api_version is specified.""" + xps = make_strategies_namespace(xp, api_version="2021.12") + expected = f"make_strategies_namespace({xp.__name__}, api_version='2021.12')" + assert repr(xps) == expected + assert str(xps) == expected diff --git a/hypothesis-python/tests/array_api/test_scalar_dtypes.py b/hypothesis-python/tests/array_api/test_scalar_dtypes.py new file mode 100644 index 0000000000..09e294bd1d --- /dev/null +++ b/hypothesis-python/tests/array_api/test_scalar_dtypes.py @@ -0,0 +1,112 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import pytest + +from hypothesis.extra.array_api import ( + COMPLEX_NAMES, + DTYPE_NAMES, + FLOAT_NAMES, + INT_NAMES, + NUMERIC_NAMES, + REAL_NAMES, + UINT_NAMES, +) + +from tests.array_api.common import MIN_VER_FOR_COMPLEX +from tests.common.debug import assert_all_examples, find_any, minimal + + +@pytest.mark.parametrize( + ("strat_name", "dtype_names"), + [ + ("integer_dtypes", INT_NAMES), + ("unsigned_integer_dtypes", UINT_NAMES), + ("floating_dtypes", FLOAT_NAMES), + ("real_dtypes", REAL_NAMES), + pytest.param( + "complex_dtypes", + COMPLEX_NAMES, + marks=pytest.mark.xp_min_version(MIN_VER_FOR_COMPLEX), + ), + ], +) +def test_all_generated_dtypes_are_of_group(xp, xps, strat_name, dtype_names): + """Strategy only generates expected dtypes.""" + strat_func = getattr(xps, strat_name) + dtypes = [getattr(xp, n) for n in dtype_names] + assert_all_examples(strat_func(), lambda dtype: dtype in dtypes) + + +def test_all_generated_scalar_dtypes_are_scalar(xp, xps): + """Strategy only generates scalar dtypes.""" + if xps.api_version > "2021.12": + dtypes = [getattr(xp, n) for n in DTYPE_NAMES] + else: + dtypes = [getattr(xp, n) for n in ("bool",) + REAL_NAMES] + assert_all_examples(xps.scalar_dtypes(), lambda dtype: dtype in dtypes) + + +def test_all_generated_numeric_dtypes_are_numeric(xp, xps): + """Strategy only generates numeric dtypes.""" + if xps.api_version > "2021.12": + dtypes = [getattr(xp, n) for n in NUMERIC_NAMES] + else: + dtypes = [getattr(xp, n) for n in REAL_NAMES] + assert_all_examples(xps.numeric_dtypes(), lambda dtype: dtype in dtypes) + + +def skipif_unsupported_complex(strat_name, dtype_name): + if not dtype_name.startswith("complex"): + return strat_name, dtype_name + mark = pytest.mark.xp_min_version(MIN_VER_FOR_COMPLEX) + return pytest.param(strat_name, dtype_name, marks=mark) + + +@pytest.mark.parametrize( + ("strat_name", "dtype_name"), + [ + *[skipif_unsupported_complex("scalar_dtypes", n) for n in DTYPE_NAMES], + *[skipif_unsupported_complex("numeric_dtypes", n) for n in NUMERIC_NAMES], + *[("integer_dtypes", n) for n in INT_NAMES], + *[("unsigned_integer_dtypes", n) for n in UINT_NAMES], + *[("floating_dtypes", n) for n in FLOAT_NAMES], + *[("real_dtypes", n) for n in REAL_NAMES], + *[skipif_unsupported_complex("complex_dtypes", n) for n in COMPLEX_NAMES], + ], +) +def test_strategy_can_generate_every_dtype(xp, xps, strat_name, dtype_name): + """Strategy generates every expected dtype.""" + strat_func = getattr(xps, strat_name) + dtype = getattr(xp, dtype_name) + find_any(strat_func(), lambda d: d == dtype) + + +def test_minimise_scalar_dtypes(xp, xps): + """Strategy minimizes to bool dtype.""" + assert minimal(xps.scalar_dtypes()) == xp.bool + + +@pytest.mark.parametrize( + "strat_name, sizes", + [ + ("integer_dtypes", 8), + ("unsigned_integer_dtypes", 8), + ("floating_dtypes", 32), + pytest.param( + "complex_dtypes", 64, marks=pytest.mark.xp_min_version(MIN_VER_FOR_COMPLEX) + ), + ], +) +def test_can_specify_sizes_as_an_int(xp, xps, strat_name, sizes): + """Strategy treats ints as a single size.""" + strat_func = getattr(xps, strat_name) + strat = strat_func(sizes=sizes) + find_any(strat) diff --git a/hypothesis-python/tests/array_api/test_strategies_namespace.py b/hypothesis-python/tests/array_api/test_strategies_namespace.py new file mode 100644 index 0000000000..d1f18a6c25 --- /dev/null +++ b/hypothesis-python/tests/array_api/test_strategies_namespace.py @@ -0,0 +1,86 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +from types import SimpleNamespace +from weakref import WeakValueDictionary + +import pytest + +from hypothesis.extra import array_api +from hypothesis.extra.array_api import ( + NOMINAL_VERSIONS, + make_strategies_namespace, + mock_xp, +) +from hypothesis.strategies import SearchStrategy + +pytestmark = pytest.mark.filterwarnings("ignore::hypothesis.errors.HypothesisWarning") + + +class HashableArrayModuleFactory: + """ + mock_xp cannot be hashed and thus cannot be used in our cache. So just for + the purposes of testing the cache, we wrap it with an unsafe hash method. + """ + + def __getattr__(self, name): + return getattr(mock_xp, name) + + def __hash__(self): + return hash(tuple(sorted(mock_xp.__dict__))) + + +@pytest.mark.parametrize("api_version", ["2021.12", None]) +def test_caching(api_version, monkeypatch): + """Caches namespaces respective to arguments.""" + xp = HashableArrayModuleFactory() + assert isinstance(array_api._args_to_xps, WeakValueDictionary) # sanity check + monkeypatch.setattr(array_api, "_args_to_xps", WeakValueDictionary()) + assert len(array_api._args_to_xps) == 0 # sanity check + xps1 = array_api.make_strategies_namespace(xp, api_version=api_version) + assert len(array_api._args_to_xps) == 1 + xps2 = array_api.make_strategies_namespace(xp, api_version=api_version) + assert len(array_api._args_to_xps) == 1 + assert isinstance(xps2, SimpleNamespace) + assert xps2 is xps1 + del xps1 + del xps2 + assert len(array_api._args_to_xps) == 0 + + +@pytest.mark.parametrize( + "api_version1, api_version2", [(None, "2021.12"), ("2021.12", None)] +) +def test_inferred_namespace_shares_cache(api_version1, api_version2, monkeypatch): + """Results from inferred versions share the same cache key as results + from specified versions.""" + xp = HashableArrayModuleFactory() + xp.__array_api_version__ = "2021.12" + assert isinstance(array_api._args_to_xps, WeakValueDictionary) # sanity check + monkeypatch.setattr(array_api, "_args_to_xps", WeakValueDictionary()) + assert len(array_api._args_to_xps) == 0 # sanity check + xps1 = array_api.make_strategies_namespace(xp, api_version=api_version1) + assert xps1.api_version == "2021.12" # sanity check + assert len(array_api._args_to_xps) == 1 + xps2 = array_api.make_strategies_namespace(xp, api_version=api_version2) + assert xps2.api_version == "2021.12" # sanity check + assert len(array_api._args_to_xps) == 1 + assert xps2 is xps1 + + +def test_complex_dtypes_raises_on_2021_12(): + """Accessing complex_dtypes() for 2021.12 strategy namespace raises helpful + error, but accessing on future versions returns expected strategy.""" + first_xps = make_strategies_namespace(mock_xp, api_version="2021.12") + with pytest.raises(AttributeError, match="attempted to access"): + first_xps.complex_dtypes() + for api_version in NOMINAL_VERSIONS[1:]: + xps = make_strategies_namespace(mock_xp, api_version=api_version) + assert isinstance(xps.complex_dtypes(), SearchStrategy) diff --git a/hypothesis-python/tests/codemods/test_codemod_cli.py b/hypothesis-python/tests/codemods/test_codemod_cli.py index 50b937ec0e..63cb78f951 100644 --- a/hypothesis-python/tests/codemods/test_codemod_cli.py +++ b/hypothesis-python/tests/codemods/test_codemod_cli.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import subprocess @@ -41,10 +36,9 @@ def run(command, tmpdir=None, input=None): return subprocess.run( command, input=input, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, + capture_output=True, shell=True, - universal_newlines=True, + text=True, cwd=tmpdir, ) diff --git a/hypothesis-python/tests/codemods/test_codemods.py b/hypothesis-python/tests/codemods/test_codemods.py index fe5a6397ed..fb9d343390 100644 --- a/hypothesis-python/tests/codemods/test_codemods.py +++ b/hypothesis-python/tests/codemods/test_codemods.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from libcst.codemod import CodemodTest @@ -75,6 +70,14 @@ def test_substitution(self) -> None: """ self.assertCodemod(before=before, after=after) + def test_noop_with_new_floats_kw(self) -> None: + before = """ + import hypothesis.strategies as st + + st.floats(0, 1, False, False, True, 32, False, False) # allow_subnormal=True + """ + self.assertCodemod(before=before, after=before) + def test_noop_if_unsure(self) -> None: before = """ import random @@ -118,3 +121,13 @@ def test_kwargs_noop(self): target(**kwargs) """ self.assertCodemod(before=before, after=before) + + def test_noop_with_too_many_arguments_passed(self) -> None: + # If there are too many arguments, we should leave this alone to raise + # TypeError on older versions instead of deleting the additional args. + before = """ + import hypothesis.strategies as st + + st.sets(st.integers(), 0, 1, True) + """ + self.assertCodemod(before=before, after=before) diff --git a/hypothesis-python/tests/common/__init__.py b/hypothesis-python/tests/common/__init__.py index e5c06705d0..7edb4f204a 100644 --- a/hypothesis-python/tests/common/__init__.py +++ b/hypothesis-python/tests/common/__init__.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import sys from collections import namedtuple @@ -81,7 +76,7 @@ def abc(x, y, z): sampled_from(("a", "b", "c")), integers(), integers(min_value=3), - integers(min_value=(-(2 ** 32)), max_value=(2 ** 64)), + integers(min_value=(-(2**32)), max_value=(2**64)), floats(), floats(min_value=-2.0, max_value=3.0), floats(), diff --git a/hypothesis-python/tests/common/arguments.py b/hypothesis-python/tests/common/arguments.py index bd78cccafc..d9f691eaad 100644 --- a/hypothesis-python/tests/common/arguments.py +++ b/hypothesis-python/tests/common/arguments.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest diff --git a/hypothesis-python/tests/common/costbounds.py b/hypothesis-python/tests/common/costbounds.py index 6f9928065f..fc82a4a142 100644 --- a/hypothesis-python/tests/common/costbounds.py +++ b/hypothesis-python/tests/common/costbounds.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis.internal.conjecture.shrinking.common import find_integer diff --git a/hypothesis-python/tests/common/debug.py b/hypothesis-python/tests/common/debug.py index 1c38c921c7..8c75d2974d 100644 --- a/hypothesis-python/tests/common/debug.py +++ b/hypothesis-python/tests/common/debug.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import ( HealthCheck, @@ -93,7 +88,7 @@ def find_any(definition, condition=lambda _: True, settings=None): def assert_no_examples(strategy, condition=lambda _: True): try: result = find_any(strategy, condition) - assert False, f"Expected no results but found {result!r}" + raise AssertionError(f"Expected no results but found {result!r}") except (Unsatisfiable, NoSuchExample): pass diff --git a/hypothesis-python/tests/common/setup.py b/hypothesis-python/tests/common/setup.py index 62c86ae1fa..d109273199 100644 --- a/hypothesis-python/tests/common/setup.py +++ b/hypothesis-python/tests/common/setup.py @@ -1,27 +1,22 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import os from tempfile import mkdtemp from warnings import filterwarnings -from hypothesis import Verbosity, settings +from hypothesis import Phase, Verbosity, settings from hypothesis._settings import not_set from hypothesis.configuration import set_hypothesis_home_dir from hypothesis.errors import NonInteractiveExampleWarning -from hypothesis.internal.charmap import charmap, charmap_file +from hypothesis.internal import charmap from hypothesis.internal.coverage import IN_COVERAGE_TESTS @@ -30,10 +25,7 @@ def run(): filterwarnings("ignore", category=ImportWarning) filterwarnings("ignore", category=FutureWarning, module="pandas._version") - # Fixed in recent versions but allowed by pytest=3.0.0; see #1630 - filterwarnings("ignore", category=DeprecationWarning, module="pluggy") - - # See https://github.com/numpy/numpy/pull/432 + # See https://github.com/numpy/numpy/pull/432; still a thing as of 2022. filterwarnings("ignore", message="numpy.dtype size changed") filterwarnings("ignore", message="numpy.ufunc size changed") @@ -47,14 +39,6 @@ def run(): category=UserWarning, ) - # Imported by Pandas in version 1.9, but fixed in later versions. - filterwarnings( - "ignore", message="Importing from numpy.testing.decorators is deprecated" - ) - filterwarnings( - "ignore", message="Importing from numpy.testing.nosetester is deprecated" - ) - # User-facing warning which does not apply to our own tests filterwarnings("ignore", category=NonInteractiveExampleWarning) @@ -62,8 +46,10 @@ def run(): set_hypothesis_home_dir(new_home) assert settings.default.database.path.startswith(new_home) - charmap() - assert os.path.exists(charmap_file()), charmap_file() + # Remove the cache because we might have saved this before setting the new home dir + charmap._charmap = None + charmap.charmap() + assert os.path.exists(charmap.charmap_file()), charmap.charmap_file() assert isinstance(settings, type) # We do a smoke test here before we mess around with settings. @@ -80,7 +66,11 @@ def run(): ) settings.register_profile( - "default", settings(max_examples=20 if IN_COVERAGE_TESTS else not_set) + "default", + settings( + max_examples=20 if IN_COVERAGE_TESTS else not_set, + phases=list(Phase), # Dogfooding the explain phase + ), ) settings.register_profile("speedy", settings(max_examples=5)) diff --git a/hypothesis-python/tests/common/strategies.py b/hypothesis-python/tests/common/strategies.py index 300567164d..e0cc13f16a 100644 --- a/hypothesis-python/tests/common/strategies.py +++ b/hypothesis-python/tests/common/strategies.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import time @@ -34,7 +29,7 @@ def __init__(self): self.accepted = set() def do_draw(self, data): - x = bytes([data.draw_bits(8) for _ in range(100)]) + x = bytes(data.draw_bits(8) for _ in range(100)) if x in self.accepted: return True ls = self.__last diff --git a/hypothesis-python/tests/common/utils.py b/hypothesis-python/tests/common/utils.py index 59e25a9c47..3af74df6cb 100644 --- a/hypothesis-python/tests/common/utils.py +++ b/hypothesis-python/tests/common/utils.py @@ -1,31 +1,53 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import contextlib import sys -import traceback from io import StringIO +from types import SimpleNamespace -from hypothesis._settings import Phase +from hypothesis import Phase, settings from hypothesis.errors import HypothesisDeprecationWarning +from hypothesis.internal.entropy import deterministic_PRNG +from hypothesis.internal.floats import next_down from hypothesis.internal.reflection import proxies from hypothesis.reporting import default, with_reporter from hypothesis.strategies._internal.core import from_type, register_type_strategy from hypothesis.strategies._internal.types import _global_type_lookup -no_shrink = tuple(set(Phase) - {Phase.shrink}) +try: + from pytest import raises +except ModuleNotFoundError: + # We are currently running under a test framework other than pytest, + # so use our own simplified implementation of `pytest.raises`. + + @contextlib.contextmanager + def raises(expected_exception, match=None): + err = SimpleNamespace(value=None) + try: + yield err + except expected_exception as e: + err.value = e + if match is not None: + import re + + assert re.search(match, e.args[0]) + else: + # This needs to be outside the try/except, so that the helper doesn't + # trick itself into thinking that an AssertionError was thrown. + raise AssertionError( + f"Expected to raise an exception ({expected_exception!r}) but didn't" + ) from None + + +no_shrink = tuple(set(settings.default.phases) - {Phase.shrink}) def flaky(max_runs, min_passes): @@ -67,24 +89,17 @@ class ExcInfo: pass -@contextlib.contextmanager -def raises(exctype): - e = ExcInfo() - try: - yield e - assert False, "Expected to raise an exception but didn't" - except exctype as err: - traceback.print_exc() - e.value = err - return - - def fails_with(e): def accepts(f): @proxies(f) def inverted_test(*arguments, **kwargs): - with raises(e): - f(*arguments, **kwargs) + # Most of these expected-failure tests are non-deterministic, so + # we rig the PRNG to avoid occasional flakiness. We do this outside + # the `raises` context manager so that any problems in rigging the + # PRNG don't accidentally count as the expected failure. + with deterministic_PRNG(): + with raises(e): + f(*arguments, **kwargs) return inverted_test @@ -153,17 +168,24 @@ def _inner(*args, **kwargs): def assert_output_contains_failure(output, test, **kwargs): assert test.__name__ + "(" in output for k, v in kwargs.items(): - assert (f"{k}={v!r}") in output + assert f"{k}={v!r}" in output, (f"{k}={v!r}", output) def assert_falsifying_output( test, example_type="Falsifying", expected_exception=AssertionError, **kwargs ): with capture_out() as out: - with raises(expected_exception): + if expected_exception is None: + # Some tests want to check the output of non-failing runs. test() - - output = out.getvalue() + msg = "" + else: + with raises(expected_exception) as exc_info: + test() + notes = "\n".join(getattr(exc_info.value, "__notes__", [])) + msg = str(exc_info.value) + "\n" + notes + + output = out.getvalue() + msg assert f"{example_type} example:" in output assert_output_contains_failure(output, test, **kwargs) @@ -184,3 +206,11 @@ def temp_registered(type_, strat_or_factory): from_type.__clear_cache() if prev is not None: register_type_strategy(type_, prev) + + +# Specifies whether we can represent subnormal floating point numbers. +# IEE-754 requires subnormal support, but it's often disabled anyway by unsafe +# compiler options like `-ffast-math`. On most hardware that's even a global +# config option, so *linking against* something built this way can break us. +# Everything is terrible +PYTHON_FTZ = next_down(sys.float_info.min) == 0.0 diff --git a/hypothesis-python/tests/conftest.py b/hypothesis-python/tests/conftest.py index 00c46886b2..cb015988fb 100644 --- a/hypothesis-python/tests/conftest.py +++ b/hypothesis-python/tests/conftest.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import gc import random @@ -20,6 +15,7 @@ import pytest +from hypothesis._settings import is_in_ci from hypothesis.internal.detection import is_hypothesis_test from tests.common import TIME_INCREMENT @@ -30,14 +26,24 @@ # Skip collection of tests which require the Django test runner, # or that don't work on the current version of Python. collect_ignore_glob = ["django/*"] -if sys.version_info < (3, 7): - collect_ignore_glob.append("cover/*py37*") if sys.version_info < (3, 8): + collect_ignore_glob.append("array_api") collect_ignore_glob.append("cover/*py38*") +if sys.version_info < (3, 9): + collect_ignore_glob.append("cover/*py39*") +if sys.version_info < (3, 10): + collect_ignore_glob.append("cover/*py310*") + +if sys.version_info >= (3, 11): + collect_ignore_glob.append("cover/test_asyncio.py") # @asyncio.coroutine removed def pytest_configure(config): config.addinivalue_line("markers", "slow: pandas expects this marker to exist.") + config.addinivalue_line( + "markers", + "xp_min_version(api_version): run when greater or equal to api_version", + ) def pytest_addoption(parser): @@ -94,11 +100,17 @@ def pytest_runtest_call(item): # See: https://github.com/HypothesisWorks/hypothesis/issues/1919 if not (hasattr(item, "obj") and is_hypothesis_test(item.obj)): yield + elif "pytest_randomly" in sys.modules: + # See https://github.com/HypothesisWorks/hypothesis/issues/3041 - this + # branch exists to make it easier on external contributors, but should + # never run in our CI (because that would disable the check entirely). + assert not is_in_ci() + yield else: # We start by peturbing the state of the PRNG, because repeatedly # leaking PRNG state resets state_after to the (previously leaked) # state_before, and that just shows as "no use of random". - random.seed(independent_random.randrange(2 ** 32)) + random.seed(independent_random.randrange(2**32)) before = random.getstate() yield after = random.getstate() diff --git a/hypothesis-python/tests/conjecture/__init__.py b/hypothesis-python/tests/conjecture/__init__.py index e61e3c2872..fcb1ac6538 100644 --- a/hypothesis-python/tests/conjecture/__init__.py +++ b/hypothesis-python/tests/conjecture/__init__.py @@ -1,14 +1,9 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER diff --git a/hypothesis-python/tests/conjecture/common.py b/hypothesis-python/tests/conjecture/common.py index a574ebe3cf..51f770c1ca 100644 --- a/hypothesis-python/tests/conjecture/common.py +++ b/hypothesis-python/tests/conjecture/common.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from contextlib import contextmanager diff --git a/hypothesis-python/tests/conjecture/test_choice_tree.py b/hypothesis-python/tests/conjecture/test_choice_tree.py index 40f332e38c..5d566cb445 100644 --- a/hypothesis-python/tests/conjecture/test_choice_tree.py +++ b/hypothesis-python/tests/conjecture/test_choice_tree.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from random import Random diff --git a/hypothesis-python/tests/conjecture/test_data_tree.py b/hypothesis-python/tests/conjecture/test_data_tree.py index bf597d6d67..d869a0ba83 100644 --- a/hypothesis-python/tests/conjecture/test_data_tree.py +++ b/hypothesis-python/tests/conjecture/test_data_tree.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from random import Random diff --git a/hypothesis-python/tests/conjecture/test_dfa.py b/hypothesis-python/tests/conjecture/test_dfa.py index 069ed6f9e8..2ccfb36c1b 100644 --- a/hypothesis-python/tests/conjecture/test_dfa.py +++ b/hypothesis-python/tests/conjecture/test_dfa.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import itertools import math diff --git a/hypothesis-python/tests/conjecture/test_engine.py b/hypothesis-python/tests/conjecture/test_engine.py index 1b9abe0ad9..5177bb20ba 100644 --- a/hypothesis-python/tests/conjecture/test_engine.py +++ b/hypothesis-python/tests/conjecture/test_engine.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import re import time @@ -113,8 +108,6 @@ def accept(data): @pytest.mark.parametrize("n", [1, 5]) def test_terminates_shrinks(n, monkeypatch): - from hypothesis.internal.conjecture import engine - db = InMemoryExampleDatabase() def generate_new_examples(self): @@ -123,7 +116,7 @@ def generate_new_examples(self): monkeypatch.setattr( ConjectureRunner, "generate_new_examples", generate_new_examples ) - monkeypatch.setattr(engine, "MAX_SHRINKS", n) + monkeypatch.setattr(engine_module, "MAX_SHRINKS", n) runner = ConjectureRunner( slow_shrinker(), @@ -157,17 +150,17 @@ def tf(data): def recur(i, data): - try: - if i >= 1: - recur(i - 1, data) - except RecursionError: - data.mark_interesting() + if i >= 1: + recur(i - 1, data) def test_recursion_error_is_not_flaky(): def tf(data): i = data.draw_bits(16) - recur(i, data) + try: + recur(i, data) + except RecursionError: + data.mark_interesting() runner = ConjectureRunner(tf) runner.run() @@ -258,7 +251,7 @@ def test_stops_after_max_examples_when_generating_more_bugs(examples): def f(data): seen.append(data.draw_bits(32)) # Rare, potentially multi-error conditions - if seen[-1] > 2 ** 31: + if seen[-1] > 2**31: bad[0] = True raise ValueError bad[1] = True @@ -309,7 +302,7 @@ def f(data): f, settings=settings(database=None, phases=(Phase.reuse, Phase.generate)) ) runner.run() - assert len(seen) == MIN_TEST_CALLS + assert len(seen) == 1 def test_reuse_phase_runs_for_max_examples_if_generation_is_disabled(): @@ -411,7 +404,7 @@ def accept(f): with pytest.raises(FailedHealthCheck) as e: runner.run() - assert e.value.health_check == label + assert str(label) in str(e.value) assert not runner.interesting_examples return accept @@ -427,14 +420,14 @@ def _(data): def test_fails_health_check_for_large_base(): @fails_health_check(HealthCheck.large_base_example) def _(data): - data.draw_bytes(10 ** 6) + data.draw_bytes(10**6) def test_fails_health_check_for_large_non_base(): @fails_health_check(HealthCheck.data_too_large) def _(data): if data.draw_bits(8): - data.draw_bytes(10 ** 6) + data.draw_bytes(10**6) def test_fails_health_check_for_slow_draws(): @@ -462,7 +455,7 @@ def data(data): def test_run_nothing(): def f(data): - assert False + raise AssertionError runner = ConjectureRunner(f, settings=settings(phases=())) runner.run() @@ -814,7 +807,7 @@ def fast_time(): return val[0] def f(data): - if data.draw_bits(64) > 2 ** 33: + if data.draw_bits(64) > 2**33: data.mark_interesting() monkeypatch.setattr(time, "perf_counter", fast_time) @@ -1234,7 +1227,7 @@ def test(data): runner.run() - assert len(runner.pareto_front) == 2 ** 4 + assert len(runner.pareto_front) == 2**4 def test_pareto_front_contains_smallest_valid_when_not_targeting(): @@ -1276,7 +1269,7 @@ def test(data): runner.run() - assert len(runner.pareto_front) == 2 ** 4 + assert len(runner.pareto_front) == 2**4 def test_database_contains_only_pareto_front(): @@ -1456,7 +1449,7 @@ def test(data): runner.run() - assert runner.best_observed_targets["n"] == (2 ** 16) - 1 + assert runner.best_observed_targets["n"] == (2**16) - 1 def test_runs_optimisation_once_when_generating(): @@ -1594,3 +1587,8 @@ def test(data): runner.cached_test_function([c]) assert runner.tree.is_exhausted + + +def test_can_convert_non_weakref_types_to_event_strings(): + runner = ConjectureRunner(lambda data: None) + runner.event_to_string(()) diff --git a/hypothesis-python/tests/conjecture/test_float_encoding.py b/hypothesis-python/tests/conjecture/test_float_encoding.py index a6b31a4bb1..d35c169738 100644 --- a/hypothesis-python/tests/conjecture/test_float_encoding.py +++ b/hypothesis-python/tests/conjecture/test_float_encoding.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import sys @@ -25,7 +20,7 @@ from hypothesis.internal.floats import float_to_int EXPONENTS = list(range(0, flt.MAX_EXPONENT + 1)) -assert len(EXPONENTS) == 2 ** 11 +assert len(EXPONENTS) == 2**11 def assert_reordered_exponents(res): @@ -57,12 +52,12 @@ def test_encode_decode(): @given(st.data()) def test_double_reverse_bounded(data): n = data.draw(st.integers(1, 64)) - i = data.draw(st.integers(0, 2 ** n - 1)) + i = data.draw(st.integers(0, 2**n - 1)) j = flt.reverse_bits(i, n) assert flt.reverse_bits(j, n) == i -@given(st.integers(0, 2 ** 64 - 1)) +@given(st.integers(0, 2**64 - 1)) def test_double_reverse(i): j = flt.reverse64(i) assert flt.reverse64(j) == i @@ -103,7 +98,7 @@ def test_floats_round_trip(f): @example(1, 0.5) -@given(st.integers(1, 2 ** 53), st.floats(0, 1).filter(lambda x: x not in (0, 1))) +@given(st.integers(1, 2**53), st.floats(0, 1).filter(lambda x: x not in (0, 1))) def test_floats_order_worse_than_their_integral_part(n, g): f = n + g assume(int(f) != f) @@ -162,7 +157,7 @@ def test_function(data): data.mark_interesting() runner = ConjectureRunner(test_function) - runner.cached_test_function(int_to_bytes(flt.float_to_lex(start), 8) + bytes(1)) + runner.cached_test_function(bytes(1) + int_to_bytes(flt.float_to_lex(start), 8)) assert runner.interesting_examples return runner diff --git a/hypothesis-python/tests/conjecture/test_intlist.py b/hypothesis-python/tests/conjecture/test_intlist.py index 881b104ec7..bef7d93542 100644 --- a/hypothesis-python/tests/conjecture/test_intlist.py +++ b/hypothesis-python/tests/conjecture/test_intlist.py @@ -1,24 +1,19 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest from hypothesis import assume, given, strategies as st from hypothesis.internal.conjecture.junkdrawer import IntList -non_neg_lists = st.lists(st.integers(min_value=0, max_value=2 ** 63 - 1)) +non_neg_lists = st.lists(st.integers(min_value=0, max_value=2**63 - 1)) @given(non_neg_lists) @@ -50,6 +45,6 @@ def test_error_on_invalid_value(): def test_extend_by_too_large(): x = IntList() - ls = [1, 10 ** 6] + ls = [1, 10**6] x.extend(ls) assert list(x) == ls diff --git a/hypothesis-python/tests/conjecture/test_junkdrawer.py b/hypothesis-python/tests/conjecture/test_junkdrawer.py index b56beb90eb..f0b0f8f5eb 100644 --- a/hypothesis-python/tests/conjecture/test_junkdrawer.py +++ b/hypothesis-python/tests/conjecture/test_junkdrawer.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import inspect @@ -122,7 +117,7 @@ def test_int_list_cannot_contain_negative(): def test_int_list_can_contain_arbitrary_size(): - n = 2 ** 65 + n = 2**65 assert list(IntList([n])) == [n] @@ -144,7 +139,7 @@ def test_int_list_equality(): def test_int_list_extend(): x = IntList.of_length(3) - n = 2 ** 64 - 1 + n = 2**64 - 1 x.extend([n]) assert list(x) == [0, 0, 0, n] diff --git a/hypothesis-python/tests/conjecture/test_lstar.py b/hypothesis-python/tests/conjecture/test_lstar.py index 5e66337e6c..473f900f06 100644 --- a/hypothesis-python/tests/conjecture/test_lstar.py +++ b/hypothesis-python/tests/conjecture/test_lstar.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import itertools @@ -204,25 +199,6 @@ def test_learning_large_dfa(): assert i == int.from_bytes(s, "big") -@st.composite -def byte_order(draw): - ls = draw(st.permutations(range(256))) - n = draw(st.integers(0, len(ls))) - return ls[:n] - - -@example({0}, [1]) -@given(st.sets(st.integers(0, 255)), byte_order()) -def test_learning_always_changes_generation(chars, order): - learner = LStar(lambda s: len(s) == 1 and s[0] in chars) - for c in order: - prev = learner.generation - s = bytes([c]) - if learner.dfa.matches(s) != learner.member(s): - learner.learn(s) - assert learner.generation > prev - - def varint_predicate(b): if not b: return False @@ -239,7 +215,7 @@ def varint(draw): result.append(draw(st.integers(1, 255))) n = result[0] & 15 assume(n > 0) - value = draw(st.integers(10, 256 ** n - 1)) + value = draw(st.integers(10, 256**n - 1)) result.extend(value.to_bytes(n, "big")) return bytes(result) diff --git a/hypothesis-python/tests/conjecture/test_minimizer.py b/hypothesis-python/tests/conjecture/test_minimizer.py index b32fcd73cd..64513dda88 100644 --- a/hypothesis-python/tests/conjecture/test_minimizer.py +++ b/hypothesis-python/tests/conjecture/test_minimizer.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from collections import Counter from random import Random diff --git a/hypothesis-python/tests/conjecture/test_optimiser.py b/hypothesis-python/tests/conjecture/test_optimiser.py index b0f01dae70..9a0e457c4d 100644 --- a/hypothesis-python/tests/conjecture/test_optimiser.py +++ b/hypothesis-python/tests/conjecture/test_optimiser.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest @@ -106,7 +101,7 @@ def test(data): assert runner.best_observed_targets[""] == 255 -@pytest.mark.parametrize("lower, upper", [(0, 1000), (13, 100), (1000, 2 ** 16 - 1)]) +@pytest.mark.parametrize("lower, upper", [(0, 1000), (13, 100), (1000, 2**16 - 1)]) @pytest.mark.parametrize("score_up", [False, True]) def test_can_find_endpoints_of_a_range(lower, upper, score_up): with deterministic_PRNG(): diff --git a/hypothesis-python/tests/conjecture/test_order_shrinking.py b/hypothesis-python/tests/conjecture/test_order_shrinking.py index 57a0f2f1e1..42046007fa 100644 --- a/hypothesis-python/tests/conjecture/test_order_shrinking.py +++ b/hypothesis-python/tests/conjecture/test_order_shrinking.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from random import Random diff --git a/hypothesis-python/tests/conjecture/test_pareto.py b/hypothesis-python/tests/conjecture/test_pareto.py index ca8346f078..521f2ccf97 100644 --- a/hypothesis-python/tests/conjecture/test_pareto.py +++ b/hypothesis-python/tests/conjecture/test_pareto.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest @@ -41,7 +36,7 @@ def test(data): runner.run() - assert len(runner.pareto_front) == 2 ** 4 + assert len(runner.pareto_front) == 2**4 def test_database_contains_only_pareto_front(): @@ -236,7 +231,7 @@ def test(data): def test_stops_optimising_once_interesting(): - hi = 2 ** 16 - 1 + hi = 2**16 - 1 def test(data): n = data.draw_bits(16) diff --git a/hypothesis-python/tests/conjecture/test_shrinker.py b/hypothesis-python/tests/conjecture/test_shrinker.py index cf8c76c271..0ae9885bcb 100644 --- a/hypothesis-python/tests/conjecture/test_shrinker.py +++ b/hypothesis-python/tests/conjecture/test_shrinker.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import time @@ -318,7 +313,7 @@ def test_float_shrink_can_run_when_canonicalisation_does_not_work(monkeypatch): # This should be an error when called monkeypatch.setattr(Float, "shrink", None) - base_buf = int_to_bytes(flt.base_float_to_lex(1000.0), 8) + bytes(1) + base_buf = bytes(1) + int_to_bytes(flt.base_float_to_lex(1000.0), 8) @shrinking_from(base_buf) def shrinker(data): diff --git a/hypothesis-python/tests/conjecture/test_shrinking_dfas.py b/hypothesis-python/tests/conjecture/test_shrinking_dfas.py index 7fb78a1cbd..b8dfedac7a 100644 --- a/hypothesis-python/tests/conjecture/test_shrinking_dfas.py +++ b/hypothesis-python/tests/conjecture/test_shrinking_dfas.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import os import sys diff --git a/hypothesis-python/tests/conjecture/test_shrinking_interface.py b/hypothesis-python/tests/conjecture/test_shrinking_interface.py index f565116983..247b72d8e1 100644 --- a/hypothesis-python/tests/conjecture/test_shrinking_interface.py +++ b/hypothesis-python/tests/conjecture/test_shrinking_interface.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from random import Random diff --git a/hypothesis-python/tests/conjecture/test_test_data.py b/hypothesis-python/tests/conjecture/test_test_data.py index b8ab34254e..346063452c 100644 --- a/hypothesis-python/tests/conjecture/test_test_data.py +++ b/hypothesis-python/tests/conjecture/test_test_data.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import itertools @@ -111,7 +106,7 @@ def test_closes_interval_on_error_in_strategy(): class BigStrategy(SearchStrategy): def do_draw(self, data): - data.draw_bytes(10 ** 6) + data.draw_bytes(10**6) def test_does_not_double_freeze_in_interval_close(): @@ -338,7 +333,7 @@ def test_will_mark_too_deep_examples_as_invalid(): s = st.none() for _ in range(MAX_DEPTH + 1): - s = s.map(lambda x: x) + s = s.map(lambda x: None) with pytest.raises(StopTest): d.draw(s) diff --git a/hypothesis-python/tests/conjecture/test_utils.py b/hypothesis-python/tests/conjecture/test_utils.py index fbb837d69e..595f5dece2 100644 --- a/hypothesis-python/tests/conjecture/test_utils.py +++ b/hypothesis-python/tests/conjecture/test_utils.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from collections import Counter from fractions import Fraction @@ -112,7 +107,7 @@ def test_drawing_an_exact_fraction_coin(): def test_too_small_to_be_useful_coin(): - assert not cu.biased_coin(ConjectureData.for_buffer([1]), 0.5 ** 65) + assert not cu.biased_coin(ConjectureData.for_buffer([1]), 0.5**65) @example([Fraction(1, 3), Fraction(1, 3), Fraction(1, 3)]) @@ -124,7 +119,7 @@ def test_too_small_to_be_useful_coin(): @settings( deadline=None, suppress_health_check=HealthCheck.all(), - phases=[Phase.explicit] if IN_COVERAGE_TESTS else tuple(Phase), + phases=[Phase.explicit] if IN_COVERAGE_TESTS else settings.default.phases, ) @given(st.lists(st.fractions(min_value=0, max_value=1), min_size=1)) def test_sampler_distribution(weights): @@ -200,7 +195,7 @@ def test_center_in_middle_above(): def test_restricted_bits(): assert ( cu.integer_range( - ConjectureData.for_buffer([1, 0, 0, 0, 0]), lower=0, upper=2 ** 64 - 1 + ConjectureData.for_buffer([1, 0, 0, 0, 0]), lower=0, upper=2**64 - 1 ) == 0 ) @@ -253,6 +248,14 @@ def test_fixed_size_draw_many(): assert not many.more() +def test_astronomically_unlikely_draw_many(): + # Our internal helper doesn't underflow to zero or negative, but nor + # will we ever generate an element for such a low average size. + buffer = ConjectureData.for_buffer(1024 * [255]) + many = cu.many(buffer, min_size=0, max_size=10, average_size=1e-5) + assert not many.more() + + def test_rejection_eventually_terminates_many(): many = cu.many( ConjectureData.for_buffer([1] * 1000), @@ -318,5 +321,9 @@ def test_can_draw_arbitrary_fractions(p, b): def test_samples_from_a_range_directly(): - s = cu.check_sample(range(10 ** 1000), "") + s = cu.check_sample(range(10**1000), "") assert isinstance(s, range) + + +def test_p_continue_to_average_saturates(): + assert cu._p_continue_to_avg(1.1, 100) == 100 diff --git a/hypothesis-python/tests/cover/__init__.py b/hypothesis-python/tests/cover/__init__.py index e61e3c2872..fcb1ac6538 100644 --- a/hypothesis-python/tests/cover/__init__.py +++ b/hypothesis-python/tests/cover/__init__.py @@ -1,14 +1,9 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER diff --git a/hypothesis-python/tests/cover/test_annotations.py b/hypothesis-python/tests/cover/test_annotations.py index 95ebea3c54..d073c12645 100644 --- a/hypothesis-python/tests/cover/test_annotations.py +++ b/hypothesis-python/tests/cover/test_annotations.py @@ -1,25 +1,19 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER -from inspect import getfullargspec +from inspect import Parameter as P, signature import attr import pytest from hypothesis import given, strategies as st -from hypothesis.errors import InvalidArgument from hypothesis.internal.reflection import ( convert_positional_arguments, define_function_signature, @@ -41,17 +35,11 @@ def has_annotation(a: int, *b, c=2) -> None: @pytest.mark.parametrize("f", [has_annotation, lambda *, a: a, lambda *, a=1: a]) -def test_copying_preserves_argspec(f): - af = getfullargspec(f) +def test_copying_preserves_signature(f): + af = signature(f) t = define_function_signature("foo", "docstring", af)(universal_acceptor) - at = getfullargspec(t) - assert af.args == at.args[: len(af.args)] - assert af.varargs == at.varargs - assert af.varkw == at.varkw - assert len(af.defaults or ()) == len(at.defaults or ()) - assert af.kwonlyargs == at.kwonlyargs - assert af.kwonlydefaults == at.kwonlydefaults - assert af.annotations == at.annotations + at = signature(t) + assert af.parameters == at.parameters @pytest.mark.parametrize( @@ -61,6 +49,7 @@ def test_copying_preserves_argspec(f): ((lambda *z, a=1: a), "lambda *z, a=1: a"), ((lambda *, a: a), "lambda *, a: a"), ((lambda *, a=1: a), "lambda *, a=1: a"), + ((lambda **kw: kw), "lambda **kw: kw"), ], ) def test_kwonly_lambda_formatting(lam, source): @@ -73,8 +62,9 @@ def test_given_notices_missing_kwonly_args(): def reqs_kwonly(*, a, b): pass - with pytest.raises(InvalidArgument): + with pytest.raises(TypeError): reqs_kwonly() + reqs_kwonly(b=None) def test_converter_handles_kwonly_args(): @@ -82,7 +72,7 @@ def f(*, a, b=2): pass out = convert_positional_arguments(f, (), {"a": 1}) - assert out == ((), {"a": 1, "b": 2}) + assert out == ((), {"a": 1}) def test_converter_notices_missing_kwonly_args(): @@ -93,8 +83,8 @@ def f(*, a, b=2): assert convert_positional_arguments(f, (), {}) -def pointless_composite(draw: None, strat: bool, nothing: list) -> int: - return 3 +def to_wrap_with_composite(draw: None, strat: bool, nothing: list) -> int: + return draw(st.none()) def return_annot() -> int: @@ -106,17 +96,18 @@ def first_annot(draw: None): def test_composite_edits_annotations(): - spec_comp = getfullargspec(st.composite(pointless_composite)) - assert spec_comp.annotations["return"] == int - assert "nothing" in spec_comp.annotations - assert "draw" not in spec_comp.annotations + sig_comp = signature(st.composite(to_wrap_with_composite)) + assert sig_comp.return_annotation == st.SearchStrategy[int] + assert sig_comp.parameters["nothing"].annotation is not P.empty + assert "draw" not in sig_comp.parameters @pytest.mark.parametrize("nargs", [1, 2, 3]) def test_given_edits_annotations(nargs): - spec_given = getfullargspec(given(*(nargs * [st.none()]))(pointless_composite)) - assert spec_given.annotations.pop("return") is None - assert len(spec_given.annotations) == 3 - nargs + sig_given = signature(given(*(nargs * [st.none()]))(to_wrap_with_composite)) + assert sig_given.return_annotation is None + assert len(sig_given.parameters) == 3 - nargs + assert all(p.annotation is not P.empty for p in sig_given.parameters.values()) def a_converter(x) -> int: diff --git a/hypothesis-python/tests/cover/test_arbitrary_data.py b/hypothesis-python/tests/cover/test_arbitrary_data.py index 649580a334..25ea056bf2 100644 --- a/hypothesis-python/tests/cover/test_arbitrary_data.py +++ b/hypothesis-python/tests/cover/test_arbitrary_data.py @@ -1,25 +1,19 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest +from pytest import raises -from hypothesis import find, given, reporting, strategies as st +from hypothesis import find, given, strategies as st from hypothesis.errors import InvalidArgument -from tests.common.utils import capture_out, raises - @given(st.integers(), st.data()) def test_conditional_draw(x, data): @@ -36,13 +30,10 @@ def test(data): if y in x: raise ValueError() - with raises(ValueError): - with capture_out() as out: - with reporting.with_reporter(reporting.default): - test() - result = out.getvalue() - assert "Draw 1: [0, 0]" in result - assert "Draw 2: 0" in result + with raises(ValueError) as err: + test() + assert "Draw 1: [0, 0]" in err.value.__notes__ + assert "Draw 2: 0" in err.value.__notes__ def test_prints_labels_if_given_on_failure(): @@ -54,13 +45,10 @@ def test(data): x.remove(y) assert y not in x - with raises(AssertionError): - with capture_out() as out: - with reporting.with_reporter(reporting.default): - test() - result = out.getvalue() - assert "Draw 1 (Some numbers): [0, 0]" in result - assert "Draw 2 (A number): 0" in result + with raises(AssertionError) as err: + test() + assert "Draw 1 (Some numbers): [0, 0]" in err.value.__notes__ + assert "Draw 2 (A number): 0" in err.value.__notes__ def test_given_twice_is_same(): @@ -70,13 +58,10 @@ def test(data1, data2): data2.draw(st.integers()) raise ValueError() - with raises(ValueError): - with capture_out() as out: - with reporting.with_reporter(reporting.default): - test() - result = out.getvalue() - assert "Draw 1: 0" in result - assert "Draw 2: 0" in result + with raises(ValueError) as err: + test() + assert "Draw 1: 0" in err.value.__notes__ + assert "Draw 2: 0" in err.value.__notes__ def test_errors_when_used_in_find(): diff --git a/hypothesis-python/tests/cover/test_async_def.py b/hypothesis-python/tests/cover/test_async_def.py index 128fef6818..d0b8bcdf84 100644 --- a/hypothesis-python/tests/cover/test_async_def.py +++ b/hypothesis-python/tests/cover/test_async_def.py @@ -1,24 +1,16 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import asyncio -import sys from unittest import TestCase -import pytest - from hypothesis import assume, given, strategies as st @@ -29,7 +21,6 @@ class TestAsyncioRun(TestCase): def execute_example(self, f): asyncio.run(f()) - @pytest.mark.skipif(sys.version_info[:2] < (3, 7), reason="asyncio.run() is new") @given(st.text()) async def test_foo(self, x): assume(x) diff --git a/hypothesis-python/tests/cover/test_asyncio.py b/hypothesis-python/tests/cover/test_asyncio.py index ec418d08ea..5fc1bc3683 100644 --- a/hypothesis-python/tests/cover/test_asyncio.py +++ b/hypothesis-python/tests/cover/test_asyncio.py @@ -1,20 +1,16 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import asyncio import sys +import warnings from unittest import TestCase import pytest @@ -25,7 +21,11 @@ if sys.version_info < (3, 8): coro_decorator = asyncio.coroutine else: - coro_decorator = pytest.mark.skip + + def coro_decorator(f): + with warnings.catch_warnings(): + warnings.simplefilter(action="ignore", category=DeprecationWarning) + return asyncio.coroutine(f) class TestAsyncio(TestCase): @@ -51,14 +51,14 @@ def g(): except BaseException as e: error = e - coro = asyncio.coroutine(g) + coro = coro_decorator(g) future = asyncio.wait_for(coro(), timeout=self.timeout) self.loop.run_until_complete(future) if error is not None: raise error @pytest.mark.skipif(PYPY, reason="Error in asyncio.new_event_loop()") - @given(st.text()) + @given(x=st.text()) @coro_decorator def test_foo(self, x): assume(x) @@ -67,14 +67,10 @@ def test_foo(self, x): class TestAsyncioRun(TestCase): - - timeout = 5 - def execute_example(self, f): asyncio.run(f()) - @pytest.mark.skipif(sys.version_info[:2] < (3, 7), reason="asyncio.run() is new") - @given(st.text()) + @given(x=st.text()) @coro_decorator def test_foo(self, x): assume(x) diff --git a/hypothesis-python/tests/cover/test_attrs_inference.py b/hypothesis-python/tests/cover/test_attrs_inference.py index a25602cdf2..e4fe7dca74 100644 --- a/hypothesis-python/tests/cover/test_attrs_inference.py +++ b/hypothesis-python/tests/cover/test_attrs_inference.py @@ -1,24 +1,19 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import typing import attr import pytest -from hypothesis import given, infer, strategies as st +from hypothesis import given, strategies as st from hypothesis.errors import ResolutionFailed @@ -75,7 +70,7 @@ class UnhelpfulConverter: a = attr.ib(converter=lambda x: x) -@given(st.builds(Inferrables, has_default=infer, has_default_factory=infer)) +@given(st.builds(Inferrables, has_default=..., has_default_factory=...)) def test_attrs_inference_builds(c): pass @@ -93,4 +88,4 @@ def test_cannot_infer(c): def test_cannot_infer_takes_self(): with pytest.raises(ResolutionFailed): - st.builds(Inferrables, has_default_factory_takes_self=infer).example() + st.builds(Inferrables, has_default_factory_takes_self=...).example() diff --git a/hypothesis-python/tests/cover/test_cache_implementation.py b/hypothesis-python/tests/cover/test_cache_implementation.py index 7d61509de8..9360f0a70b 100644 --- a/hypothesis-python/tests/cover/test_cache_implementation.py +++ b/hypothesis-python/tests/cover/test_cache_implementation.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import threading from functools import partial @@ -321,3 +316,19 @@ def target(): worker.join() assert not errors + + +def test_pin_and_unpin_are_noops_if_dropped(): + # See https://github.com/HypothesisWorks/hypothesis/issues/3169 + cache = LRUReusedCache(max_size=10) + cache[30] = True + assert 30 in cache + + for i in range(20): + cache[i] = False + + assert 30 not in cache + cache.pin(30) + assert 30 not in cache + cache.unpin(30) + assert 30 not in cache diff --git a/hypothesis-python/tests/cover/test_caching.py b/hypothesis-python/tests/cover/test_caching.py index 751a7da308..8b84de7208 100644 --- a/hypothesis-python/tests/cover/test_caching.py +++ b/hypothesis-python/tests/cover/test_caching.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest diff --git a/hypothesis-python/tests/cover/test_cathetus.py b/hypothesis-python/tests/cover/test_cathetus.py index e3323e1b31..11324cee3d 100644 --- a/hypothesis-python/tests/cover/test_cathetus.py +++ b/hypothesis-python/tests/cover/test_cathetus.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import math import sys diff --git a/hypothesis-python/tests/cover/test_charmap.py b/hypothesis-python/tests/cover/test_charmap.py index fb15ac21f8..4c6a64d46a 100644 --- a/hypothesis-python/tests/cover/test_charmap.py +++ b/hypothesis-python/tests/cover/test_charmap.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import os import sys diff --git a/hypothesis-python/tests/cover/test_compat.py b/hypothesis-python/tests/cover/test_compat.py new file mode 100644 index 0000000000..673b19e07a --- /dev/null +++ b/hypothesis-python/tests/cover/test_compat.py @@ -0,0 +1,47 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import math +from inspect import Parameter, Signature + +import pytest + +from hypothesis.internal.compat import ceil, floor, get_type_hints + +floor_ceil_values = [ + -10.7, + -10.3, + -0.5, + -0.0, + 0, + 0.5, + 10.3, + 10.7, +] + + +@pytest.mark.parametrize("value", floor_ceil_values) +def test_our_floor_agrees_with_math_floor(value): + assert floor(value) == math.floor(value) + + +@pytest.mark.parametrize("value", floor_ceil_values) +def test_our_ceil_agrees_with_math_ceil(value): + assert ceil(value) == math.ceil(value) + + +class WeirdSig: + __signature__ = Signature( + parameters=[Parameter(name="args", kind=Parameter.VAR_POSITIONAL)] + ) + + +def test_no_type_hints(): + assert get_type_hints(WeirdSig) == {} diff --git a/hypothesis-python/tests/cover/test_complex_numbers.py b/hypothesis-python/tests/cover/test_complex_numbers.py index f942a0b047..90dc65cab5 100644 --- a/hypothesis-python/tests/cover/test_complex_numbers.py +++ b/hypothesis-python/tests/cover/test_complex_numbers.py @@ -1,25 +1,23 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import math import sys +import pytest + from hypothesis import given, reject, strategies as st +from hypothesis.errors import InvalidArgument from hypothesis.strategies import complex_numbers -from tests.common.debug import minimal +from tests.common.debug import assert_no_examples, find_any, minimal def test_minimal(): @@ -50,7 +48,7 @@ def test_minimal_quadrant4(): assert minimal(complex_numbers(), lambda x: x.imag < 0 and x.real > 0) == 1 - 1j -@given(st.data(), st.integers(-5, 5).map(lambda x: 10 ** x)) +@given(st.data(), st.integers(-5, 5).map(lambda x: 10**x)) def test_max_magnitude_respected(data, mag): c = data.draw(complex_numbers(max_magnitude=mag)) assert abs(c) <= mag * (1 + sys.float_info.epsilon) @@ -61,7 +59,7 @@ def test_max_magnitude_zero(val): assert val == 0 -@given(st.data(), st.integers(-5, 5).map(lambda x: 10 ** x)) +@given(st.data(), st.integers(-5, 5).map(lambda x: 10**x)) def test_min_magnitude_respected(data, mag): c = data.draw(complex_numbers(min_magnitude=mag)) assert ( @@ -94,3 +92,36 @@ def test_minmax_magnitude_equal(data, mag): assert math.isclose(abs(val), mag) except OverflowError: reject() + + +def _is_subnormal(x): + return 0 < abs(x) < sys.float_info.min + + +@pytest.mark.parametrize( + "allow_subnormal, min_magnitude, max_magnitude", + [ + (True, 0, None), + (True, 1, None), + (False, 0, None), + ], +) +def test_allow_subnormal(allow_subnormal, min_magnitude, max_magnitude): + strat = complex_numbers( + min_magnitude=min_magnitude, + max_magnitude=max_magnitude, + allow_subnormal=allow_subnormal, + ).filter(lambda x: x.real != 0 and x.imag != 0) + + if allow_subnormal: + find_any(strat, lambda x: _is_subnormal(x.real) or _is_subnormal(x.imag)) + else: + assert_no_examples( + strat, lambda x: _is_subnormal(x.real) or _is_subnormal(x.imag) + ) + + +@pytest.mark.parametrize("allow_subnormal", [1, 0.0, "False"]) +def test_allow_subnormal_validation(allow_subnormal): + with pytest.raises(InvalidArgument): + complex_numbers(allow_subnormal=allow_subnormal).example() diff --git a/hypothesis-python/tests/cover/test_composite.py b/hypothesis-python/tests/cover/test_composite.py index fa45270c4a..6b19fd3a4c 100644 --- a/hypothesis-python/tests/cover/test_composite.py +++ b/hypothesis-python/tests/cover/test_composite.py @@ -1,22 +1,17 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest from hypothesis import assume, given, strategies as st -from hypothesis.errors import InvalidArgument +from hypothesis.errors import HypothesisDeprecationWarning, InvalidArgument from tests.common.debug import minimal from tests.common.utils import flaky @@ -80,6 +75,14 @@ def foo(**kwargs): pass +def test_warning_given_no_drawfn_call(): + with pytest.warns(HypothesisDeprecationWarning): + + @st.composite + def foo(_): + return "bar" + + def test_can_use_pure_args(): @st.composite def stuff(*args): @@ -127,6 +130,7 @@ def test_does_not_change_arguments(data, ls): # regression test for issue #1017 or other argument mutation @st.composite def strat(draw, arg): + draw(st.none()) return arg ex = data.draw(strat(ls)) @@ -136,12 +140,12 @@ def strat(draw, arg): class ClsWithStrategyMethods: @classmethod @st.composite - def st_classmethod_then_composite(draw, cls): + def st_classmethod_then_composite(draw, cls): # noqa: B902 return draw(st.integers(0, 10)) @st.composite @classmethod - def st_composite_then_classmethod(draw, cls): + def st_composite_then_classmethod(draw, cls): # noqa: B902 return draw(st.integers(0, 10)) @staticmethod @@ -155,7 +159,7 @@ def st_composite_then_staticmethod(draw): return draw(st.integers(0, 10)) @st.composite - def st_composite_method(draw, self): + def st_composite_method(draw, self): # noqa: B902 return draw(st.integers(0, 10)) @@ -176,3 +180,8 @@ def test_applying_composite_decorator_to_methods(data): x = data.draw(strategy) assert isinstance(x, int) assert 0 <= x <= 10 + + +def test_drawfn_cannot_be_instantiated(): + with pytest.raises(TypeError): + st.DrawFn() diff --git a/hypothesis-python/tests/cover/test_composite_kwonlyargs.py b/hypothesis-python/tests/cover/test_composite_kwonlyargs.py index 641d284b18..d9373affbe 100644 --- a/hypothesis-python/tests/cover/test_composite_kwonlyargs.py +++ b/hypothesis-python/tests/cover/test_composite_kwonlyargs.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import given, strategies as st @@ -31,4 +26,4 @@ def kwonlyargs_composites(draw, *, kwarg1=None): ) ) def test_composite_with_keyword_only_args(a): - assert True + pass diff --git a/hypothesis-python/tests/cover/test_control.py b/hypothesis-python/tests/cover/test_control.py index c02c4bd52d..41c1eb1c55 100644 --- a/hypothesis-python/tests/cover/test_control.py +++ b/hypothesis-python/tests/cover/test_control.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest @@ -25,7 +20,8 @@ event, note, ) -from hypothesis.errors import CleanupFailed, InvalidArgument +from hypothesis.errors import InvalidArgument +from hypothesis.internal.compat import ExceptionGroup from hypothesis.internal.conjecture.data import ConjectureData as TD from hypothesis.stateful import RuleBasedStateMachine, rule from hypothesis.strategies import integers @@ -73,49 +69,46 @@ def test_can_nest_build_context(): def test_does_not_suppress_exceptions(): with pytest.raises(AssertionError): with bc(): - assert False + raise AssertionError assert _current_build_context.value is None def test_suppresses_exceptions_in_teardown(): - with capture_out() as o: - with pytest.raises(AssertionError): - with bc(): + with pytest.raises(ValueError) as exc_info: + with bc(): - def foo(): - raise ValueError() + def foo(): + raise ValueError - cleanup(foo) - assert False + cleanup(foo) + raise AssertionError - assert "ValueError" in o.getvalue() - assert _current_build_context.value is None + assert isinstance(exc_info.value, ValueError) + assert isinstance(exc_info.value.__cause__, AssertionError) def test_runs_multiple_cleanup_with_teardown(): - with capture_out() as o: - with pytest.raises(AssertionError): - with bc(): - - def foo(): - raise ValueError() + with pytest.raises(ExceptionGroup) as exc_info: + with bc(): - cleanup(foo) + def foo(): + raise ValueError - def bar(): - raise TypeError() + def bar(): + raise TypeError - cleanup(foo) - cleanup(bar) - assert False + cleanup(foo) + cleanup(bar) + raise AssertionError - assert "ValueError" in o.getvalue() - assert "TypeError" in o.getvalue() + assert isinstance(exc_info.value, ExceptionGroup) + assert isinstance(exc_info.value.__cause__, AssertionError) + assert {type(e) for e in exc_info.value.exceptions} == {ValueError, TypeError} assert _current_build_context.value is None def test_raises_error_if_cleanup_fails_but_block_does_not(): - with pytest.raises(CleanupFailed): + with pytest.raises(ValueError): with bc(): def foo(): diff --git a/hypothesis-python/tests/cover/test_core.py b/hypothesis-python/tests/cover/test_core.py index 247462bf0f..c9b25987dd 100644 --- a/hypothesis-python/tests/cover/test_core.py +++ b/hypothesis-python/tests/cover/test_core.py @@ -1,22 +1,20 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER + +import unittest import pytest from _pytest.outcomes import Failed, Skipped -from hypothesis import find, given, reject, settings, strategies as s +from hypothesis import Phase, example, find, given, reject, settings, strategies as s +from hypothesis.database import InMemoryExampleDatabase from hypothesis.errors import InvalidArgument, NoSuchExample, Unsatisfiable @@ -116,3 +114,32 @@ def test_method_with_bad_strategy(self, x): instance = TestStrategyValidation() with pytest.raises(InvalidArgument): instance.test_method_with_bad_strategy() + + +@example(1) +@given(s.integers()) +@settings(phases=[Phase.target, Phase.shrink, Phase.explain]) +def no_phases(_): + raise Exception + + +@given(s.integers()) +@settings(phases=[Phase.explicit]) +def no_explicit(_): + raise Exception + + +@given(s.integers()) +@settings(phases=[Phase.reuse], database=InMemoryExampleDatabase()) +def empty_db(_): + raise Exception + + +@pytest.mark.parametrize( + "test_fn", + [no_phases, no_explicit, empty_db], + ids=lambda t: t.__name__, +) +def test_non_executed_tests_raise_skipped(test_fn): + with pytest.raises(unittest.SkipTest): + test_fn() diff --git a/hypothesis-python/tests/cover/test_custom_reprs.py b/hypothesis-python/tests/cover/test_custom_reprs.py index 6b1f032d45..4669d3ee92 100644 --- a/hypothesis-python/tests/cover/test_custom_reprs.py +++ b/hypothesis-python/tests/cover/test_custom_reprs.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest @@ -24,7 +19,7 @@ def test_includes_non_default_args_in_repr(): def test_sampled_repr_leaves_range_as_range(): - huge = 10 ** 100 + huge = 10**100 assert repr(st.sampled_from(range(huge))) == f"sampled_from(range(0, {huge}))" diff --git a/hypothesis-python/tests/cover/test_database_backend.py b/hypothesis-python/tests/cover/test_database_backend.py index c09a580c1f..8e0c4249f5 100644 --- a/hypothesis-python/tests/cover/test_database_backend.py +++ b/hypothesis-python/tests/cover/test_database_backend.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import os @@ -78,9 +73,8 @@ def test_does_not_error_when_fetching_when_not_exist(tmpdir): def exampledatabase(request, tmpdir): if request.param == "memory": return ExampleDatabase() - if request.param == "directory": - return DirectoryBasedExampleDatabase(str(tmpdir.join("examples"))) - assert False + assert request.param == "directory" + return DirectoryBasedExampleDatabase(str(tmpdir.join("examples"))) def test_can_delete_a_key_that_is_not_present(exampledatabase): diff --git a/hypothesis-python/tests/cover/test_datetimes.py b/hypothesis-python/tests/cover/test_datetimes.py index c41413aaee..9062501d7d 100644 --- a/hypothesis-python/tests/cover/test_datetimes.py +++ b/hypothesis-python/tests/cover/test_datetimes.py @@ -1,24 +1,18 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import datetime as dt import pytest from hypothesis import given, settings -from hypothesis.internal.compat import PYPY from hypothesis.strategies import dates, datetimes, timedeltas, times from tests.common.debug import find_any, minimal @@ -30,7 +24,7 @@ def test_can_find_positive_delta(): def test_can_find_negative_delta(): assert minimal( - timedeltas(max_value=dt.timedelta(10 ** 6)), lambda x: x.days < 0 + timedeltas(max_value=dt.timedelta(10**6)), lambda x: x.days < 0 ) == dt.timedelta(-1) @@ -93,7 +87,7 @@ def test_can_find_before_the_year_2000(): @pytest.mark.parametrize("month", range(1, 13)) def test_can_find_each_month(month): - find_any(dates(), lambda x: x.month == month, settings(max_examples=10 ** 6)) + find_any(dates(), lambda x: x.month == month, settings(max_examples=10**6)) def test_min_year_is_respected(): @@ -139,13 +133,10 @@ def test_naive_times_are_naive(dt): assert dt.tzinfo is None -# pypy3.6 seems to canonicalise fold to 0 for non-ambiguous times? -@pytest.mark.skipif(PYPY, reason="see comment") def test_can_generate_datetime_with_fold_1(): find_any(datetimes(), lambda d: d.fold) -@pytest.mark.skipif(PYPY, reason="see comment") def test_can_generate_time_with_fold_1(): find_any(times(), lambda d: d.fold) diff --git a/hypothesis-python/tests/cover/test_deadline.py b/hypothesis-python/tests/cover/test_deadline.py index 1d97be0616..d7927d9560 100644 --- a/hypothesis-python/tests/cover/test_deadline.py +++ b/hypothesis-python/tests/cover/test_deadline.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import time @@ -20,7 +15,7 @@ from hypothesis import given, settings, strategies as st from hypothesis.errors import DeadlineExceeded, Flaky, InvalidArgument -from tests.common.utils import assert_falsifying_output, capture_out, fails_with +from tests.common.utils import assert_falsifying_output, fails_with def test_raises_deadline_on_slow_test(): @@ -114,11 +109,10 @@ def slow_once(i): once[0] = False time.sleep(0.2) - with capture_out() as o: - with pytest.raises(Flaky): - slow_once() - assert "Unreliable test timing" in o.getvalue() - assert "took 2" in o.getvalue() + with pytest.raises(Flaky) as err: + slow_once() + assert "Unreliable test timing" in "\n".join(err.value.__notes__) + assert "took 2" in "\n".join(err.value.__notes__) @pytest.mark.parametrize("slow_strategy", [False, True]) diff --git a/hypothesis-python/tests/cover/test_debug_information.py b/hypothesis-python/tests/cover/test_debug_information.py index 576493474b..7d96a3c5f8 100644 --- a/hypothesis-python/tests/cover/test_debug_information.py +++ b/hypothesis-python/tests/cover/test_debug_information.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import re diff --git a/hypothesis-python/tests/cover/test_deferred_strategies.py b/hypothesis-python/tests/cover/test_deferred_strategies.py index 42cf076656..1621604fea 100644 --- a/hypothesis-python/tests/cover/test_deferred_strategies.py +++ b/hypothesis-python/tests/cover/test_deferred_strategies.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest diff --git a/hypothesis-python/tests/cover/test_detection.py b/hypothesis-python/tests/cover/test_detection.py index 2881090882..06db3d9698 100644 --- a/hypothesis-python/tests/cover/test_detection.py +++ b/hypothesis-python/tests/cover/test_detection.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import given from hypothesis.internal.detection import is_hypothesis_test @@ -28,7 +23,7 @@ def foo(): def test_methods_default_to_not_tests(): class Foo: - def foo(): + def foo(self): pass assert not is_hypothesis_test(Foo().foo) diff --git a/hypothesis-python/tests/cover/test_direct_strategies.py b/hypothesis-python/tests/cover/test_direct_strategies.py index ab64b7750d..fec2c37685 100644 --- a/hypothesis-python/tests/cover/test_direct_strategies.py +++ b/hypothesis-python/tests/cover/test_direct_strategies.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import collections import decimal @@ -51,7 +46,7 @@ def fn_ktest(*fnkwargs): return pytest.mark.parametrize( ("fn", "kwargs"), fnkwargs, - ids=["{}(**{})".format(fn.__name__, pretty(kwargs)) for fn, kwargs in fnkwargs], + ids=[f"{fn.__name__}(**{pretty(kwargs)})" for fn, kwargs in fnkwargs], ) @@ -127,6 +122,12 @@ def fn_ktest(*fnkwargs): (ds.text, {"min_size": 10, "max_size": 9}), (ds.text, {"alphabet": [1]}), (ds.text, {"alphabet": ["abc"]}), + (ds.text, {"alphabet": ds.just("abc")}), + (ds.text, {"alphabet": ds.sampled_from(["abc", "def"])}), + (ds.text, {"alphabet": ds.just(123)}), + (ds.text, {"alphabet": ds.sampled_from([123, 456])}), + (ds.text, {"alphabet": ds.builds(lambda: "abc")}), + (ds.text, {"alphabet": ds.builds(lambda: 123)}), (ds.binary, {"min_size": 10, "max_size": 9}), (ds.floats, {"min_value": math.nan}), (ds.floats, {"min_value": "0"}), @@ -252,7 +253,9 @@ def test_validates_keyword_arguments(fn, kwargs): (ds.text, {"alphabet": "abc"}), (ds.text, {"alphabet": set("abc")}), (ds.text, {"alphabet": ""}), + (ds.text, {"alphabet": ds.just("a")}), (ds.text, {"alphabet": ds.sampled_from("abc")}), + (ds.text, {"alphabet": ds.builds(lambda: "a")}), (ds.characters, {"whitelist_categories": ["N"]}), (ds.characters, {"blacklist_categories": []}), (ds.ip_addresses, {}), @@ -298,13 +301,13 @@ def test_build_class_with_target_kwarg(): def test_builds_raises_with_no_target(): - with pytest.raises(InvalidArgument): + with pytest.raises(TypeError): ds.builds().example() @pytest.mark.parametrize("non_callable", [1, "abc", ds.integers()]) def test_builds_raises_if_non_callable_as_target_kwarg(non_callable): - with pytest.raises(InvalidArgument): + with pytest.raises(TypeError): ds.builds(target=non_callable).example() @@ -363,15 +366,13 @@ def test_decimal_is_in_bounds(x): def test_float_can_find_max_value_inf(): - assert minimal(ds.floats(max_value=math.inf), lambda x: math.isinf(x)) == float( - "inf" - ) - assert minimal(ds.floats(min_value=0.0), lambda x: math.isinf(x)) == math.inf + assert minimal(ds.floats(max_value=math.inf), math.isinf) == float("inf") + assert minimal(ds.floats(min_value=0.0), math.isinf) == math.inf def test_float_can_find_min_value_inf(): minimal(ds.floats(), lambda x: x < 0 and math.isinf(x)) - minimal(ds.floats(min_value=-math.inf, max_value=0.0), lambda x: math.isinf(x)) + minimal(ds.floats(min_value=-math.inf, max_value=0.0), math.isinf) def test_can_find_none_list(): @@ -484,7 +485,7 @@ def test_chained_filter(x): def test_chained_filter_tracks_all_conditions(): s = ds.integers().filter(bool).filter(lambda x: x % 3) - assert len(s.flat_conditions) == 2 + assert len(s.wrapped_strategy.flat_conditions) == 2 @pytest.mark.parametrize("version", [4, 6]) diff --git a/hypothesis-python/tests/cover/test_draw_example.py b/hypothesis-python/tests/cover/test_draw_example.py index f910c2e44a..8fd5c16672 100644 --- a/hypothesis-python/tests/cover/test_draw_example.py +++ b/hypothesis-python/tests/cover/test_draw_example.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest diff --git a/hypothesis-python/tests/cover/test_error_in_draw.py b/hypothesis-python/tests/cover/test_error_in_draw.py index 03499f4a9e..08fb8fe981 100644 --- a/hypothesis-python/tests/cover/test_error_in_draw.py +++ b/hypothesis-python/tests/cover/test_error_in_draw.py @@ -1,23 +1,17 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest from hypothesis import given, strategies as st - -from tests.common.utils import capture_out +from hypothesis.errors import HypothesisWarning def test_error_is_in_finally(): @@ -28,8 +22,17 @@ def test(d): finally: raise ValueError() - with capture_out() as o: - with pytest.raises(ValueError): - test() + with pytest.raises(ValueError) as err: + test() + + assert "[0, 1, -1]" in "\n".join(err.value.__notes__) + - assert "[0, 1, -1]" in o.getvalue() +@given(st.data()) +def test_warns_on_bool_strategy(data): + with pytest.warns( + HypothesisWarning, + match=r"bool\(.+\) is always True, did you mean to draw a value\?", + ): + if st.booleans(): # 'forgot' to draw from the strategy + pass diff --git a/hypothesis-python/tests/cover/test_escalation.py b/hypothesis-python/tests/cover/test_escalation.py index 9474ea2b15..e8ad8623fe 100644 --- a/hypothesis-python/tests/cover/test_escalation.py +++ b/hypothesis-python/tests/cover/test_escalation.py @@ -1,29 +1,26 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import os import pytest import hypothesis +from hypothesis import errors from hypothesis.internal import escalation as esc +from hypothesis.internal.compat import BaseExceptionGroup def test_does_not_escalate_errors_in_non_hypothesis_file(): try: - assert False + raise AssertionError except AssertionError: esc.escalate_hypothesis_internal_error() @@ -33,7 +30,7 @@ def test_does_escalate_errors_in_hypothesis_file(monkeypatch): with pytest.raises(AssertionError): try: - assert False + raise AssertionError except AssertionError: esc.escalate_hypothesis_internal_error() @@ -43,7 +40,7 @@ def test_does_not_escalate_errors_in_hypothesis_file_if_disabled(monkeypatch): monkeypatch.setattr(esc, "PREVENT_ESCALATION", True) try: - assert False + raise AssertionError except AssertionError: esc.escalate_hypothesis_internal_error() @@ -67,3 +64,13 @@ def test_is_hypothesis_file_not_confused_by_prefix(monkeypatch): @pytest.mark.parametrize("fname", ["", ""]) def test_is_hypothesis_file_does_not_error_on_invalid_paths_issue_2319(fname): assert not esc.is_hypothesis_file(fname) + + +def test_multiplefailures_deprecation(): + with pytest.warns(errors.HypothesisDeprecationWarning): + exc = errors.MultipleFailures + assert exc is BaseExceptionGroup + + +def test_handles_null_traceback(): + esc.get_interesting_origin(Exception()) diff --git a/hypothesis-python/tests/cover/test_example.py b/hypothesis-python/tests/cover/test_example.py index 8a82e93708..dc272ad9c0 100644 --- a/hypothesis-python/tests/cover/test_example.py +++ b/hypothesis-python/tests/cover/test_example.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import sys import warnings @@ -79,7 +74,7 @@ def test_non_interactive_example_emits_warning(): def test_interactive_example_does_not_emit_warning(): try: child = pexpect.spawn(f"{sys.executable} -Werror") - child.expect(">>> ", timeout=1) + child.expect(">>> ", timeout=10) except pexpect.exceptions.EOF: pytest.skip( "Unable to run python with -Werror. This may be because you are " diff --git a/hypothesis-python/tests/cover/test_executors.py b/hypothesis-python/tests/cover/test_executors.py index 2ccf3bed9e..808788df0c 100644 --- a/hypothesis-python/tests/cover/test_executors.py +++ b/hypothesis-python/tests/cover/test_executors.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import inspect from unittest import TestCase @@ -69,7 +64,7 @@ def test_no_boom(self, x): @given(integers()) def test_boom(self, x): - assert False + raise AssertionError def test_boom(): diff --git a/hypothesis-python/tests/cover/test_explicit_examples.py b/hypothesis-python/tests/cover/test_explicit_examples.py index 35155c7a0b..d47e9ae41e 100644 --- a/hypothesis-python/tests/cover/test_explicit_examples.py +++ b/hypothesis-python/tests/cover/test_explicit_examples.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import time from unittest import TestCase @@ -28,13 +23,9 @@ reporting, settings, ) -from hypothesis.errors import ( - DeadlineExceeded, - HypothesisWarning, - InvalidArgument, - MultipleFailures, -) -from hypothesis.strategies import floats, integers, nothing, text +from hypothesis.errors import DeadlineExceeded, HypothesisWarning, InvalidArgument +from hypothesis.internal.compat import ExceptionGroup +from hypothesis.strategies import floats, integers, text from tests.common.utils import assert_falsifying_output, capture_out @@ -173,7 +164,12 @@ def test_prints_verbose_output_for_explicit_examples(): def test_always_passes(x): pass - assert_falsifying_output(test_always_passes, "Trying explicit", x="NOT AN INTEGER") + assert_falsifying_output( + test_always_passes, + expected_exception=None, + example_type="Trying explicit", + x="NOT AN INTEGER", + ) def test_captures_original_repr_of_example(): @@ -210,14 +206,10 @@ def test(x): note(f"x -> {x}") assert x == 42 - with capture_out() as out: - with reporting.with_reporter(reporting.default): - with pytest.raises(AssertionError): - test() - v = out.getvalue() - print(v) - assert "x -> 43" in v - assert "x -> 42" not in v + with pytest.raises(AssertionError) as err: + test() + assert "x -> 43" in err.value.__notes__ + assert "x -> 42" not in err.value.__notes__ def test_must_agree_with_number_of_arguments(): @@ -233,7 +225,7 @@ def test(a): def test_runs_deadline_for_examples(): @example(10) @settings(phases=[Phase.explicit]) - @given(nothing()) + @given(integers()) def test(x): time.sleep(10) @@ -250,11 +242,11 @@ def test_unsatisfied_assumption_during_explicit_example(threshold, value): assume(value < threshold) -@pytest.mark.parametrize("exc", [MultipleFailures, AssertionError]) +@pytest.mark.parametrize("exc", [ExceptionGroup, AssertionError]) def test_multiple_example_reporting(exc): @example(1) @example(2) - @settings(report_multiple_bugs=exc is MultipleFailures, phases=[Phase.explicit]) + @settings(report_multiple_bugs=exc is ExceptionGroup, phases=[Phase.explicit]) @given(integers()) def inner_test_multiple_failing_examples(x): assert x < 2 @@ -282,3 +274,11 @@ def t(s): assert isinstance(err.__cause__, AssertionError) else: raise NotImplementedError("should be unreachable") + + +def test_stop_silently_dropping_examples_when_decorator_is_applied_to_itself(): + def f(): + pass + + test = example("outer")(example("inner"))(f) + assert len(test.hypothesis_explicit_examples) == 2 diff --git a/hypothesis-python/tests/cover/test_falsifying_example_output.py b/hypothesis-python/tests/cover/test_falsifying_example_output.py index 09de9ca60f..6d942aa1b2 100644 --- a/hypothesis-python/tests/cover/test_falsifying_example_output.py +++ b/hypothesis-python/tests/cover/test_falsifying_example_output.py @@ -1,35 +1,27 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest -from hypothesis import example, given, strategies as st - -from tests.common.utils import capture_out +from hypothesis import Phase, example, given, settings, strategies as st -OUTPUT_NO_LINE_BREAK = """ +OUTPUT_NO_BREAK = """ Falsifying explicit example: test( - x=%(input)s, y=%(input)s, + x={0!r}, y={0!r}, ) """ - -OUTPUT_WITH_LINE_BREAK = """ +OUTPUT_WITH_BREAK = """ Falsifying explicit example: test( - x=%(input)s, - y=%(input)s, + x={0!r}, + y={0!r}, ) """ @@ -41,26 +33,32 @@ def test_inserts_line_breaks_only_at_appropriate_lengths(line_break, input): def test(x, y): assert x < y - with capture_out() as cap: - with pytest.raises(AssertionError): - test() + with pytest.raises(AssertionError) as err: + test() - template = OUTPUT_WITH_LINE_BREAK if line_break else OUTPUT_NO_LINE_BREAK + expected = (OUTPUT_WITH_BREAK if line_break else OUTPUT_NO_BREAK).format(input) + assert expected.strip() == "\n".join(err.value.__notes__) - desired_output = template % {"input": repr(input)} - actual_output = cap.getvalue() +@given(kw=st.none()) +def generate_phase(*args, kw): + assert args != (1, 2, 3) - assert desired_output.strip() == actual_output.strip() +@given(kw=st.none()) +@example(kw=None) +@settings(phases=[Phase.explicit]) +def explicit_phase(*args, kw): + assert args != (1, 2, 3) -def test_vararg_output(): - @given(foo=st.text()) - def test(*args, foo): - assert False - with capture_out() as cap: - with pytest.raises(AssertionError): - test(1, 2, 3) +@pytest.mark.parametrize( + "fn", + [generate_phase, explicit_phase], + ids=lambda fn: fn.__name__, +) +def test_vararg_output(fn): + with pytest.raises(AssertionError) as err: + fn(1, 2, 3) - assert "1, 2, 3" in cap.getvalue() + assert "1, 2, 3" in "\n".join(err.value.__notes__) diff --git a/hypothesis-python/tests/cover/test_feature_flags.py b/hypothesis-python/tests/cover/test_feature_flags.py index 9f68f88def..cb71ccea95 100644 --- a/hypothesis-python/tests/cover/test_feature_flags.py +++ b/hypothesis-python/tests/cover/test_feature_flags.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import given, strategies as st from hypothesis.strategies._internal.featureflags import FeatureFlags, FeatureStrategy diff --git a/hypothesis-python/tests/cover/test_filestorage.py b/hypothesis-python/tests/cover/test_filestorage.py index bb6c73ef6b..1cd8cfe3e6 100644 --- a/hypothesis-python/tests/cover/test_filestorage.py +++ b/hypothesis-python/tests/cover/test_filestorage.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import os diff --git a/hypothesis-python/tests/cover/test_filter_rewriting.py b/hypothesis-python/tests/cover/test_filter_rewriting.py new file mode 100644 index 0000000000..b505f11a8e --- /dev/null +++ b/hypothesis-python/tests/cover/test_filter_rewriting.py @@ -0,0 +1,365 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import decimal +import math +import operator +from fractions import Fraction +from functools import partial +from sys import float_info + +import pytest + +from hypothesis import given, strategies as st +from hypothesis.errors import HypothesisWarning, Unsatisfiable +from hypothesis.internal.floats import next_down, next_up +from hypothesis.internal.reflection import get_pretty_function_description +from hypothesis.strategies._internal.lazy import LazyStrategy, unwrap_strategies +from hypothesis.strategies._internal.numbers import FloatStrategy, IntegersStrategy +from hypothesis.strategies._internal.strategies import FilteredStrategy + +from tests.common.utils import fails_with + + +@pytest.mark.parametrize( + "strategy, predicate, start, end", + [ + # Finitude check + (st.integers(1, 5), math.isfinite, 1, 5), + # Integers with integer bounds + (st.integers(1, 5), partial(operator.lt, 3), 4, 5), # lambda x: 3 < x + (st.integers(1, 5), partial(operator.le, 3), 3, 5), # lambda x: 3 <= x + (st.integers(1, 5), partial(operator.eq, 3), 3, 3), # lambda x: 3 == x + (st.integers(1, 5), partial(operator.ge, 3), 1, 3), # lambda x: 3 >= x + (st.integers(1, 5), partial(operator.gt, 3), 1, 2), # lambda x: 3 > x + # Integers with non-integer bounds + (st.integers(1, 5), partial(operator.lt, 3.5), 4, 5), + (st.integers(1, 5), partial(operator.le, 3.5), 4, 5), + (st.integers(1, 5), partial(operator.ge, 3.5), 1, 3), + (st.integers(1, 5), partial(operator.gt, 3.5), 1, 3), + (st.integers(1, 5), partial(operator.lt, -math.inf), 1, 5), + (st.integers(1, 5), partial(operator.gt, math.inf), 1, 5), + # Integers with only one bound + (st.integers(min_value=1), partial(operator.lt, 3), 4, None), + (st.integers(min_value=1), partial(operator.le, 3), 3, None), + (st.integers(max_value=5), partial(operator.ge, 3), None, 3), + (st.integers(max_value=5), partial(operator.gt, 3), None, 2), + # Unbounded integers + (st.integers(), partial(operator.lt, 3), 4, None), + (st.integers(), partial(operator.le, 3), 3, None), + (st.integers(), partial(operator.eq, 3), 3, 3), + (st.integers(), partial(operator.ge, 3), None, 3), + (st.integers(), partial(operator.gt, 3), None, 2), + # Simple lambdas + (st.integers(), lambda x: x < 3, None, 2), + (st.integers(), lambda x: x <= 3, None, 3), + (st.integers(), lambda x: x == 3, 3, 3), + (st.integers(), lambda x: x >= 3, 3, None), + (st.integers(), lambda x: x > 3, 4, None), + # Simple lambdas, reverse comparison + (st.integers(), lambda x: 3 > x, None, 2), + (st.integers(), lambda x: 3 >= x, None, 3), + (st.integers(), lambda x: 3 == x, 3, 3), + (st.integers(), lambda x: 3 <= x, 3, None), + (st.integers(), lambda x: 3 < x, 4, None), + # More complicated lambdas + (st.integers(), lambda x: 0 < x < 5, 1, 4), + (st.integers(), lambda x: 0 < x >= 1, 1, None), + (st.integers(), lambda x: 1 > x <= 0, None, 0), + (st.integers(), lambda x: x > 0 and x > 0, 1, None), + (st.integers(), lambda x: x < 1 and x < 1, None, 0), + (st.integers(), lambda x: x > 1 and x > 0, 2, None), + (st.integers(), lambda x: x < 1 and x < 2, None, 0), + ], + ids=get_pretty_function_description, +) +@given(data=st.data()) +def test_filter_rewriting_ints(data, strategy, predicate, start, end): + s = strategy.filter(predicate) + assert isinstance(s, LazyStrategy) + assert isinstance(s.wrapped_strategy, IntegersStrategy) + assert s.wrapped_strategy.start == start + assert s.wrapped_strategy.end == end + value = data.draw(s) + assert predicate(value) + + +@pytest.mark.parametrize( + "strategy, predicate, min_value, max_value", + [ + # Floats with integer bounds + (st.floats(1, 5), partial(operator.lt, 3), next_up(3.0), 5), # 3 < x + (st.floats(1, 5), partial(operator.le, 3), 3, 5), # lambda x: 3 <= x + (st.floats(1, 5), partial(operator.eq, 3), 3, 3), # lambda x: 3 == x + (st.floats(1, 5), partial(operator.ge, 3), 1, 3), # lambda x: 3 >= x + (st.floats(1, 5), partial(operator.gt, 3), 1, next_down(3.0)), # 3 > x + # Floats with non-integer bounds + (st.floats(1, 5), partial(operator.lt, 3.5), next_up(3.5), 5), + (st.floats(1, 5), partial(operator.le, 3.5), 3.5, 5), + (st.floats(1, 5), partial(operator.ge, 3.5), 1, 3.5), + (st.floats(1, 5), partial(operator.gt, 3.5), 1, next_down(3.5)), + (st.floats(1, 5), partial(operator.lt, -math.inf), 1, 5), + (st.floats(1, 5), partial(operator.gt, math.inf), 1, 5), + # Floats with only one bound + (st.floats(min_value=1), partial(operator.lt, 3), next_up(3.0), math.inf), + (st.floats(min_value=1), partial(operator.le, 3), 3, math.inf), + (st.floats(max_value=5), partial(operator.ge, 3), -math.inf, 3), + (st.floats(max_value=5), partial(operator.gt, 3), -math.inf, next_down(3.0)), + # Unbounded floats + (st.floats(), partial(operator.lt, 3), next_up(3.0), math.inf), + (st.floats(), partial(operator.le, 3), 3, math.inf), + (st.floats(), partial(operator.eq, 3), 3, 3), + (st.floats(), partial(operator.ge, 3), -math.inf, 3), + (st.floats(), partial(operator.gt, 3), -math.inf, next_down(3.0)), + # Simple lambdas + (st.floats(), lambda x: x < 3, -math.inf, next_down(3.0)), + (st.floats(), lambda x: x <= 3, -math.inf, 3), + (st.floats(), lambda x: x == 3, 3, 3), + (st.floats(), lambda x: x >= 3, 3, math.inf), + (st.floats(), lambda x: x > 3, next_up(3.0), math.inf), + # Simple lambdas, reverse comparison + (st.floats(), lambda x: 3 > x, -math.inf, next_down(3.0)), + (st.floats(), lambda x: 3 >= x, -math.inf, 3), + (st.floats(), lambda x: 3 == x, 3, 3), + (st.floats(), lambda x: 3 <= x, 3, math.inf), + (st.floats(), lambda x: 3 < x, next_up(3.0), math.inf), + # More complicated lambdas + (st.floats(), lambda x: 0 < x < 5, next_up(0.0), next_down(5.0)), + (st.floats(), lambda x: 0 < x >= 1, 1, math.inf), + (st.floats(), lambda x: 1 > x <= 0, -math.inf, 0), + (st.floats(), lambda x: x > 0 and x > 0, next_up(0.0), math.inf), + (st.floats(), lambda x: x < 1 and x < 1, -math.inf, next_down(1.0)), + (st.floats(), lambda x: x > 1 and x > 0, next_up(1.0), math.inf), + (st.floats(), lambda x: x < 1 and x < 2, -math.inf, next_down(1.0)), + # Specific named functions + (st.floats(), math.isfinite, next_up(-math.inf), next_down(math.inf)), + ], + ids=get_pretty_function_description, +) +@given(data=st.data()) +def test_filter_rewriting_floats(data, strategy, predicate, min_value, max_value): + s = strategy.filter(predicate) + assert isinstance(s, LazyStrategy) + assert isinstance(s.wrapped_strategy, FloatStrategy) + assert s.wrapped_strategy.min_value == min_value + assert s.wrapped_strategy.max_value == max_value + value = data.draw(s) + assert predicate(value) + + +@pytest.mark.parametrize( + "pred", + [ + math.isinf, + math.isnan, + partial(operator.lt, 6), + partial(operator.eq, Fraction(10, 3)), + partial(operator.ge, 0), + partial(operator.lt, math.inf), + partial(operator.gt, -math.inf), + ], +) +@pytest.mark.parametrize("s", [st.integers(1, 5), st.floats(1, 5)]) +def test_rewrite_unsatisfiable_filter(s, pred): + assert s.filter(pred).is_empty + + +@pytest.mark.parametrize( + "pred", + [ + partial(operator.eq, "numbers are never equal to strings"), + ], +) +@pytest.mark.parametrize("s", [st.integers(1, 5), st.floats(1, 5)]) +@fails_with(Unsatisfiable) +def test_erroring_rewrite_unsatisfiable_filter(s, pred): + s.filter(pred).example() + + +@pytest.mark.parametrize( + "strategy, predicate", + [ + (st.floats(), math.isinf), + (st.floats(0, math.inf), math.isinf), + (st.floats(), math.isnan), + ], +) +@given(data=st.data()) +def test_misc_sat_filter_rewrites(data, strategy, predicate): + s = strategy.filter(predicate).wrapped_strategy + assert not isinstance(s, FloatStrategy) + value = data.draw(s) + assert predicate(value) + + +@pytest.mark.parametrize( + "strategy, predicate", + [ + (st.floats(allow_infinity=False), math.isinf), + (st.floats(0, math.inf), math.isnan), + (st.floats(allow_nan=False), math.isnan), + ], +) +@given(data=st.data()) +def test_misc_unsat_filter_rewrites(data, strategy, predicate): + assert strategy.filter(predicate).is_empty + + +@given(st.integers(0, 2).filter(partial(operator.ne, 1))) +def test_unhandled_operator(x): + assert x in (0, 2) + + +def test_rewriting_does_not_compare_decimal_snan(): + s = st.integers(1, 5).filter(partial(operator.eq, decimal.Decimal("snan"))) + s.wrapped_strategy + with pytest.raises(decimal.InvalidOperation): + s.example() + + +@pytest.mark.parametrize("strategy", [st.integers(0, 1), st.floats(0, 1)], ids=repr) +def test_applying_noop_filter_returns_self(strategy): + s = strategy.wrapped_strategy + s2 = s.filter(partial(operator.le, -1)).filter(partial(operator.ge, 2)) + assert s is s2 + + +def mod2(x): + return x % 2 + + +Y = 2**20 + + +@pytest.mark.parametrize("s", [st.integers(1, 5), st.floats(1, 5)]) +@given( + data=st.data(), + predicates=st.permutations( + [ + partial(operator.lt, 1), + partial(operator.le, 2), + partial(operator.ge, 4), + partial(operator.gt, 5), + mod2, + lambda x: x > 2 or x % 7, + lambda x: 0 < x <= Y, + ] + ), +) +def test_rewrite_filter_chains_with_some_unhandled(data, predicates, s): + # Set up our strategy + for p in predicates: + s = s.filter(p) + + # Whatever value we draw is in fact valid for these strategies + value = data.draw(s) + for p in predicates: + assert p(value), f"p={p!r}, value={value}" + + # No matter the order of the filters, we get the same resulting structure + unwrapped = s.wrapped_strategy + assert isinstance(unwrapped, FilteredStrategy) + assert isinstance(unwrapped.filtered_strategy, (IntegersStrategy, FloatStrategy)) + for pred in unwrapped.flat_conditions: + assert pred is mod2 or pred.__name__ == "" + + +class NotAFunction: + def __call__(self, bar): + return True + + +lambda_without_source = eval("lambda x: x > 2", {}, {}) +assert get_pretty_function_description(lambda_without_source) == "lambda x: " + + +@pytest.mark.parametrize( + "start, end, predicate", + [ + (1, 4, lambda x: 0 < x < 5 and x % 7), + (0, 9, lambda x: 0 <= x < 10 and x % 3), + (1, None, lambda x: 0 < x <= Y), + (None, None, lambda x: x == x), + (None, None, lambda x: 1 == 1), + (None, None, lambda x: 1 <= 2), + (None, None, lambda x: x != 0), + (None, None, NotAFunction()), + (None, None, lambda_without_source), + (None, None, lambda x, y=2: x >= 0), + ], +) +@given(data=st.data()) +def test_rewriting_partially_understood_filters(data, start, end, predicate): + s = st.integers().filter(predicate).wrapped_strategy + + assert isinstance(s, FilteredStrategy) + assert isinstance(s.filtered_strategy, IntegersStrategy) + assert s.filtered_strategy.start == start + assert s.filtered_strategy.end == end + assert s.flat_conditions == (predicate,) + + value = data.draw(s) + assert predicate(value) + + +@pytest.mark.parametrize( + "strategy", + [ + st.text(), + st.text(min_size=2), + st.lists(st.none()), + st.lists(st.none(), min_size=2), + ], +) +@pytest.mark.parametrize( + "predicate", + [bool, len, tuple, list, lambda x: x], + ids=get_pretty_function_description, +) +def test_sequence_filter_rewriting(strategy, predicate): + s = unwrap_strategies(strategy) + fs = s.filter(predicate) + assert not isinstance(fs, FilteredStrategy) + if s.min_size > 0: + assert fs is s + else: + assert fs.min_size == 1 + + +@pytest.mark.parametrize("method", [str.lower, str.title, str.upper]) +def test_warns_on_suspicious_string_methods(method): + s = unwrap_strategies(st.text()) + with pytest.warns( + HypothesisWarning, match="this allows all nonempty strings! Did you mean" + ): + fs = s.filter(method) + assert fs.min_size == 1 + + +@pytest.mark.parametrize("method", [str.isidentifier, str.isalnum]) +def test_bumps_min_size_and_filters_for_content_str_methods(method): + s = unwrap_strategies(st.text()) + fs = s.filter(method) + assert fs.filtered_strategy.min_size == 1 + assert fs.flat_conditions == (method,) + + +@pytest.mark.parametrize( + "op, attr, value, expected", + [ + (operator.lt, "min_value", -float_info.min / 2, 0), + (operator.lt, "min_value", float_info.min / 2, float_info.min), + (operator.gt, "max_value", float_info.min / 2, 0), + (operator.gt, "max_value", -float_info.min / 2, -float_info.min), + ], +) +def test_filter_floats_can_skip_subnormals(op, attr, value, expected): + base = st.floats(allow_subnormal=False).filter(partial(op, value)) + assert getattr(base.wrapped_strategy, attr) == expected diff --git a/hypothesis-python/tests/cover/test_filtered_strategy.py b/hypothesis-python/tests/cover/test_filtered_strategy.py index 01ad281bb4..02a821f2f6 100644 --- a/hypothesis-python/tests/cover/test_filtered_strategy.py +++ b/hypothesis-python/tests/cover/test_filtered_strategy.py @@ -1,27 +1,43 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import hypothesis.strategies as st from hypothesis.internal.conjecture.data import ConjectureData +from hypothesis.strategies._internal.strategies import FilteredStrategy def test_filter_iterations_are_marked_as_discarded(): - x = st.integers(0, 255).filter(lambda x: x == 0) + variable_equal_to_zero = 0 # non-local references disables filter-rewriting + x = st.integers(0, 255).filter(lambda x: x == variable_equal_to_zero) - data = ConjectureData.for_buffer([2, 1, 0]) + data = ConjectureData.for_buffer([0, 2, 1, 0]) assert data.draw(x) == 0 assert data.has_discards + + +def test_filtered_branches_are_all_filtered(): + s = FilteredStrategy(st.integers() | st.text(), (bool,)) + assert all(isinstance(x, FilteredStrategy) for x in s.branches) + + +def test_filter_conditions_may_be_empty(): + s = FilteredStrategy(st.integers(), conditions=()) + s.condition(0) + + +def test_nested_filteredstrategy_flattens_conditions(): + s = FilteredStrategy( + FilteredStrategy(st.text(), conditions=(bool,)), + conditions=(len,), + ) + assert s.filtered_strategy is st.text() + assert s.flat_conditions == (bool, len) diff --git a/hypothesis-python/tests/cover/test_find.py b/hypothesis-python/tests/cover/test_find.py index adf47a76ae..3aa7226e7a 100644 --- a/hypothesis-python/tests/cover/test_find.py +++ b/hypothesis-python/tests/cover/test_find.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from random import Random diff --git a/hypothesis-python/tests/cover/test_flakiness.py b/hypothesis-python/tests/cover/test_flakiness.py index 37ac714dd2..62251d11d8 100644 --- a/hypothesis-python/tests/cover/test_flakiness.py +++ b/hypothesis-python/tests/cover/test_flakiness.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest @@ -48,7 +43,7 @@ def test_gives_flaky_error_if_assumption_is_flaky(): def oops(s): assume(s not in seen) seen.add(s) - assert False + raise AssertionError with pytest.raises(Flaky): oops() @@ -126,4 +121,4 @@ def test(x): except (Nope, Unsatisfiable, Flaky): pass except UnsatisfiedAssumption: - raise SatisfyMe() + raise SatisfyMe() from None diff --git a/hypothesis-python/tests/cover/test_float_nastiness.py b/hypothesis-python/tests/cover/test_float_nastiness.py index 1a12ba3f42..d0a7977d72 100644 --- a/hypothesis-python/tests/cover/test_float_nastiness.py +++ b/hypothesis-python/tests/cover/test_float_nastiness.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import math import sys @@ -21,7 +16,6 @@ from hypothesis import assume, given, strategies as st from hypothesis.errors import InvalidArgument -from hypothesis.internal.compat import WINDOWS from hypothesis.internal.floats import ( float_of, float_to_int, @@ -84,10 +78,6 @@ def test_half_bounded_generates_zero(): find_any(st.floats(max_value=1.0), lambda x: x == 0.0) -@pytest.mark.xfail( - WINDOWS, - reason=("Seems to be triggering a floating point bug on 2.7 + windows + x64"), -) @given(st.floats(max_value=-0.0)) def test_half_bounded_respects_sign_of_upper_bound(x): assert math.copysign(1, x) == -1 @@ -182,14 +172,14 @@ def test_float16_can_exclude_infinity(x): @pytest.mark.parametrize( "kwargs", [ - {"min_value": 10 ** 5, "width": 16}, - {"max_value": 10 ** 5, "width": 16}, - {"min_value": 10 ** 40, "width": 32}, - {"max_value": 10 ** 40, "width": 32}, - {"min_value": 10 ** 400, "width": 64}, - {"max_value": 10 ** 400, "width": 64}, - {"min_value": 10 ** 400}, - {"max_value": 10 ** 400}, + {"min_value": 10**5, "width": 16}, + {"max_value": 10**5, "width": 16}, + {"min_value": 10**40, "width": 32}, + {"max_value": 10**40, "width": 32}, + {"min_value": 10**400, "width": 64}, + {"max_value": 10**400, "width": 64}, + {"min_value": 10**400}, + {"max_value": 10**400}, ], ) def test_out_of_range(kwargs): @@ -203,7 +193,7 @@ def test_disallowed_width(): def test_no_single_floats_in_range(): - low = 2.0 ** 25 + 1 + low = 2.0**25 + 1 high = low + 2 st.floats(low, high).validate() # Note: OK for 64bit floats with pytest.raises(InvalidArgument): diff --git a/hypothesis-python/tests/cover/test_float_utils.py b/hypothesis-python/tests/cover/test_float_utils.py index 6555d6078b..7122a13436 100644 --- a/hypothesis-python/tests/cover/test_float_utils.py +++ b/hypothesis-python/tests/cover/test_float_utils.py @@ -1,23 +1,25 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import math +from sys import float_info import pytest -from hypothesis.internal.floats import count_between_floats, next_down, next_up +from hypothesis import example, given, strategies as st +from hypothesis.internal.floats import ( + count_between_floats, + make_float_clamper, + next_down, + next_up, +) def test_can_handle_straddling_zero(): @@ -40,3 +42,42 @@ def test_next_float_equal(func, val): assert math.isnan(func(val)) else: assert func(val) == val + + +# invalid order -> clamper is None: +@example(2.0, 1.0, 3.0) +# exponent comparisons: +@example(1, float_info.max, 0) +@example(1, float_info.max, 1) +@example(1, float_info.max, 10) +@example(1, float_info.max, float_info.max) +@example(1, float_info.max, math.inf) +# mantissa comparisons: +@example(100.0001, 100.0003, 100.0001) +@example(100.0001, 100.0003, 100.0002) +@example(100.0001, 100.0003, 100.0003) +@given(st.floats(min_value=0), st.floats(min_value=0), st.floats(min_value=0)) +def test_float_clamper(min_value, max_value, input_value): + clamper = make_float_clamper(min_value, max_value, allow_zero=False) + if max_value < min_value: + assert clamper is None + return + clamped = clamper(input_value) + if min_value <= input_value <= max_value: + assert input_value == clamped + else: + assert min_value <= clamped <= max_value + + +@example(0.01, math.inf, 0.0) +@given(st.floats(min_value=0), st.floats(min_value=0), st.floats(min_value=0)) +def test_float_clamper_with_allowed_zeros(min_value, max_value, input_value): + clamper = make_float_clamper(min_value, max_value, allow_zero=True) + assert clamper is not None + clamped = clamper(input_value) + if input_value == 0.0 or max_value < min_value: + assert clamped == 0.0 + elif min_value <= input_value <= max_value: + assert input_value == clamped + else: + assert min_value <= clamped <= max_value diff --git a/hypothesis-python/tests/cover/test_functions.py b/hypothesis-python/tests/cover/test_functions.py index 893147a1f7..e0e3868c3d 100644 --- a/hypothesis-python/tests/cover/test_functions.py +++ b/hypothesis-python/tests/cover/test_functions.py @@ -1,24 +1,20 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER -from inspect import getfullargspec +from inspect import signature import pytest -from hypothesis import assume, given +from hypothesis import Verbosity, assume, given, settings from hypothesis.errors import InvalidArgument, InvalidState +from hypothesis.reporting import with_reporter from hypothesis.strategies import booleans, functions, integers @@ -115,9 +111,10 @@ def t(f): def test_can_call_default_like_arg(): # This test is somewhat silly, but coverage complains about the uncovered # branch for calling it otherwise and alternative workarounds are worse. - defaults = getfullargspec(functions).kwonlydefaults - assert defaults["like"]() is None - assert defaults["returns"] is None + like, returns, pure = signature(functions).parameters.values() + assert like.default() is None + assert returns.default is ... + assert pure.default is False def func(arg, *, kwonly_arg): @@ -184,3 +181,23 @@ def test_functions_pure_two_functions_same_args_different_result(f1, f2, arg1, a r2 = f2(arg1, arg2) assume(r1 != r2) # If this is never true, the test will fail with Unsatisfiable + + +@settings(verbosity=Verbosity.verbose) +@given(functions(pure=False)) +def test_functions_note_all_calls_to_impure_functions(f): + ls = [] + with with_reporter(ls.append): + f() + f() + assert len(ls) == 2 + + +@settings(verbosity=Verbosity.verbose) +@given(functions(pure=True)) +def test_functions_note_only_first_to_pure_functions(f): + ls = [] + with with_reporter(ls.append): + f() + f() + assert len(ls) == 1 diff --git a/hypothesis-python/tests/cover/test_fuzz_one_input.py b/hypothesis-python/tests/cover/test_fuzz_one_input.py index 7fc85391ab..9918cc3223 100644 --- a/hypothesis-python/tests/cover/test_fuzz_one_input.py +++ b/hypothesis-python/tests/cover/test_fuzz_one_input.py @@ -1,28 +1,29 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import io -import random +import unittest from operator import attrgetter import pytest from hypothesis import Phase, given, settings, strategies as st from hypothesis.database import InMemoryExampleDatabase +from hypothesis.errors import InvalidArgument from hypothesis.internal.conjecture.shrinker import sort_key +try: + from random import randbytes +except ImportError: # New in Python 3.9 + from secrets import token_bytes as randbytes + @pytest.mark.parametrize( "buffer_type", @@ -45,14 +46,15 @@ def test(s): # Before running fuzz_one_input, there's nothing in `db`, and so the test passes # (because example generation is disabled by the custom settings) - test() + with pytest.raises(unittest.SkipTest): # because this generates no examples + test() assert len(seen) == 0 # If we run a lot of random bytestrings through fuzz_one_input, we'll eventually # find a failing example. with pytest.raises(AssertionError): for _ in range(1000): - buf = bytes(random.getrandbits(8) for _ in range(1000)) + buf = randbytes(1000) seeds.append(buf) test.hypothesis.fuzz_one_input(buffer_type(buf)) @@ -78,7 +80,7 @@ def test_can_fuzz_with_database_eq_None(): @given(st.none()) @settings(database=None) def test(s): - assert False + raise AssertionError with pytest.raises(AssertionError): test.hypothesis.fuzz_one_input(b"\x00\x00") @@ -94,7 +96,7 @@ def test(s): raise AssertionError("Unreachable because there are no valid examples") for _ in range(100): - buf = bytes(random.getrandbits(8) for _ in range(3)) + buf = randbytes(3) ret = test.hypothesis.fuzz_one_input(buf) assert ret is None @@ -155,3 +157,14 @@ def test(s): (saved_examples,) = db.data.values() assert seen == buffers assert len(saved_examples) == db_size + + +def test_fuzzing_invalid_test_raises_error(): + # Invalid: @given with too many positional arguments + @given(st.integers(), st.integers()) + def invalid_test(s): + pass + + with pytest.raises(InvalidArgument, match="Too many positional arguments"): + # access the property to check error happens during setup + invalid_test.hypothesis.fuzz_one_input diff --git a/hypothesis-python/tests/cover/test_given_error_conditions.py b/hypothesis-python/tests/cover/test_given_error_conditions.py index f1bafc3831..e417a6fc1a 100644 --- a/hypothesis-python/tests/cover/test_given_error_conditions.py +++ b/hypothesis-python/tests/cover/test_given_error_conditions.py @@ -1,23 +1,18 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest -from hypothesis import assume, given, infer, reject, settings +from hypothesis import assume, given, reject, settings from hypothesis.errors import InvalidArgument, Unsatisfiable -from hypothesis.strategies import booleans, integers +from hypothesis.strategies import booleans, integers, nothing from tests.common.utils import fails_with @@ -39,8 +34,29 @@ def test_assume_x(x): test_assume_x() +def test_raises_unsatisfiable_if_passed_explicit_nothing(): + @given(x=nothing()) + def test_never_runs(x): + raise Exception("Can't ever execute this") + + with pytest.raises( + Unsatisfiable, + match=r"Cannot generate examples from empty strategy: x=nothing\(\)", + ): + test_never_runs() + + def test_error_if_has_no_hints(): - @given(a=infer) + @given(a=...) + def inner(a): + pass + + with pytest.raises(InvalidArgument): + inner() + + +def test_error_if_infer_all_and_has_no_hints(): + @given(...) def inner(a): pass @@ -49,8 +65,17 @@ def inner(a): def test_error_if_infer_is_posarg(): - @given(infer) - def inner(ex): + @given(..., ...) + def inner(ex1: int, ex2: int): + pass + + with pytest.raises(InvalidArgument): + inner() + + +def test_error_if_infer_is_posarg_mixed_with_kwarg(): + @given(..., ex2=...) + def inner(ex1: int, ex2: int): pass with pytest.raises(InvalidArgument): @@ -74,3 +99,16 @@ def test_given_is_not_a_class_decorator(): class test_given_is_not_a_class_decorator: def __init__(self, i): pass + + +def test_specific_error_for_coroutine_functions(): + @settings(database=None) + @given(booleans()) + async def foo(x): + pass + + with pytest.raises( + InvalidArgument, + match="Hypothesis doesn't know how to run async test functions", + ): + foo() diff --git a/hypothesis-python/tests/cover/test_health_checks.py b/hypothesis-python/tests/cover/test_health_checks.py index 840521eceb..406c56b238 100644 --- a/hypothesis-python/tests/cover/test_health_checks.py +++ b/hypothesis-python/tests/cover/test_health_checks.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import time @@ -24,6 +19,13 @@ from hypothesis.internal.compat import int_from_bytes from hypothesis.internal.conjecture.data import ConjectureData from hypothesis.internal.entropy import deterministic_PRNG +from hypothesis.stateful import ( + RuleBasedStateMachine, + initialize, + invariant, + rule, + run_state_machine_as_test, +) from hypothesis.strategies._internal.lazy import LazyStrategy from hypothesis.strategies._internal.strategies import SearchStrategy @@ -43,7 +45,7 @@ def test(x): def test_slow_generation_inline_fails_a_health_check(): - @settings(HEALTH_CHECK_SETTINGS, deadline=None) + @HEALTH_CHECK_SETTINGS @given(st.data()) def test(data): data.draw(st.integers().map(lambda x: time.sleep(0.2))) @@ -120,7 +122,7 @@ def test(x): def test_large_data_will_fail_a_health_check(): - @given(st.none() | st.binary(min_size=10 ** 5, max_size=10 ** 5)) + @given(st.none() | st.binary(min_size=10**5, max_size=10**5)) @settings(database=None) def test(x): pass @@ -174,7 +176,7 @@ def test(b): with pytest.raises(FailedHealthCheck) as exc: test() - assert exc.value.health_check == HealthCheck.large_base_example + assert str(HealthCheck.large_base_example) in str(exc.value) def test_example_that_shrinks_to_overrun_fails_health_check(): @@ -185,7 +187,7 @@ def test(b): with pytest.raises(FailedHealthCheck) as exc: test() - assert exc.value.health_check == HealthCheck.large_base_example + assert str(HealthCheck.large_base_example) in str(exc.value) def test_it_is_an_error_to_suppress_non_iterables(): @@ -215,7 +217,7 @@ def slow_init_integers(*args, **kwargs): def test_lazy_slow_initialization_issue_2108_regression(data): # Slow init in strategies wrapped in a LazyStrategy, inside an interactive draw, # should be attributed to drawing from the strategy (not the test function). - # Specificially, this used to fail with a DeadlineExceeded error. + # Specifically, this used to fail with a DeadlineExceeded error. data.draw(LazyStrategy(slow_init_integers, (), {})) @@ -260,3 +262,33 @@ def test(b): pass test() + + +class ReturningRuleMachine(RuleBasedStateMachine): + @rule() + def r(self): + return "any non-None value" + + +class ReturningInitializeMachine(RuleBasedStateMachine): + _ = rule()(lambda self: None) + + @initialize() + def r(self): + return "any non-None value" + + +class ReturningInvariantMachine(RuleBasedStateMachine): + _ = rule()(lambda self: None) + + @invariant(check_during_init=True) + def r(self): + return "any non-None value" + + +@pytest.mark.parametrize( + "cls", [ReturningRuleMachine, ReturningInitializeMachine, ReturningInvariantMachine] +) +def test_stateful_returnvalue_healthcheck(cls): + with pytest.raises(FailedHealthCheck): + run_state_machine_as_test(cls, settings=settings()) diff --git a/hypothesis-python/tests/cover/test_internal_helpers.py b/hypothesis-python/tests/cover/test_internal_helpers.py index 77b653930d..a167c7aba3 100644 --- a/hypothesis-python/tests/cover/test_internal_helpers.py +++ b/hypothesis-python/tests/cover/test_internal_helpers.py @@ -1,25 +1,20 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest -from hypothesis.internal.floats import sign +from hypothesis.internal.floats import is_negative -def test_sign_gives_good_type_error(): +def test_is_negative_gives_good_type_error(): x = "foo" with pytest.raises(TypeError) as e: - sign(x) + is_negative(x) assert repr(x) in e.value.args[0] diff --git a/hypothesis-python/tests/cover/test_intervalset.py b/hypothesis-python/tests/cover/test_intervalset.py index 39f0deaea5..9b2f2d3485 100644 --- a/hypothesis-python/tests/cover/test_intervalset.py +++ b/hypothesis-python/tests/cover/test_intervalset.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest diff --git a/hypothesis-python/tests/cover/test_lambda_formatting.py b/hypothesis-python/tests/cover/test_lambda_formatting.py index 303491e19b..380e19710e 100644 --- a/hypothesis-python/tests/cover/test_lambda_formatting.py +++ b/hypothesis-python/tests/cover/test_lambda_formatting.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis.internal.reflection import get_pretty_function_description @@ -26,7 +21,7 @@ def test_no_whitespace_before_colon_with_no_args(): def test_can_have_unicode_in_lambda_sources(): t = lambda x: "é" not in x - assert get_pretty_function_description(t) == ('lambda x: "é" not in x') + assert get_pretty_function_description(t) == 'lambda x: "é" not in x' # fmt: off diff --git a/hypothesis-python/tests/cover/test_lazy_import.py b/hypothesis-python/tests/cover/test_lazy_import.py index 8cf06fb146..811c7a9970 100644 --- a/hypothesis-python/tests/cover/test_lazy_import.py +++ b/hypothesis-python/tests/cover/test_lazy_import.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import subprocess import sys diff --git a/hypothesis-python/tests/cover/test_lookup.py b/hypothesis-python/tests/cover/test_lookup.py index 8e57ccc167..07922ab94b 100644 --- a/hypothesis-python/tests/cover/test_lookup.py +++ b/hypothesis-python/tests/cover/test_lookup.py @@ -1,19 +1,15 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import abc +import builtins import collections import datetime import enum @@ -28,9 +24,10 @@ import pytest -from hypothesis import HealthCheck, assume, given, infer, settings, strategies as st +from hypothesis import HealthCheck, assume, given, settings, strategies as st from hypothesis.errors import InvalidArgument, ResolutionFailed -from hypothesis.internal.compat import PYPY, get_type_hints, typing_root_type +from hypothesis.internal.compat import get_type_hints +from hypothesis.internal.reflection import get_pretty_function_description from hypothesis.strategies import from_type from hypothesis.strategies._internal import types @@ -38,16 +35,20 @@ from tests.common.utils import fails_with, temp_registered sentinel = object() +BUILTIN_TYPES = tuple( + v + for v in vars(builtins).values() + if isinstance(v, type) and v.__name__ != "BuiltinImporter" +) generics = sorted( ( t for t in types._global_type_lookup # We ignore TypeVar, because it is not a Generic type: - if isinstance(t, typing_root_type) and t != typing.TypeVar + if isinstance(t, types.typing_root_type) and t != typing.TypeVar ), key=str, ) -xfail_on_39 = () if sys.version_info[:2] < (3, 9) else pytest.mark.xfail @pytest.mark.parametrize("typ", generics, ids=repr) @@ -98,11 +99,10 @@ def test_typing_Type_Union(ex): "typ", [ collections.abc.ByteString, - # These are nonexistent or exist-but-are-not-types on Python 3.6 - typing.Match if sys.version_info[:2] >= (3, 7) else int, - typing.Pattern if sys.version_info[:2] >= (3, 7) else int, - getattr(re, "Match", int), - getattr(re, "Pattern", int), + typing.Match, + typing.Pattern, + re.Match, + re.Pattern, ], ids=repr, ) @@ -182,7 +182,6 @@ def test_ItemsView(ex): assert all(all(isinstance(e, Elem) for e in elem) for elem in ex) -@pytest.mark.skipif(sys.version_info[:2] == (3, 6), reason="not a type on py36") @pytest.mark.parametrize("generic", [typing.Match, typing.Pattern]) @pytest.mark.parametrize("typ", [bytes, str]) @given(data=st.data()) @@ -191,7 +190,7 @@ def test_regex_types(data, generic, typ): assert isinstance(x[0] if generic is typing.Match else x.pattern, typ) -@given(x=infer) +@given(x=...) def test_Generator(x: typing.Generator[Elem, None, ElemValue]): assert isinstance(x, typing.Generator) try: @@ -257,7 +256,7 @@ def if_available(name): @pytest.mark.parametrize( "typ", [ - pytest.param(typing.Sequence, marks=xfail_on_39), + typing.Sequence, typing.Container, typing.Mapping, typing.Reversible, @@ -269,6 +268,7 @@ def if_available(name): typing.SupportsRound, if_available("SupportsIndex"), ], + ids=get_pretty_function_description, ) def test_resolves_weird_types(typ): from_type(typ).example() @@ -331,15 +331,16 @@ def test_distinct_typevars_distinct_type(): ) -@given(st.data()) -def test_same_typevars_same_type(data): - """Ensures that single type argument will always have the same type in a single context.""" - A = typing.TypeVar("A") +A = typing.TypeVar("A") + - def same_type_args(a: A, b: A): - assert type(a) == type(b) +def same_type_args(a: A, b: A): + assert type(a) == type(b) - data.draw(st.builds(same_type_args)) + +@given(st.builds(same_type_args)) +def test_same_typevars_same_type(_): + """Ensures that single type argument will always have the same type in a single context.""" def test_typevars_can_be_redefined(): @@ -383,7 +384,7 @@ def test_force_builds_to_infer_strategies_for_default_args(): # By default, leaves args with defaults and minimises to 2+4=6 assert minimal(st.builds(annotated_func), lambda ex: True) == 6 # Inferring integers() for args makes it minimise to zero - assert minimal(st.builds(annotated_func, b=infer, d=infer), lambda ex: True) == 0 + assert minimal(st.builds(annotated_func, b=..., d=...), lambda ex: True) == 0 def non_annotated_func(a, b=2, *, c, d=4): @@ -392,14 +393,14 @@ def non_annotated_func(a, b=2, *, c, d=4): def test_cannot_pass_infer_as_posarg(): with pytest.raises(InvalidArgument): - st.builds(annotated_func, infer).example() + st.builds(annotated_func, ...).example() def test_cannot_force_inference_for_unannotated_arg(): with pytest.raises(InvalidArgument): - st.builds(non_annotated_func, a=infer, c=st.none()).example() + st.builds(non_annotated_func, a=..., c=st.none()).example() with pytest.raises(InvalidArgument): - st.builds(non_annotated_func, a=st.none(), c=infer).example() + st.builds(non_annotated_func, a=st.none(), c=...).example() class UnknownType: @@ -425,16 +426,16 @@ def test_raises_for_arg_with_unresolvable_annotation(): with pytest.raises(ResolutionFailed): st.builds(unknown_annotated_func).example() with pytest.raises(ResolutionFailed): - st.builds(unknown_annotated_func, a=st.none(), c=infer).example() + st.builds(unknown_annotated_func, a=st.none(), c=...).example() -@given(a=infer, b=infer) +@given(a=..., b=...) def test_can_use_type_hints(a: int, b: float): assert isinstance(a, int) and isinstance(b, float) def test_error_if_has_unresolvable_hints(): - @given(a=infer) + @given(a=...) def inner(a: UnknownType): pass @@ -522,7 +523,7 @@ def test_override_args_for_namedtuple(thing): @pytest.mark.parametrize("thing", [typing.Optional, typing.List, typing.Type]) def test_cannot_resolve_bare_forward_reference(thing): with pytest.raises(InvalidArgument): - t = thing["int"] + t = thing["ConcreteFoo"] st.from_type(t).example() @@ -539,6 +540,32 @@ def test_resolving_recursive_type(): assert isinstance(st.builds(Tree).example(), Tree) +class SomeClass: + def __init__(self, value: int, next_node: typing.Optional["SomeClass"]) -> None: + assert value > 0 + self.value = value + self.next_node = next_node + + def __repr__(self) -> str: + return f"SomeClass({self.value}, next_node={self.next_node})" + + +def test_resolving_recursive_type_with_registered_constraint(): + with temp_registered( + SomeClass, st.builds(SomeClass, value=st.integers(min_value=1)) + ): + find_any(st.from_type(SomeClass), lambda s: s.next_node is None) + + +def test_resolving_recursive_type_with_registered_constraint_not_none(): + with temp_registered( + SomeClass, st.builds(SomeClass, value=st.integers(min_value=1)) + ): + s = st.from_type(SomeClass) + print(s, s.wrapped_strategy) + find_any(s, lambda s: s.next_node is not None) + + @given(from_type(typing.Tuple[()])) def test_resolves_empty_Tuple_issue_1583_regression(ex): # See e.g. https://github.com/python/mypy/commit/71332d58 @@ -600,7 +627,13 @@ def bar(self): @fails_with(ResolutionFailed) @given(st.from_type(AbstractBar)) def test_cannot_resolve_abstract_class_with_no_concrete_subclass(instance): - assert False, "test body unreachable as strategy cannot resolve" + raise AssertionError("test body unreachable as strategy cannot resolve") + + +@fails_with(ResolutionFailed) +@given(st.from_type(typing.Type["ConcreteFoo"])) +def test_cannot_resolve_type_with_forwardref(instance): + raise AssertionError("test body unreachable as strategy cannot resolve") @pytest.mark.parametrize("typ", [typing.Hashable, typing.Sized]) @@ -650,7 +683,7 @@ def test_supportsop_types_support_protocol(protocol, data): [ (typing.SupportsFloat, float), (typing.SupportsInt, int), - (typing.SupportsBytes, bytes), # noqa: B1 + (typing.SupportsBytes, bytes), (typing.SupportsComplex, complex), ], ) @@ -689,13 +722,18 @@ def test_generic_collections_only_use_hashable_elements(typ, data): data.draw(from_type(typ)) +@given(st.sets(st.integers() | st.binary(), min_size=2)) +def test_no_byteswarning(_): + pass + + def test_hashable_type_unhashable_value(): # Decimal("snan") is not hashable; we should be able to generate it. # See https://github.com/HypothesisWorks/hypothesis/issues/2320 find_any( from_type(typing.Hashable), lambda x: not types._can_hash(x), - settings(max_examples=10 ** 5), + settings(max_examples=10**5), ) @@ -717,7 +755,6 @@ class TreeForwardRefs(typing.NamedTuple): r: typing.Optional["TreeForwardRefs"] -@pytest.mark.skipif(PYPY, reason="pypy36 does not resolve the forward refs") @given(st.builds(TreeForwardRefs)) def test_resolves_forward_references_outside_annotations(t): assert isinstance(t, TreeForwardRefs) @@ -738,11 +775,15 @@ def __init__(self, **kwargs): def test_compat_get_type_hints_aware_of_None_default(): # Regression test for https://github.com/HypothesisWorks/hypothesis/issues/2648 - strategy = st.builds(WithOptionalInSignature, a=infer) + strategy = st.builds(WithOptionalInSignature, a=...) find_any(strategy, lambda x: x.a is None) find_any(strategy, lambda x: x.a is not None) - assert typing.get_type_hints(constructor)["a"] == typing.Optional[str] + if sys.version_info[:2] >= (3, 11): + # https://docs.python.org/3.11/library/typing.html#typing.get_type_hints + assert typing.get_type_hints(constructor)["a"] == str + else: + assert typing.get_type_hints(constructor)["a"] == typing.Optional[str] assert inspect.signature(constructor).parameters["a"].annotation == str @@ -790,10 +831,33 @@ def __init__(self, value: int) -> None: assert isinstance(value, str) -@given(st.data()) -def test_signature_is_the_most_important_source(data): +def selfless_signature(value: str) -> None: + ... + + +class AnnotatedConstructorWithSelflessSignature(AnnotatedConstructorWithSignature): + __signature__ = signature(selfless_signature) + + +def really_takes_str(value: int) -> None: + """By this example we show, that ``__signature__`` is the most important source.""" + assert isinstance(value, str) + + +really_takes_str.__signature__ = signature(selfless_signature) + + +@pytest.mark.parametrize( + "thing", + [ + AnnotatedConstructorWithSignature, + AnnotatedConstructorWithSelflessSignature, + really_takes_str, + ], +) +def test_signature_is_the_most_important_source(thing): """Signature types should take precedence over all other annotations.""" - data.draw(st.builds(AnnotatedConstructorWithSignature)) + find_any(st.builds(thing)) class AnnotatedAndDefault: @@ -804,3 +868,49 @@ def __init__(self, foo: bool = None): def test_from_type_can_be_default_or_annotation(): find_any(st.from_type(AnnotatedAndDefault), lambda x: x.foo is None) find_any(st.from_type(AnnotatedAndDefault), lambda x: isinstance(x.foo, bool)) + + +@pytest.mark.parametrize("t", BUILTIN_TYPES, ids=lambda t: t.__name__) +def test_resolves_builtin_types(t): + v = st.from_type(t).example() + assert isinstance(v, t) + + +@pytest.mark.parametrize("t", BUILTIN_TYPES, ids=lambda t: t.__name__) +def test_resolves_forwardrefs_to_builtin_types(t): + v = st.from_type(typing.ForwardRef(t.__name__)).example() + assert isinstance(v, t) + + +@pytest.mark.parametrize("t", BUILTIN_TYPES, ids=lambda t: t.__name__) +def test_resolves_type_of_builtin_types(t): + v = st.from_type(typing.Type[t.__name__]).example() + assert v is t + + +@given(st.from_type(typing.Type[typing.Union["str", "int"]])) +def test_resolves_type_of_union_of_forwardrefs_to_builtins(x): + assert x in (str, int) + + +@pytest.mark.parametrize("type_", [typing.List[int], typing.Optional[int]]) +def test_builds_suggests_from_type(type_): + with pytest.raises( + InvalidArgument, match=re.escape(f"try using from_type({type_!r})") + ): + st.builds(type_).example() + try: + st.builds(type_, st.just("has an argument")).example() + raise AssertionError("Expected strategy to raise an error") + except TypeError as err: + assert not isinstance(err, InvalidArgument) + + +def test_builds_mentions_no_type_check(): + @typing.no_type_check + def f(x: int): + pass + + msg = "@no_type_check decorator prevented Hypothesis from inferring a strategy" + with pytest.raises(TypeError, match=msg): + st.builds(f).example() diff --git a/hypothesis-python/tests/cover/test_lookup_py310.py b/hypothesis-python/tests/cover/test_lookup_py310.py new file mode 100644 index 0000000000..c24d1268ef --- /dev/null +++ b/hypothesis-python/tests/cover/test_lookup_py310.py @@ -0,0 +1,19 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +from hypothesis import strategies as st + +from tests.common.debug import find_any + + +def test_native_unions(): + s = st.from_type(int | list[str]) + find_any(s, lambda x: isinstance(x, int)) + find_any(s, lambda x: isinstance(x, list)) diff --git a/hypothesis-python/tests/cover/test_lookup_py37.py b/hypothesis-python/tests/cover/test_lookup_py37.py index e73ab1d388..6a1955eb6b 100644 --- a/hypothesis-python/tests/cover/test_lookup_py37.py +++ b/hypothesis-python/tests/cover/test_lookup_py37.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from __future__ import annotations @@ -23,10 +18,9 @@ import pytest -from hypothesis import assume, given, infer +from hypothesis import assume, given -# This test file is skippped on Python <= 3.6, where `list[Elem]` is always a -# SyntaxError. On Python 3.7 and 3.8, `from __future__ import annotations` means +# On Python 3.7 and 3.8, `from __future__ import annotations` means # that the syntax is supported; but the feature fails at runtime. On Python # 3.9 and later, it should all work. # @@ -34,7 +28,7 @@ if sys.version_info < (3, 9): pytestmark = pytest.mark.xfail( - raises=Exception, strict=True, reason="Requires Python 3.9 (PEP 585) or later." + raises=Exception, reason="Requires Python 3.9 (PEP 585) or later." ) @@ -52,48 +46,48 @@ def check(t, ex): assume(ex) -@given(x=infer) +@given(...) def test_resolving_standard_tuple1_as_generic(x: tuple[Elem]): check(tuple, x) -@given(x=infer) +@given(...) def test_resolving_standard_tuple2_as_generic(x: tuple[Elem, Elem]): check(tuple, x) -@given(x=infer) +@given(...) def test_resolving_standard_tuple_variadic_as_generic(x: tuple[Elem, ...]): check(tuple, x) -@given(x=infer) +@given(...) def test_resolving_standard_list_as_generic(x: list[Elem]): check(list, x) -@given(x=infer) +@given(...) def test_resolving_standard_dict_as_generic(x: dict[Elem, Value]): check(dict, x) assert all(isinstance(e, Value) for e in x.values()) -@given(x=infer) +@given(...) def test_resolving_standard_set_as_generic(x: set[Elem]): check(set, x) -@given(x=infer) +@given(...) def test_resolving_standard_frozenset_as_generic(x: frozenset[Elem]): check(frozenset, x) -@given(x=infer) +@given(...) def test_resolving_standard_deque_as_generic(x: collections.deque[Elem]): check(collections.deque, x) -@given(x=infer) +@given(...) def test_resolving_standard_defaultdict_as_generic( x: collections.defaultdict[Elem, Value] ): @@ -101,7 +95,7 @@ def test_resolving_standard_defaultdict_as_generic( assert all(isinstance(e, Value) for e in x.values()) -@given(x=infer) +@given(...) def test_resolving_standard_ordered_dict_as_generic( x: collections.OrderedDict[Elem, Value] ): @@ -109,29 +103,29 @@ def test_resolving_standard_ordered_dict_as_generic( assert all(isinstance(e, Value) for e in x.values()) -@given(x=infer) +@given(...) def test_resolving_standard_counter_as_generic(x: collections.Counter[Elem]): check(collections.Counter, x) assume(any(x.values())) # Check that we generated at least one nonzero count -@given(x=infer) +@given(...) def test_resolving_standard_chainmap_as_generic(x: collections.ChainMap[Elem, Value]): check(collections.ChainMap, x) assert all(isinstance(e, Value) for e in x.values()) -@given(x=infer) +@given(...) def test_resolving_standard_iterable_as_generic(x: collections.abc.Iterable[Elem]): check(collections.abc.Iterable, x) -@given(x=infer) +@given(...) def test_resolving_standard_iterator_as_generic(x: collections.abc.Iterator[Elem]): check(collections.abc.Iterator, x) -@given(x=infer) +@given(...) def test_resolving_standard_generator_as_generic( x: collections.abc.Generator[Elem, None, Value] ): @@ -145,22 +139,22 @@ def test_resolving_standard_generator_as_generic( assert isinstance(stop.value, Value) -@given(x=infer) +@given(...) def test_resolving_standard_reversible_as_generic(x: collections.abc.Reversible[Elem]): check(collections.abc.Reversible, x) -@given(x=infer) +@given(...) def test_resolving_standard_container_as_generic(x: collections.abc.Container[Elem]): check(collections.abc.Container, x) -@given(x=infer) +@given(...) def test_resolving_standard_collection_as_generic(x: collections.abc.Collection[Elem]): check(collections.abc.Collection, x) -@given(x=infer) +@given(...) def test_resolving_standard_callable_ellipsis(x: collections.abc.Callable[..., Elem]): assert isinstance(x, collections.abc.Callable) assert callable(x) @@ -169,7 +163,7 @@ def test_resolving_standard_callable_ellipsis(x: collections.abc.Callable[..., E assert isinstance(x(1, 2, 3, a=4, b=5, c=6), Elem) -@given(x=infer) +@given(...) def test_resolving_standard_callable_no_args(x: collections.abc.Callable[[], Elem]): assert isinstance(x, collections.abc.Callable) assert callable(x) @@ -181,25 +175,25 @@ def test_resolving_standard_callable_no_args(x: collections.abc.Callable[[], Ele x(a=1) -@given(x=infer) +@given(...) def test_resolving_standard_collections_set_as_generic(x: collections.abc.Set[Elem]): check(collections.abc.Set, x) -@given(x=infer) +@given(...) def test_resolving_standard_collections_mutableset_as_generic( x: collections.abc.MutableSet[Elem], ): check(collections.abc.MutableSet, x) -@given(x=infer) +@given(...) def test_resolving_standard_mapping_as_generic(x: collections.abc.Mapping[Elem, Value]): check(collections.abc.Mapping, x) assert all(isinstance(e, Value) for e in x.values()) -@given(x=infer) +@given(...) def test_resolving_standard_mutable_mapping_as_generic( x: collections.abc.MutableMapping[Elem, Value], ): @@ -207,24 +201,24 @@ def test_resolving_standard_mutable_mapping_as_generic( assert all(isinstance(e, Value) for e in x.values()) -@given(x=infer) +@given(...) def test_resolving_standard_sequence_as_generic(x: collections.abc.Sequence[Elem]): check(collections.abc.Sequence, x) -@given(x=infer) +@given(...) def test_resolving_standard_mutable_sequence_as_generic( x: collections.abc.MutableSequence[Elem], ): check(collections.abc.MutableSequence, x) -@given(x=infer) +@given(...) def test_resolving_standard_keysview_as_generic(x: collections.abc.KeysView[Elem]): check(collections.abc.KeysView, x) -@given(x=infer) +@given(...) def test_resolving_standard_itemsview_as_generic( x: collections.abc.ItemsView[Elem, Value] ): @@ -233,37 +227,38 @@ def test_resolving_standard_itemsview_as_generic( assume(x) -@given(x=infer) +@given(...) def test_resolving_standard_valuesview_as_generic(x: collections.abc.ValuesView[Elem]): check(collections.abc.ValuesView, x) -@given(x=infer) +@pytest.mark.xfail # Weird interaction with fixes in PR #2952 +@given(...) def test_resolving_standard_contextmanager_as_generic( x: contextlib.AbstractContextManager[Elem], ): assert isinstance(x, contextlib.AbstractContextManager) -@given(x=infer) +@given(...) def test_resolving_standard_re_match_bytes_as_generic(x: re.Match[bytes]): assert isinstance(x, re.Match) assert isinstance(x[0], bytes) -@given(x=infer) +@given(...) def test_resolving_standard_re_match_str_as_generic(x: re.Match[str]): assert isinstance(x, re.Match) assert isinstance(x[0], str) -@given(x=infer) +@given(...) def test_resolving_standard_re_pattern_bytes_as_generic(x: re.Pattern[bytes]): assert isinstance(x, re.Pattern) assert isinstance(x.pattern, bytes) -@given(x=infer) +@given(...) def test_resolving_standard_re_pattern_str_as_generic(x: re.Pattern[str]): assert isinstance(x, re.Pattern) assert isinstance(x.pattern, str) diff --git a/hypothesis-python/tests/cover/test_lookup_py38.py b/hypothesis-python/tests/cover/test_lookup_py38.py index 3bfe4c3c1a..5b65389998 100644 --- a/hypothesis-python/tests/cover/test_lookup_py38.py +++ b/hypothesis-python/tests/cover/test_lookup_py38.py @@ -1,27 +1,31 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import dataclasses +import re +import sys import typing +from types import SimpleNamespace import pytest -from hypothesis import given, strategies as st +from hypothesis import example, given, strategies as st +from hypothesis.errors import InvalidArgument, Unsatisfiable +from hypothesis.internal.reflection import ( + convert_positional_arguments, + get_pretty_function_description, +) from hypothesis.strategies import from_type from tests.common.debug import find_any +from tests.common.utils import fails_with, temp_registered @given(st.data()) @@ -75,7 +79,15 @@ def test_typeddict_with_optional(value): assert isinstance(value["b"], bool) -@pytest.mark.xfail +if sys.version_info[:2] < (3, 9): + xfail_on_38 = pytest.mark.xfail(raises=Unsatisfiable) +else: + + def xfail_on_38(f): + return f + + +@xfail_on_38 def test_simple_optional_key_is_optional(): # Optional keys are not currently supported, as PEP-589 leaves no traces # at runtime. See https://github.com/python/cpython/pull/17214 @@ -97,7 +109,18 @@ def test_typeddict_with_optional_then_required_again(value): assert isinstance(value["c"], str) -@pytest.mark.xfail +class NestedDict(typing.TypedDict): + inner: A + + +@given(from_type(NestedDict)) +def test_typeddict_with_nested_value(value): + assert type(value) == dict + assert set(value) == {"inner"} + assert isinstance(value["inner"]["a"], int) + + +@xfail_on_38 def test_layered_optional_key_is_optional(): # Optional keys are not currently supported, as PEP-589 leaves no traces # at runtime. See https://github.com/python/cpython/pull/17214 @@ -113,3 +136,116 @@ class Node: @given(st.builds(Node)) def test_can_resolve_recursive_dataclass(val): assert isinstance(val, Node) + + +def test_can_register_new_type_for_typeddicts(): + sentinel = object() + with temp_registered(C, st.just(sentinel)): + assert st.from_type(C).example() is sentinel + + +@pytest.mark.parametrize( + "lam,source", + [ + ((lambda a, /, b: a), "lambda a, /, b: a"), + ((lambda a=None, /, b=None: a), "lambda a=None, /, b=None: a"), + ], +) +def test_posonly_lambda_formatting(lam, source): + # Testing posonly lambdas, with and without default values + assert get_pretty_function_description(lam) == source + + +def test_does_not_convert_posonly_to_keyword(): + args, kws = convert_positional_arguments(lambda x, /: None, (1,), {}) + assert args + assert not kws + + +@given(x=st.booleans()) +def test_given_works_with_keyword_only_params(*, x): + pass + + +def test_given_works_with_keyword_only_params_some_unbound(): + @given(x=st.booleans()) + def test(*, x, y): + assert y is None + + test(y=None) + + +def test_given_works_with_positional_only_params(): + @given(y=st.booleans()) + def test(x, /, y): + pass + + test(None) + + +def test_cannot_pass_strategies_by_position_if_there_are_posonly_args(): + @given(st.booleans()) + def test(x, /, y): + pass + + with pytest.raises(InvalidArgument): + test(None) + + +@fails_with(InvalidArgument) +@given(st.booleans()) +def test_cannot_pass_strategies_for_posonly_args(x, /): + pass + + +@given(y=st.booleans()) +def has_posonly_args(x, /, y): + pass + + +def test_example_argument_validation(): + example(y=None)(has_posonly_args)(1) # Basic case is OK + + with pytest.raises( + InvalidArgument, + match=re.escape( + "Cannot pass positional arguments to @example() when decorating " + "a test function which has positional-only parameters." + ), + ): + example(None)(has_posonly_args)(1) + + with pytest.raises( + InvalidArgument, + match=re.escape( + "Inconsistent args: @given() got strategies for 'y', " + "but @example() got arguments for 'x'" + ), + ): + example(x=None)(has_posonly_args)(1) + + +class FooProtocol(typing.Protocol): + def frozzle(self, x): + pass + + +class BarProtocol(typing.Protocol): + def bazzle(self, y): + pass + + +@given(st.data()) +def test_can_resolve_registered_protocol(data): + with temp_registered( + FooProtocol, + st.builds(SimpleNamespace, frozzle=st.functions(like=lambda x: ...)), + ): + obj = data.draw(st.from_type(FooProtocol)) + assert obj.frozzle(x=1) is None + + +def test_cannot_resolve_un_registered_protocol(): + msg = "Instance and class checks can only be used with @runtime_checkable protocols" + with pytest.raises(TypeError, match=msg): + st.from_type(BarProtocol).example() diff --git a/hypothesis-python/tests/cover/test_lookup_py39.py b/hypothesis-python/tests/cover/test_lookup_py39.py new file mode 100644 index 0000000000..7d7a5ac447 --- /dev/null +++ b/hypothesis-python/tests/cover/test_lookup_py39.py @@ -0,0 +1,93 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import dataclasses +import sys +import typing + +import pytest + +from hypothesis import given, strategies as st +from hypothesis.errors import InvalidArgument + +from tests.common.debug import find_any + + +@pytest.mark.parametrize( + "annotated_type,expected_strategy_repr", + [ + (typing.Annotated[int, "foo"], "integers()"), + (typing.Annotated[typing.List[float], "foo"], "lists(floats())"), + (typing.Annotated[typing.Annotated[str, "foo"], "bar"], "text()"), + ( + typing.Annotated[ + typing.Annotated[typing.List[typing.Dict[str, bool]], "foo"], "bar" + ], + "lists(dictionaries(keys=text(), values=booleans()))", + ), + ], +) +def test_typing_Annotated(annotated_type, expected_strategy_repr): + assert repr(st.from_type(annotated_type)) == expected_strategy_repr + + +PositiveInt = typing.Annotated[int, st.integers(min_value=1)] +MoreThenTenInt = typing.Annotated[PositiveInt, st.integers(min_value=10 + 1)] +WithTwoStrategies = typing.Annotated[int, st.integers(), st.none()] +ExtraAnnotationNoStrategy = typing.Annotated[PositiveInt, "metadata"] + + +def arg_positive(x: PositiveInt): + assert x > 0 + + +def arg_more_than_ten(x: MoreThenTenInt): + assert x > 10 + + +@given(st.data()) +def test_annotated_positive_int(data): + data.draw(st.builds(arg_positive)) + + +@given(st.data()) +def test_annotated_more_than_ten(data): + data.draw(st.builds(arg_more_than_ten)) + + +@given(st.data()) +def test_annotated_with_two_strategies(data): + assert data.draw(st.from_type(WithTwoStrategies)) is None + + +@given(st.data()) +def test_annotated_extra_metadata(data): + assert data.draw(st.from_type(ExtraAnnotationNoStrategy)) > 0 + + +@dataclasses.dataclass +class User: + id: int + following: list["User"] # works with typing.List + + +@pytest.mark.skipif(sys.version_info[:2] >= (3, 11), reason="works in new Pythons") +def test_string_forward_ref_message(): + # See https://github.com/HypothesisWorks/hypothesis/issues/3016 + s = st.builds(User) + with pytest.raises(InvalidArgument, match="`from __future__ import annotations`"): + s.example() + + +def test_issue_3080(): + # Check for https://github.com/HypothesisWorks/hypothesis/issues/3080 + s = st.from_type(typing.Union[list[int], int]) + find_any(s, lambda x: isinstance(x, int)) + find_any(s, lambda x: isinstance(x, list)) diff --git a/hypothesis-python/tests/cover/test_map.py b/hypothesis-python/tests/cover/test_map.py index 56a22455ee..11eed6dc1f 100644 --- a/hypothesis-python/tests/cover/test_map.py +++ b/hypothesis-python/tests/cover/test_map.py @@ -1,19 +1,15 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import assume, given, strategies as st +from hypothesis.strategies._internal.lazy import unwrap_strategies from tests.common.debug import assert_no_examples @@ -25,3 +21,8 @@ def test_can_assume_in_map(x): def test_assume_in_just_raises_immediately(): assert_no_examples(st.just(1).map(lambda x: assume(x == 2))) + + +def test_identity_map_is_noop(): + s = unwrap_strategies(st.integers()) + assert s.map(lambda x: x) is s diff --git a/hypothesis-python/tests/cover/test_mock.py b/hypothesis-python/tests/cover/test_mock.py index 09f2b33b4a..ef11610fc8 100644 --- a/hypothesis-python/tests/cover/test_mock.py +++ b/hypothesis-python/tests/cover/test_mock.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """Checks that @given, @mock.patch, and pytest fixtures work as expected.""" @@ -19,7 +14,10 @@ import math from unittest import mock -from _pytest.config import Config +try: + from pytest import Config +except ImportError: # pytest<7.0.0 + from _pytest.config import Config from hypothesis import given, strategies as st diff --git a/hypothesis-python/tests/cover/test_nothing.py b/hypothesis-python/tests/cover/test_nothing.py index efd1ecb596..1d61387173 100644 --- a/hypothesis-python/tests/cover/test_nothing.py +++ b/hypothesis-python/tests/cover/test_nothing.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest diff --git a/hypothesis-python/tests/cover/test_numerics.py b/hypothesis-python/tests/cover/test_numerics.py index bd405b1f11..de63164608 100644 --- a/hypothesis-python/tests/cover/test_numerics.py +++ b/hypothesis-python/tests/cover/test_numerics.py @@ -1,26 +1,21 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import decimal from math import copysign, inf import pytest -from hypothesis import assume, given, reject, settings -from hypothesis.errors import HypothesisDeprecationWarning, InvalidArgument -from hypothesis.internal.floats import next_down +from hypothesis import HealthCheck, assume, given, reject, settings +from hypothesis.errors import InvalidArgument +from hypothesis.internal.floats import next_down, next_up from hypothesis.strategies import ( booleans, data, @@ -29,36 +24,38 @@ fractions, integers, none, + sampled_from, tuples, ) from tests.common.debug import find_any +@settings(suppress_health_check=HealthCheck.all()) @given(data()) def test_fuzz_floats_bounds(data): - bound = none() | floats(allow_nan=False) + width = data.draw(sampled_from([64, 32, 16])) + bound = none() | floats(allow_nan=False, width=width) low, high = data.draw(tuples(bound, bound), label="low, high") + if low is not None and high is not None and low > high: + low, high = high, low if low is not None and high is not None and low > high: low, high = high, low exmin = low is not None and low != inf and data.draw(booleans(), label="ex_min") exmax = high is not None and high != -inf and data.draw(booleans(), label="ex_max") - try: - val = data.draw( - floats(low, high, exclude_min=exmin, exclude_max=exmax), label="value" - ) - assume(val) # positive/negative zero is an issue - except (InvalidArgument, HypothesisDeprecationWarning): - assert ( - (exmin and exmax and low == next_down(high)) - or (low == high and (exmin or exmax)) - or ( - low == high == 0 - and copysign(1.0, low) == 1 - and copysign(1.0, high) == -1 - ) - ) - reject() # no floats in required range + + if low is not None and high is not None: + lo = next_up(low, width) if exmin else low + hi = next_down(high, width) if exmax else high + # There must actually be floats between these bounds + assume(lo <= hi) + if lo == hi == 0: + assume(not exmin and not exmax and copysign(1.0, lo) <= copysign(1.0, hi)) + + s = floats(low, high, exclude_min=exmin, exclude_max=exmax, width=width) + val = data.draw(s, label="value") + assume(val) # positive/negative zero is an issue + if low is not None: assert low <= val if high is not None: @@ -130,7 +127,7 @@ def test_decimals_include_nan(): def test_decimals_include_inf(): - find_any(decimals(), lambda x: x.is_infinite(), settings(max_examples=10 ** 6)) + find_any(decimals(), lambda x: x.is_infinite(), settings(max_examples=10**6)) @given(decimals(allow_nan=False)) @@ -175,3 +172,30 @@ def test_consistent_decimal_error(): with decimal.localcontext(decimal.Context(traps=[])): decimals(bad).example() assert str(excinfo.value) == str(excinfo2.value) + + +@pytest.mark.parametrize( + "s, msg", + [ + ( + floats(min_value=inf, allow_infinity=False), + "allow_infinity=False excludes min_value=inf", + ), + ( + floats(min_value=next_down(inf), exclude_min=True, allow_infinity=False), + "exclude_min=True turns min_value=.+? into inf, but allow_infinity=False", + ), + ( + floats(max_value=-inf, allow_infinity=False), + "allow_infinity=False excludes max_value=-inf", + ), + ( + floats(max_value=next_up(-inf), exclude_max=True, allow_infinity=False), + "exclude_max=True turns max_value=.+? into -inf, but allow_infinity=False", + ), + ], +) +def test_floats_message(s, msg): + # https://github.com/HypothesisWorks/hypothesis/issues/3207 + with pytest.raises(InvalidArgument, match=msg): + s.validate() diff --git a/hypothesis-python/tests/cover/test_one_of.py b/hypothesis-python/tests/cover/test_one_of.py index 8c9ddbc024..72bc7b55d3 100644 --- a/hypothesis-python/tests/cover/test_one_of.py +++ b/hypothesis-python/tests/cover/test_one_of.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import re diff --git a/hypothesis-python/tests/cover/test_permutations.py b/hypothesis-python/tests/cover/test_permutations.py index 57ec129211..c6712bb56f 100644 --- a/hypothesis-python/tests/cover/test_permutations.py +++ b/hypothesis-python/tests/cover/test_permutations.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import given from hypothesis.errors import InvalidArgument diff --git a/hypothesis-python/tests/cover/test_phases.py b/hypothesis-python/tests/cover/test_phases.py index 8e3a6807d2..8ccca601ed 100644 --- a/hypothesis-python/tests/cover/test_phases.py +++ b/hypothesis-python/tests/cover/test_phases.py @@ -1,21 +1,17 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest from hypothesis import Phase, example, given, settings, strategies as st +from hypothesis._settings import all_settings from hypothesis.database import ExampleDatabase, InMemoryExampleDatabase from hypothesis.errors import InvalidArgument @@ -37,7 +33,7 @@ def test_does_not_use_explicit_examples(i): @settings(phases=(Phase.reuse, Phase.shrink)) @given(st.booleans()) def test_this_would_fail_if_you_ran_it(b): - assert False + raise AssertionError @pytest.mark.parametrize( @@ -51,8 +47,8 @@ def test_sorts_and_dedupes_phases(arg, expected): assert settings(phases=arg).phases == expected -def test_phases_default_to_all(): - assert settings().phases == tuple(Phase) +def test_phases_default_to_all_except_explain(): + assert all_settings["phases"].default + (Phase.explain,) == tuple(Phase) def test_does_not_reuse_saved_examples_if_reuse_not_in_phases(): @@ -90,7 +86,7 @@ def test_usage(i): with pytest.raises(ValueError): test_usage() - (saved,) = [v for k, v in database.data.items() if b"pareto" not in k] + (saved,) = (v for k, v in database.data.items() if b"pareto" not in k) assert len(saved) == 1 diff --git a/hypothesis-python/tests/cover/test_posonly_args_py38.py b/hypothesis-python/tests/cover/test_posonly_args_py38.py new file mode 100644 index 0000000000..f22082a4d7 --- /dev/null +++ b/hypothesis-python/tests/cover/test_posonly_args_py38.py @@ -0,0 +1,34 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import pytest + +from hypothesis import given, strategies as st + + +@st.composite +def strat(draw, x=0, /): + return draw(st.integers(min_value=x)) + + +@given(st.data(), st.integers()) +def test_composite_with_posonly_args(data, min_value): + v = data.draw(strat(min_value)) + assert min_value <= v + + +def test_preserves_signature(): + with pytest.raises(TypeError): + strat(x=1) + + +def test_builds_real_pos_only(): + with pytest.raises(TypeError): + st.builds() # requires a target! diff --git a/hypothesis-python/tests/cover/test_pretty.py b/hypothesis-python/tests/cover/test_pretty.py index 7744a73d58..86359362fd 100644 --- a/hypothesis-python/tests/cover/test_pretty.py +++ b/hypothesis-python/tests/cover/test_pretty.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """This file originates in the IPython project and is made use of under the following licensing terms: @@ -59,37 +54,9 @@ import pytest from hypothesis.internal.compat import PYPY +from hypothesis.strategies._internal.numbers import SIGNALING_NAN from hypothesis.vendor import pretty -from tests.common.utils import capture_out - - -def unicode_to_str(x, encoding=None): - return x - - -def assert_equal(x, y): - assert x == y - - -def assert_true(x): - assert x - - -def assert_in(x, xs): - assert x in xs - - -def skip_without(mod): - try: - __import__(mod) - return lambda f: f - except ImportError: - return pytest.mark.skipif(True, reason=f"Missing {mod}") - - -assert_raises = pytest.raises - class MyList: def __init__(self, content): @@ -167,6 +134,7 @@ def test_list(): def test_dict(): assert pretty.pretty({}) == "{}" assert pretty.pretty({1: 1}) == "{1: 1}" + assert pretty.pretty({1: 1, 0: 0}) == "{1: 1, 0: 0}" def test_tuple(): @@ -208,7 +176,7 @@ def test_indentation(): gotoutput = pretty.pretty(MyList(range(count))) expectedoutput = "MyList(\n" + ",\n".join(f" {i}" for i in range(count)) + ")" - assert_equal(gotoutput, expectedoutput) + assert gotoutput == expectedoutput def test_dispatch(): @@ -217,7 +185,7 @@ def test_dispatch(): gotoutput = pretty.pretty(MyDict()) expectedoutput = "MyDict(...)" - assert_equal(gotoutput, expectedoutput) + assert gotoutput == expectedoutput def test_callability_checking(): @@ -226,7 +194,7 @@ def test_callability_checking(): gotoutput = pretty.pretty(Dummy2()) expectedoutput = "Dummy1(...)" - assert_equal(gotoutput, expectedoutput) + assert gotoutput == expectedoutput def test_sets(): @@ -251,7 +219,7 @@ def test_sets(): ] for obj, expected_output in zip(objects, expected): got_output = pretty.pretty(obj) - assert_equal(got_output, expected_output) + assert got_output == expected_output def test_unsortable_set(): @@ -268,38 +236,29 @@ def test_unsortable_dict(): assert pretty.pretty(x) in p -@skip_without("xxlimited") -def test_pprint_heap_allocated_type(): - """Test that pprint works for heap allocated types.""" - import xxlimited - - output = pretty.pretty(xxlimited.Null) - assert_equal(output, "xxlimited.Null") - - def test_pprint_nomod(): """Test that pprint works for classes with no __module__.""" output = pretty.pretty(NoModule) - assert_equal(output, "NoModule") + assert output == "NoModule" def test_pprint_break(): """Test that p.break_ produces expected output.""" output = pretty.pretty(Breaking()) expected = "TG: Breaking(\n ):" - assert_equal(output, expected) + assert output == expected def test_pprint_break_repr(): """Test that p.break_ is used in repr.""" output = pretty.pretty(BreakingReprParent()) expected = "TG: Breaking(\n ):" - assert_equal(output, expected) + assert output == expected def test_bad_repr(): """Don't catch bad repr errors.""" - with assert_raises(ZeroDivisionError): + with pytest.raises(ZeroDivisionError): pretty.pretty(BadRepr()) @@ -320,7 +279,7 @@ def __repr__(self): def test_really_bad_repr(): - with assert_raises(BadException): + with pytest.raises(BadException): pretty.pretty(ReallyBadRepr()) @@ -337,12 +296,11 @@ class SB(SA): def test_super_repr(): output = pretty.pretty(super(SA)) - assert_in("SA", output) + assert "SA" in output sb = SB() output = pretty.pretty(super(SA, sb)) - assert_in("SA", output) - + assert "SA" in output except AttributeError: @@ -356,41 +314,40 @@ def test_long_list(): lis = list(range(10000)) p = pretty.pretty(lis) last2 = p.rsplit("\n", 2)[-2:] - assert_equal(last2, [" 999,", " ...]"]) + assert last2 == [" 999,", " ...]"] def test_long_set(): s = set(range(10000)) p = pretty.pretty(s) last2 = p.rsplit("\n", 2)[-2:] - assert_equal(last2, [" 999,", " ...}"]) + assert last2 == [" 999,", " ...}"] def test_long_tuple(): tup = tuple(range(10000)) p = pretty.pretty(tup) last2 = p.rsplit("\n", 2)[-2:] - assert_equal(last2, [" 999,", " ...)"]) + assert last2 == [" 999,", " ...)"] def test_long_dict(): d = {n: n for n in range(10000)} p = pretty.pretty(d) last2 = p.rsplit("\n", 2)[-2:] - assert_equal(last2, [" 999: 999,", " ...}"]) + assert last2 == [" 999: 999,", " ...}"] def test_unbound_method(): - output = pretty.pretty(MyObj.somemethod) - assert_in("MyObj.somemethod", output) + assert pretty.pretty(MyObj.somemethod) == "somemethod" class MetaClass(type): - def __new__(cls, name): - return type.__new__(cls, name, (object,), {"name": name}) + def __new__(metacls, name): + return type.__new__(metacls, name, (object,), {"name": name}) - def __repr__(self): - return f"[CUSTOM REPR FOR CLASS {self.name}]" + def __repr__(cls): + return f"[CUSTOM REPR FOR CLASS {cls.name}]" ClassWithMeta = MetaClass("ClassWithMeta") @@ -398,22 +355,21 @@ def __repr__(self): def test_metaclass_repr(): output = pretty.pretty(ClassWithMeta) - assert_equal(output, "[CUSTOM REPR FOR CLASS ClassWithMeta]") + assert output == "[CUSTOM REPR FOR CLASS ClassWithMeta]" def test_unicode_repr(): u = "üniçodé" - ustr = unicode_to_str(u) class C: def __repr__(self): - return ustr + return u c = C() p = pretty.pretty(c) - assert_equal(p, u) + assert p == u p = pretty.pretty([c]) - assert_equal(p, f"[{u}]") + assert p == f"[{u}]" def test_basic_class(): @@ -431,8 +387,8 @@ def type_pprint_wrapper(obj, p, cycle): printer.flush() output = stream.getvalue() - assert_equal(output, f"{__name__}.MyObj") - assert_true(type_pprint_wrapper.called) + assert output == f"{__name__}.MyObj" + assert type_pprint_wrapper.called def test_collections_defaultdict(): @@ -455,7 +411,7 @@ def test_collections_defaultdict(): (b, "defaultdict(list, {'key': defaultdict(...)})"), ] for obj, expected in cases: - assert_equal(pretty.pretty(obj), expected) + assert pretty.pretty(obj) == expected @pytest.mark.skipif(PYPY, reason="slightly different on PyPy3") @@ -482,7 +438,7 @@ def test_collections_ordereddict(): (a, "OrderedDict([('key', OrderedDict(...))])"), ] for obj, expected in cases: - assert_equal(pretty.pretty(obj), expected) + assert pretty.pretty(obj) == expected def test_collections_deque(): @@ -518,7 +474,7 @@ def test_collections_deque(): (a, "deque([deque(...)])"), ] for obj, expected in cases: - assert_equal(pretty.pretty(obj), expected) + assert pretty.pretty(obj) == expected def test_collections_counter(): @@ -531,7 +487,7 @@ class MyCounter(Counter): (MyCounter(a=1), "MyCounter({'a': 1})"), ] for obj, expected in cases: - assert_equal(pretty.pretty(obj), expected) + assert pretty.pretty(obj) == expected def test_cyclic_list(): @@ -583,13 +539,6 @@ def test_cyclic_set(): assert pretty.pretty(x) == "{{...}}" -def test_pprint(): - t = {"hi": 1} - with capture_out() as o: - pretty.pprint(t) - assert o.getvalue().strip() == pretty.pretty(t) - - class BigList(list): def _repr_pretty_(self, printer, cycle): if cycle: @@ -620,7 +569,7 @@ def test_re_evals(): for r in [ re.compile(r"hi"), re.compile(r"b\nc", re.MULTILINE), - re.compile(br"hi", 0), + re.compile(rb"hi", 0), re.compile("foo", re.MULTILINE | re.UNICODE), ]: r2 = eval(pretty.pretty(r), globals()) @@ -648,11 +597,11 @@ def test_custom(): def test_print_builtin_function(): - assert pretty.pretty(abs) == "" + assert pretty.pretty(abs) == "abs" def test_pretty_function(): - assert "." in pretty.pretty(test_pretty_function) + assert pretty.pretty(test_pretty_function) == "test_pretty_function" def test_empty_printer(): @@ -668,3 +617,17 @@ def test_empty_printer(): def test_breakable_at_group_boundary(): assert "\n" in pretty.pretty([[], "000000"], max_width=5) + + +@pytest.mark.parametrize( + "obj, rep", + [ + (float("nan"), "nan"), + (-float("nan"), "-nan"), + (SIGNALING_NAN, "nan # Saw 1 signaling NaN"), + (-SIGNALING_NAN, "-nan # Saw 1 signaling NaN"), + ((SIGNALING_NAN, SIGNALING_NAN), "(nan, nan) # Saw 2 signaling NaNs"), + ], +) +def test_nan_reprs(obj, rep): + assert pretty.pretty(obj) == rep diff --git a/hypothesis-python/tests/cover/test_provisional_strategies.py b/hypothesis-python/tests/cover/test_provisional_strategies.py index 320f049ba9..621db25631 100644 --- a/hypothesis-python/tests/cover/test_provisional_strategies.py +++ b/hypothesis-python/tests/cover/test_provisional_strategies.py @@ -1,26 +1,26 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import re import string import pytest -from hypothesis import given +from hypothesis import given, settings from hypothesis.errors import InvalidArgument -from hypothesis.provisional import domains, urls +from hypothesis.provisional import ( + FRAGMENT_SAFE_CHARACTERS, + _url_fragments_strategy, + domains, + urls, +) from tests.common.debug import find_any @@ -29,12 +29,23 @@ def test_is_URL(url): allowed_chars = set(string.ascii_letters + string.digits + "$-_.+!*'(),~%/") url_schemeless = url.split("://", 1)[1] - path = url_schemeless.split("/", 1)[1] if "/" in url_schemeless else "" + components = url_schemeless.split("#", 1) + + domain_path = components[0] + path = domain_path.split("/", 1)[1] if "/" in domain_path else "" assert all(c in allowed_chars for c in path) assert all( re.match("^[0-9A-Fa-f]{2}", after_perc) for after_perc in path.split("%")[1:] ) + fragment = components[1] if "#" in url_schemeless else "" + fragment_allowed_chars = allowed_chars | {"?"} + assert all(c in fragment_allowed_chars for c in fragment) + assert all( + re.match("^[0-9A-Fa-f]{2}", after_perc) + for after_perc in fragment.split("%")[1:] + ) + @pytest.mark.parametrize("max_length", [-1, 0, 3, 4.0, 256]) @pytest.mark.parametrize("max_element_length", [-1, 0, 4.0, 64, 128]) @@ -52,3 +63,18 @@ def test_valid_domains_arguments(max_length, max_element_length): @pytest.mark.parametrize("strategy", [domains(), urls()]) def test_find_any_non_empty(strategy): find_any(strategy, lambda s: len(s) > 0) + + +@given(_url_fragments_strategy) +# There's a lambda in the implementation that only gets run if we generate at +# least one percent-escape sequence, so we derandomize to ensure that coverage +# isn't flaky. +@settings(derandomize=True) +def test_url_fragments_contain_legal_chars(fragment): + assert fragment.startswith("#") + + # Strip all legal escape sequences. Any remaining % characters were not + # part of a legal escape sequence. + without_escapes = re.sub(r"(?ai)%[0-9a-f][0-9a-f]", "", fragment[1:]) + + assert set(without_escapes).issubset(FRAGMENT_SAFE_CHARACTERS) diff --git a/hypothesis-python/tests/cover/test_random_module.py b/hypothesis-python/tests/cover/test_random_module.py index 75d7063563..f8979db831 100644 --- a/hypothesis-python/tests/cover/test_random_module.py +++ b/hypothesis-python/tests/cover/test_random_module.py @@ -1,41 +1,42 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER +import gc import random import pytest -from hypothesis import find, given, register_random, reporting, strategies as st -from hypothesis.errors import InvalidArgument +from hypothesis import core, find, given, register_random, strategies as st +from hypothesis.errors import HypothesisWarning, InvalidArgument from hypothesis.internal import entropy +from hypothesis.internal.compat import PYPY from hypothesis.internal.entropy import deterministic_PRNG -from tests.common.utils import capture_out +def gc_on_pypy(): + # CPython uses reference counting, so objects (without circular refs) + # are collected immediately on `del`, breaking weak references. + # PyPy doesn't, so we use this function in tests before counting the + # surviving references to ensure that they're deterministic. + if PYPY: + gc.collect() -def test_can_seed_random(): - with capture_out() as out: - with reporting.with_reporter(reporting.default): - with pytest.raises(AssertionError): - @given(st.random_module()) - def test(r): - assert False +def test_can_seed_random(): + @given(st.random_module()) + def test(r): + raise AssertionError - test() - assert "RandomSeeder(0)" in out.getvalue() + with pytest.raises(AssertionError) as err: + test() + assert "RandomSeeder(0)" in "\n".join(err.value.__notes__) @given(st.random_module(), st.random_module()) @@ -53,12 +54,19 @@ def test_cannot_register_non_Random(): register_random("not a Random instance") +@pytest.mark.filterwarnings( + "ignore:It looks like `register_random` was passed an object that could be garbage collected" +) def test_registering_a_Random_is_idempotent(): + gc_on_pypy() + n_registered = len(entropy.RANDOMS_TO_MANAGE) r = random.Random() register_random(r) register_random(r) - assert entropy.RANDOMS_TO_MANAGE.pop() is r - assert r not in entropy.RANDOMS_TO_MANAGE + assert len(entropy.RANDOMS_TO_MANAGE) == n_registered + 1 + del r + gc_on_pypy() + assert len(entropy.RANDOMS_TO_MANAGE) == n_registered def test_manages_registered_Random_instance(): @@ -78,9 +86,6 @@ def inner(x): inner() assert state == r.getstate() - entropy.RANDOMS_TO_MANAGE.remove(r) - assert r not in entropy.RANDOMS_TO_MANAGE - def test_registered_Random_is_seeded_by_random_module_strategy(): r = random.Random() @@ -98,9 +103,6 @@ def inner(x): assert count[0] > len(results) * 0.9, "too few unique random numbers" assert state == r.getstate() - entropy.RANDOMS_TO_MANAGE.remove(r) - assert r not in entropy.RANDOMS_TO_MANAGE - @given(st.random_module()) def test_will_actually_use_the_random_seed(rnd): @@ -120,11 +122,14 @@ def test(r): test() state_a = random.getstate() + state_a2 = core._hypothesis_global_random.getstate() test() state_b = random.getstate() + state_b2 = core._hypothesis_global_random.getstate() - assert state_a != state_b + assert state_a == state_b + assert state_a2 != state_b2 def test_find_does_not_pollute_state(): @@ -132,8 +137,98 @@ def test_find_does_not_pollute_state(): find(st.random_module(), lambda r: True) state_a = random.getstate() + state_a2 = core._hypothesis_global_random.getstate() find(st.random_module(), lambda r: True) state_b = random.getstate() - - assert state_a != state_b + state_b2 = core._hypothesis_global_random.getstate() + + assert state_a == state_b + assert state_a2 != state_b2 + + +@pytest.mark.filterwarnings( + "ignore:It looks like `register_random` was passed an object that could be garbage collected" +) +def test_evil_prng_registration_nonsense(): + gc_on_pypy() + n_registered = len(entropy.RANDOMS_TO_MANAGE) + r1, r2, r3 = random.Random(1), random.Random(2), random.Random(3) + s2 = r2.getstate() + + # We're going to be totally evil here: register two randoms, then + # drop one and add another, and finally check that we reset only + # the states that we collected before we started + register_random(r1) + k = max(entropy.RANDOMS_TO_MANAGE) # get a handle to check if r1 still exists + register_random(r2) + assert len(entropy.RANDOMS_TO_MANAGE) == n_registered + 2 + + with deterministic_PRNG(0): + del r1 + gc_on_pypy() + assert k not in entropy.RANDOMS_TO_MANAGE, "r1 has been garbage-collected" + assert len(entropy.RANDOMS_TO_MANAGE) == n_registered + 1 + + r2.seed(4) + register_random(r3) + r3.seed(4) + s4 = r3.getstate() + + # Implicit check, no exception was raised in __exit__ + assert r2.getstate() == s2, "reset previously registered random state" + assert r3.getstate() == s4, "retained state when registered within the context" + + +@pytest.mark.skipif( + PYPY, reason="We can't guard against bad no-reference patterns in pypy." +) +def test_passing_unreferenced_instance_raises(): + with pytest.raises(ReferenceError): + register_random(random.Random(0)) + + +@pytest.mark.skipif( + PYPY, reason="We can't guard against bad no-reference patterns in pypy." +) +def test_passing_unreferenced_instance_within_function_scope_raises(): + def f(): + register_random(random.Random(0)) + + with pytest.raises(ReferenceError): + f() + + +@pytest.mark.skipif( + PYPY, reason="We can't guard against bad no-reference patterns in pypy." +) +def test_passing_referenced_instance_within_function_scope_warns(): + def f(): + r = random.Random(0) + register_random(r) + + with pytest.warns( + HypothesisWarning, + match="It looks like `register_random` was passed an object that could be" + " garbage collected", + ): + f() + + +@pytest.mark.filterwarnings( + "ignore:It looks like `register_random` was passed an object that could be garbage collected" +) +@pytest.mark.skipif( + PYPY, reason="We can't guard against bad no-reference patterns in pypy." +) +def test_register_random_within_nested_function_scope(): + n_registered = len(entropy.RANDOMS_TO_MANAGE) + + def f(): + r = random.Random() + register_random(r) + assert len(entropy.RANDOMS_TO_MANAGE) == n_registered + 1 + + f() + gc_on_pypy() + assert len(entropy.RANDOMS_TO_MANAGE) == n_registered diff --git a/hypothesis-python/tests/cover/test_randoms.py b/hypothesis-python/tests/cover/test_randoms.py index 6f906a212f..4b4db455e3 100644 --- a/hypothesis-python/tests/cover/test_randoms.py +++ b/hypothesis-python/tests/cover/test_randoms.py @@ -1,26 +1,22 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import inspect import math +import sys from copy import copy import pytest from hypothesis import assume, given, strategies as st -from hypothesis.errors import MultipleFailures +from hypothesis.internal.compat import ExceptionGroup from hypothesis.strategies._internal.random import ( RANDOM_METHODS, HypothesisRandom, @@ -30,7 +26,6 @@ ) from tests.common.debug import find_any -from tests.common.utils import capture_out def test_implements_all_random_methods(): @@ -60,7 +55,7 @@ def define_method_strategy(name, **kwargs): define_method_strategy("choice", seq=seq_param) define_method_strategy("choices", population=seq_param, k=st.integers(1, 100)) define_method_strategy("expovariate", lambd=beta_param) -define_method_strategy("_randbelow", n=st.integers(1, 2 ** 64)) +define_method_strategy("_randbelow", n=st.integers(1, 2**64)) define_method_strategy("random") define_method_strategy("getrandbits", n=st.integers(1, 128)) define_method_strategy("gauss", mu=st.floats(-1000, 1000), sigma=beta_param) @@ -78,7 +73,7 @@ def define_method_strategy(name, **kwargs): define_method_strategy("randbytes", n=st.integers(0, 100)) -INT64 = st.integers(-(2 ** 63), 2 ** 63 - 1) +INT64 = st.integers(-(2**63), 2**63 - 1) @st.composite @@ -97,7 +92,7 @@ def any_call_of_method(draw, method): b = draw(INT64) assume(a != b) a, b = sorted((a, b)) - if a == 0 and draw(st.booleans()): + if a == 0 and sys.version_info[:2] < (3, 10) and draw(st.booleans()): start = b stop = None else: @@ -111,8 +106,6 @@ def any_call_of_method(draw, method): a, b = sorted((a, b)) if draw(st.booleans()): draw(st.floats(a, b)) - else: - pass kwargs = {"low": a, "high": b, "mode": None} elif method == "uniform": a = normalize_zero(draw(st.floats(allow_infinity=False, allow_nan=False))) @@ -246,12 +239,11 @@ def test_outputs_random_calls(use_true_random): @given(st.randoms(use_true_random=use_true_random, note_method_calls=True)) def test(rnd): rnd.uniform(0.1, 0.5) - assert False + raise AssertionError - with capture_out() as out: - with pytest.raises(AssertionError): - test() - assert ".uniform(0.1, 0.5)" in out.getvalue() + with pytest.raises(AssertionError) as err: + test() + assert ".uniform(0.1, 0.5)" in "\n".join(err.value.__notes__) @pytest.mark.skipif( @@ -263,12 +255,11 @@ def test_converts_kwargs_correctly_in_output(use_true_random): @given(st.randoms(use_true_random=use_true_random, note_method_calls=True)) def test(rnd): rnd.choices([1, 2, 3, 4], k=2) - assert False + raise AssertionError - with capture_out() as out: - with pytest.raises(AssertionError): - test() - assert ".choices([1, 2, 3, 4], k=2)" in out.getvalue() + with pytest.raises(AssertionError) as err: + test() + assert ".choices([1, 2, 3, 4], k=2)" in "\n".join(err.value.__notes__) @given(st.randoms(use_true_random=False)) @@ -304,7 +295,7 @@ def test(rnd): assert x < 0.5 assert x > 0.5 - with pytest.raises(MultipleFailures): + with pytest.raises(ExceptionGroup): test() @@ -338,11 +329,11 @@ def test_randbytes_have_right_length(rnd, n): @given(any_random) def test_can_manage_very_long_ranges_with_step(rnd): - i = rnd.randrange(0, 2 ** 256, 3) + i = rnd.randrange(0, 2**256, 3) assert i % 3 == 0 - assert 0 <= i < 2 ** 256 - assert i in range(0, 2 ** 256, 3) + assert 0 <= i < 2**256 + assert i in range(0, 2**256, 3) @given(any_random, st.data()) diff --git a/hypothesis-python/tests/cover/test_recursive.py b/hypothesis-python/tests/cover/test_recursive.py index 976e466c47..b85bd324db 100644 --- a/hypothesis-python/tests/cover/test_recursive.py +++ b/hypothesis-python/tests/cover/test_recursive.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest diff --git a/hypothesis-python/tests/cover/test_reflection.py b/hypothesis-python/tests/cover/test_reflection.py index 3bd20b5406..b4639494d2 100644 --- a/hypothesis-python/tests/cover/test_reflection.py +++ b/hypothesis-python/tests/cover/test_reflection.py @@ -1,46 +1,40 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import sys from copy import deepcopy from datetime import time -from functools import partial -from inspect import FullArgSpec, getfullargspec +from functools import partial, wraps +from inspect import Parameter, Signature, signature from unittest.mock import MagicMock, Mock, NonCallableMagicMock, NonCallableMock import pytest +from pytest import raises -from hypothesis import strategies as st +from hypothesis import given, strategies as st from hypothesis.internal import reflection from hypothesis.internal.reflection import ( - arg_string, convert_keyword_arguments, convert_positional_arguments, define_function_signature, - fully_qualified_name, function_digest, get_pretty_function_description, + get_signature, + is_first_param_referenced_in_function, is_mock, proxies, + repr_call, required_args, source_exec_as_module, - unbind_method, ) -from tests.common.utils import raises - def do_conversion_test(f, args, kwargs): result = f(*args, **kwargs) @@ -66,16 +60,6 @@ def foo(a, b, c): do_conversion_test(foo, (1,), {"c": 2, "b": "foo"}) -def test_populates_defaults(): - def bar(x=[], y=1): - pass - - assert convert_keyword_arguments(bar, (), {}) == (([], 1), {}) - assert convert_keyword_arguments(bar, (), {"y": 42}) == (([], 42), {}) - do_conversion_test(bar, (), {}) - do_conversion_test(bar, (1,), {}) - - def test_leaves_unknown_kwargs_in_dict(): def bar(x, **kwargs): pass @@ -126,22 +110,19 @@ def test_errors_on_extra_kwargs(): def foo(a): pass - with raises(TypeError) as e: + with raises(TypeError, match="keyword"): convert_keyword_arguments(foo, (1,), {"b": 1}) - assert "keyword" in e.value.args[0] - with raises(TypeError) as e2: + with raises(TypeError, match="keyword"): convert_keyword_arguments(foo, (1,), {"b": 1, "c": 2}) - assert "keyword" in e2.value.args[0] def test_positional_errors_if_too_many_args(): def foo(a): pass - with raises(TypeError) as e: + with raises(TypeError, match="too many positional arguments"): convert_positional_arguments(foo, (1, 2), {}) - assert "2 given" in e.value.args[0] def test_positional_errors_if_too_few_args(): @@ -163,18 +144,16 @@ def test_positional_errors_if_given_bad_kwargs(): def foo(a): pass - with raises(TypeError) as e: + with raises(TypeError, match="missing a required argument: 'a'"): convert_positional_arguments(foo, (), {"b": 1}) - assert "unexpected keyword argument" in e.value.args[0] def test_positional_errors_if_given_duplicate_kwargs(): def foo(a): pass - with raises(TypeError) as e: + with raises(TypeError, match="multiple values"): convert_positional_arguments(foo, (2,), {"a": 1}) - assert "multiple values" in e.value.args[0] def test_names_of_functions_are_pretty(): @@ -254,9 +233,10 @@ def test_arg_string_is_in_order(): def foo(c, a, b, f, a1): pass - assert arg_string(foo, (1, 2, 3, 4, 5), {}) == "c=1, a=2, b=3, f=4, a1=5" + assert repr_call(foo, (1, 2, 3, 4, 5), {}) == "foo(c=1, a=2, b=3, f=4, a1=5)" assert ( - arg_string(foo, (1, 2), {"b": 3, "f": 4, "a1": 5}) == "c=1, a=2, b=3, f=4, a1=5" + repr_call(foo, (1, 2), {"b": 3, "f": 4, "a1": 5}) + == "foo(c=1, a=2, b=3, f=4, a1=5)" ) @@ -265,8 +245,8 @@ def foo(d, e, f, **kwargs): pass assert ( - arg_string(foo, (), {"a": 1, "b": 2, "c": 3, "d": 4, "e": 5, "f": 6}) - == "d=4, e=5, f=6, a=1, b=2, c=3" + repr_call(foo, (), {"a": 1, "b": 2, "c": 3, "d": 4, "e": 5, "f": 6}) + == "foo(d=4, e=5, f=6, a=1, b=2, c=3)" ) @@ -274,50 +254,21 @@ def test_varargs_come_without_equals(): def foo(a, *args): pass - assert arg_string(foo, (1, 2, 3, 4), {}) == "2, 3, 4, a=1" + assert repr_call(foo, (1, 2, 3, 4), {}) == "foo(2, 3, 4, a=1)" def test_can_mix_varargs_and_varkwargs(): def foo(*args, **kwargs): pass - assert arg_string(foo, (1, 2, 3), {"c": 7}) == "1, 2, 3, c=7" + assert repr_call(foo, (1, 2, 3), {"c": 7}) == "foo(1, 2, 3, c=7)" def test_arg_string_does_not_include_unprovided_defaults(): def foo(a, b, c=9, d=10): pass - assert arg_string(foo, (1,), {"b": 1, "d": 11}) == "a=1, b=1, d=11" - - -class A: - def f(self): - pass - - def g(self): - pass - - -class B(A): - pass - - -class C(A): - def f(self): - pass - - -def test_unbind_gives_parent_class_function(): - assert unbind_method(B().f) == unbind_method(A.f) - - -def test_unbind_distinguishes_different_functions(): - assert unbind_method(A.f) != unbind_method(A.g) - - -def test_unbind_distinguishes_overridden_functions(): - assert unbind_method(C().f) != unbind_method(A.f) + assert repr_call(foo, (1,), {"b": 1, "d": 11}) == "foo(a=1, b=1, d=11)" def universal_acceptor(*args, **kwargs): @@ -345,24 +296,18 @@ def has_kwargs(**kwargs): @pytest.mark.parametrize("f", [has_one_arg, has_two_args, has_varargs, has_kwargs]) -def test_copying_preserves_argspec(f): - af = getfullargspec(f) +def test_copying_preserves_signature(f): + af = get_signature(f) t = define_function_signature("foo", "docstring", af)(universal_acceptor) - at = getfullargspec(t) - assert af.args == at.args - assert af.varargs == at.varargs - assert af.varkw == at.varkw - assert len(af.defaults or ()) == len(at.defaults or ()) - assert af.kwonlyargs == at.kwonlyargs - assert af.kwonlydefaults == at.kwonlydefaults - assert af.annotations == at.annotations + at = get_signature(t) + assert af == at def test_name_does_not_clash_with_function_names(): def f(): pass - @define_function_signature("f", "A docstring for f", getfullargspec(f)) + @define_function_signature("f", "A docstring for f", signature(f)) def g(): pass @@ -371,29 +316,29 @@ def g(): def test_copying_sets_name(): f = define_function_signature( - "hello_world", "A docstring for hello_world", getfullargspec(has_two_args) + "hello_world", "A docstring for hello_world", signature(has_two_args) )(universal_acceptor) assert f.__name__ == "hello_world" def test_copying_sets_docstring(): f = define_function_signature( - "foo", "A docstring for foo", getfullargspec(has_two_args) + "foo", "A docstring for foo", signature(has_two_args) )(universal_acceptor) assert f.__doc__ == "A docstring for foo" def test_uses_defaults(): f = define_function_signature( - "foo", "A docstring for foo", getfullargspec(has_a_default) + "foo", "A docstring for foo", signature(has_a_default) )(universal_acceptor) assert f(3, 2) == ((3, 2, 1), {}) def test_uses_varargs(): - f = define_function_signature( - "foo", "A docstring for foo", getfullargspec(has_varargs) - )(universal_acceptor) + f = define_function_signature("foo", "A docstring for foo", signature(has_varargs))( + universal_acceptor + ) assert f(1, 2) == ((1, 2), {}) @@ -427,92 +372,39 @@ def accepts_everything(*args, **kwargs): define_function_signature( "hello", "A docstring for hello", - FullArgSpec( - args=("f",), - varargs=None, - varkw=None, - defaults=None, - kwonlyargs=[], - kwonlydefaults=None, - annotations={}, - ), + Signature(parameters=[Parameter("f", Parameter.POSITIONAL_OR_KEYWORD)]), )(accepts_everything)(1) define_function_signature( "hello", "A docstring for hello", - FullArgSpec( - args=(), - varargs="f", - varkw=None, - defaults=None, - kwonlyargs=[], - kwonlydefaults=None, - annotations={}, - ), + Signature(parameters=[Parameter("f", Parameter.VAR_POSITIONAL)]), )(accepts_everything)(1) define_function_signature( "hello", "A docstring for hello", - FullArgSpec( - args=(), - varargs=None, - varkw="f", - defaults=None, - kwonlyargs=[], - kwonlydefaults=None, - annotations={}, - ), + Signature(parameters=[Parameter("f", Parameter.VAR_KEYWORD)]), )(accepts_everything)() define_function_signature( "hello", "A docstring for hello", - FullArgSpec( - args=("f", "f_3"), - varargs="f_1", - varkw="f_2", - defaults=None, - kwonlyargs=[], - kwonlydefaults=None, - annotations={}, + Signature( + parameters=[ + Parameter("f", Parameter.POSITIONAL_OR_KEYWORD), + Parameter("f_3", Parameter.POSITIONAL_OR_KEYWORD), + Parameter("f_1", Parameter.VAR_POSITIONAL), + Parameter("f_2", Parameter.VAR_KEYWORD), + ] ), )(accepts_everything)(1, 2) -def test_define_function_signature_validates_arguments(): - with raises(ValueError): - define_function_signature( - "hello_world", - None, - FullArgSpec( - args=["a b"], - varargs=None, - varkw=None, - defaults=None, - kwonlyargs=[], - kwonlydefaults=None, - annotations={}, - ), - ) - - def test_define_function_signature_validates_function_name(): + define_function_signature("hello_world", None, Signature()) with raises(ValueError): - define_function_signature( - "hello world", - None, - FullArgSpec( - args=["a", "b"], - varargs=None, - varkw=None, - defaults=None, - kwonlyargs=[], - kwonlydefaults=None, - annotations={}, - ), - ) + define_function_signature("hello world", None, Signature()) class Container: @@ -520,29 +412,6 @@ def funcy(self): pass -def test_fully_qualified_name(): - assert ( - fully_qualified_name(test_copying_preserves_argspec) - == "tests.cover.test_reflection.test_copying_preserves_argspec" - ) - assert ( - fully_qualified_name(Container.funcy) - == "tests.cover.test_reflection.Container.funcy" - ) - assert ( - fully_qualified_name(fully_qualified_name) - == "hypothesis.internal.reflection.fully_qualified_name" - ) - - -def test_qualname_of_function_with_none_module_is_name(): - def f(): - pass - - f.__module__ = None - assert fully_qualified_name(f)[-1] == "f" - - def test_can_proxy_functions_with_mixed_args_and_varargs(): def foo(a, *args): return (a, args) @@ -569,7 +438,7 @@ def bar(**kwargs): "func,args,expected", [ (lambda: None, (), None), - (lambda a: a ** 2, (2,), 4), + (lambda a: a**2, (2,), 4), (lambda *a: a, [1, 2, 3], (1, 2, 3)), ], ) @@ -596,8 +465,8 @@ def test_can_handle_unicode_repr(): def foo(x): pass - assert arg_string(foo, [Snowman()], {}) == "x=☃" - assert arg_string(foo, [], {"x": Snowman()}) == "x=☃" + assert repr_call(foo, [Snowman()], {}) == "foo(x=☃)" + assert repr_call(foo, [], {"x": Snowman()}) == "foo(x=☃)" class NoRepr: @@ -608,23 +477,23 @@ def test_can_handle_repr_on_type(): def foo(x): pass - assert arg_string(foo, [Snowman], {}) == "x=Snowman" - assert arg_string(foo, [NoRepr], {}) == "x=NoRepr" + assert repr_call(foo, [Snowman], {}) == "foo(x=Snowman)" + assert repr_call(foo, [NoRepr], {}) == "foo(x=NoRepr)" def test_can_handle_repr_of_none(): def foo(x): pass - assert arg_string(foo, [None], {}) == "x=None" - assert arg_string(foo, [], {"x": None}) == "x=None" + assert repr_call(foo, [None], {}) == "foo(x=None)" + assert repr_call(foo, [], {"x": None}) == "foo(x=None)" def test_kwargs_appear_in_arg_string(): def varargs(*args, **kwargs): pass - assert "x=1" in arg_string(varargs, (), {"x": 1}) + assert "x=1" in repr_call(varargs, (), {"x": 1}) def test_is_mock_with_negative_cases(): @@ -706,3 +575,79 @@ def test_too_many_posargs_fails(): def test_overlapping_posarg_kwarg_fails(): with pytest.raises(TypeError): st.times(time.min, time.max, st.none(), timezones=st.just(None)).validate() + + +def test_inline_given_handles_self(): + # Regression test for https://github.com/HypothesisWorks/hypothesis/issues/961 + class Cls: + def method(self, **kwargs): + assert isinstance(self, Cls) + assert kwargs["k"] is sentinel + + sentinel = object() + given(k=st.just(sentinel))(Cls().method)() + + +def logged(f): + @wraps(f) + def wrapper(*a, **kw): + print("I was called") + return f(*a, **kw) + + return wrapper + + +class Bar: + @logged + def __init__(self, i: int): + pass + + +@given(st.builds(Bar)) +def test_issue_2495_regression(_): + """See https://github.com/HypothesisWorks/hypothesis/issues/2495""" + + +@pytest.mark.skipif( + sys.version_info[:2] >= (3, 11), + reason="handled upstream in https://github.com/python/cpython/pull/92065", +) +def test_error_on_keyword_parameter_name(): + def f(source): + pass + + f.__signature__ = Signature( + parameters=[Parameter("from", Parameter.KEYWORD_ONLY)], + return_annotation=Parameter.empty, + ) + + with pytest.raises(ValueError, match="SyntaxError because `from` is a keyword"): + get_signature(f) + + +def test_param_is_called_within_func(): + def f(any_name): + any_name() + + assert is_first_param_referenced_in_function(f) + + +def test_param_is_called_within_subfunc(): + def f(any_name): + def f2(): + any_name() + + assert is_first_param_referenced_in_function(f) + + +def test_param_is_not_called_within_func(): + def f(any_name): + pass + + assert not is_first_param_referenced_in_function(f) + + +def test_param_called_within_defaults_on_error(): + # Create a function object for which we cannot retrieve the source. + f = compile("lambda: ...", "_.py", "eval") + assert is_first_param_referenced_in_function(f) diff --git a/hypothesis-python/tests/cover/test_regex.py b/hypothesis-python/tests/cover/test_regex.py index e0594b62d5..4ce0d1144a 100644 --- a/hypothesis-python/tests/cover/test_regex.py +++ b/hypothesis-python/tests/cover/test_regex.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import re import sys @@ -30,6 +25,7 @@ UNICODE_SPACE_CHARS, UNICODE_WORD_CATEGORIES, base_regex_strategy, + regex_strategy, ) from tests.common.debug import assert_all_examples, assert_no_examples, find_any @@ -179,14 +175,14 @@ def test_any_doesnt_generate_newline(): @pytest.mark.parametrize("pattern", [re.compile("\\A.\\Z", re.DOTALL), "(?s)\\A.\\Z"]) def test_any_with_dotall_generate_newline(pattern): find_any( - st.from_regex(pattern), lambda s: s == "\n", settings(max_examples=10 ** 6) + st.from_regex(pattern), lambda s: s == "\n", settings(max_examples=10**6) ) @pytest.mark.parametrize("pattern", [re.compile(b"\\A.\\Z", re.DOTALL), b"(?s)\\A.\\Z"]) def test_any_with_dotall_generate_newline_binary(pattern): find_any( - st.from_regex(pattern), lambda s: s == b"\n", settings(max_examples=10 ** 6) + st.from_regex(pattern), lambda s: s == b"\n", settings(max_examples=10**6) ) @@ -425,20 +421,24 @@ def test_fullmatch_generates_example(pattern, matching_str): find_any( st.from_regex(pattern, fullmatch=True), lambda s: s == matching_str, - settings(max_examples=10 ** 6), + settings(max_examples=10**6), ) @pytest.mark.parametrize( "pattern,eqiv_pattern", [ + (r"", r"\A\Z"), + (b"", rb"\A\Z"), + (r"(?#comment)", r"\A\Z"), + (rb"(?#comment)", rb"\A\Z"), ("a", "\\Aa\\Z"), ("[Aa]", "\\A[Aa]\\Z"), ("[ab]*", "\\A[ab]*\\Z"), - (b"[Aa]", br"\A[Aa]\Z"), - (b"[ab]*", br"\A[ab]*\Z"), + (b"[Aa]", rb"\A[Aa]\Z"), + (b"[ab]*", rb"\A[ab]*\Z"), (re.compile("[ab]*", re.IGNORECASE), re.compile("\\A[ab]*\\Z", re.IGNORECASE)), - (re.compile(br"[ab]", re.IGNORECASE), re.compile(br"\A[ab]\Z", re.IGNORECASE)), + (re.compile(rb"[ab]", re.IGNORECASE), re.compile(rb"\A[ab]\Z", re.IGNORECASE)), ], ) def test_fullmatch_matches(pattern, eqiv_pattern): @@ -463,3 +463,12 @@ def test_sets_allow_multichar_output_in_ignorecase_mode(): st.from_regex(re.compile("[\u0130_]", re.IGNORECASE)), lambda s: len(s) > 1, ) + + +def test_internals_can_disable_newline_from_dollar_for_jsonschema(): + pattern = "^abc$" + find_any(st.from_regex(pattern), lambda s: s == "abc\n") + assert_all_examples( + regex_strategy(pattern, False, _temp_jsonschema_hack_no_end_newline=True), + lambda s: s == "abc", + ) diff --git a/hypothesis-python/tests/cover/test_regressions.py b/hypothesis-python/tests/cover/test_regressions.py index ced4e87596..8f26a4b843 100644 --- a/hypothesis-python/tests/cover/test_regressions.py +++ b/hypothesis-python/tests/cover/test_regressions.py @@ -1,24 +1,29 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER +import pickle import random +from datetime import timedelta from unittest.mock import Mock import pytest -from hypothesis import Verbosity, assume, given, seed, settings, strategies as st +from hypothesis import ( + Verbosity, + assume, + errors, + given, + seed, + settings, + strategies as st, +) def strat(): @@ -116,3 +121,27 @@ def test_prng_state_unpolluted_by_given_issue_1266(): assert second == third else: assert second != third + + +exc_instances = [ + errors.NoSuchExample("foobar", extra="baz"), + errors.DeadlineExceeded( + runtime=timedelta(seconds=1.5), deadline=timedelta(seconds=1.0) + ), +] + + +@pytest.mark.parametrize("exc", exc_instances, ids=repr) +def test_exceptions_are_picklable(exc): + # See https://github.com/HypothesisWorks/hypothesis/issues/3426 + pickle.loads(pickle.dumps(exc)) + + +def test_no_missed_custom_init_exceptions(): + untested_errors_with_custom_init = { + et + for et in vars(errors).values() + if isinstance(et, type) and issubclass(et, Exception) and "__init__" in vars(et) + } - {type(exc) for exc in exc_instances} + print(untested_errors_with_custom_init) + assert not untested_errors_with_custom_init diff --git a/hypothesis-python/tests/cover/test_reporting.py b/hypothesis-python/tests/cover/test_reporting.py index ab42f4c237..7faaf0c2f4 100644 --- a/hypothesis-python/tests/cover/test_reporting.py +++ b/hypothesis-python/tests/cover/test_reporting.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import os import sys @@ -26,18 +21,6 @@ from tests.common.utils import capture_out -def test_can_suppress_output(): - @given(integers()) - def test_int(x): - assert False - - with capture_out() as o: - with reporting.with_reporter(reporting.silent): - with pytest.raises(AssertionError): - test_int() - assert "Falsifying example" not in o.getvalue() - - def test_can_print_bytes(): with capture_out() as o: with reporting.with_reporter(reporting.default): @@ -48,13 +31,11 @@ def test_can_print_bytes(): def test_prints_output_by_default(): @given(integers()) def test_int(x): - assert False + raise AssertionError - with capture_out() as o: - with reporting.with_reporter(reporting.default): - with pytest.raises(AssertionError): - test_int() - assert "Falsifying example" in o.getvalue() + with pytest.raises(AssertionError) as err: + test_int() + assert "Falsifying example" in "\n".join(err.value.__notes__) def test_does_not_print_debug_in_verbose(): diff --git a/hypothesis-python/tests/cover/test_reproduce_failure.py b/hypothesis-python/tests/cover/test_reproduce_failure.py index 77365ca4cc..cb954ace4c 100644 --- a/hypothesis-python/tests/cover/test_reproduce_failure.py +++ b/hypothesis-python/tests/cover/test_reproduce_failure.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import base64 import re @@ -133,13 +128,13 @@ def test(i): failing_example[0] = i assert i not in failing_example - with capture_out() as o: - with pytest.raises(AssertionError): - test() - assert "@reproduce_failure" in o.getvalue() + with pytest.raises(AssertionError) as err: + test() + notes = "\n".join(err.value.__notes__) + assert "@reproduce_failure" in notes exp = re.compile(r"reproduce_failure\(([^)]+)\)", re.MULTILINE) - extract = exp.search(o.getvalue()) + extract = exp.search(notes) reproduction = eval(extract.group(0)) test = reproduction(test) @@ -151,7 +146,7 @@ def test_does_not_print_reproduction_for_simple_examples_by_default(): @settings(print_blob=False) @given(st.integers()) def test(i): - assert False + raise AssertionError with capture_out() as o: with pytest.raises(AssertionError): @@ -164,7 +159,7 @@ def test_does_not_print_reproduction_for_simple_data_examples_by_default(): @given(st.data()) def test(data): data.draw(st.integers()) - assert False + raise AssertionError with capture_out() as o: with pytest.raises(AssertionError): diff --git a/hypothesis-python/tests/cover/test_reusable_values.py b/hypothesis-python/tests/cover/test_reusable_values.py deleted file mode 100644 index ff343dae20..0000000000 --- a/hypothesis-python/tests/cover/test_reusable_values.py +++ /dev/null @@ -1,100 +0,0 @@ -# This file is part of Hypothesis, which may be found at -# https://github.com/HypothesisWorks/hypothesis/ -# -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. -# -# This Source Code Form is subject to the terms of the Mozilla Public License, -# v. 2.0. If a copy of the MPL was not distributed with this file, You can -# obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER - -import pytest - -from hypothesis import example, given, reject, strategies as st -from hypothesis.errors import HypothesisDeprecationWarning, InvalidArgument - -base_reusable_strategies = ( - st.text(), - st.binary(), - st.dates(), - st.times(), - st.timedeltas(), - st.booleans(), - st.complex_numbers(), - st.floats(), - st.floats(-1.0, 1.0), - st.integers(), - st.integers(1, 10), - st.integers(1), -) - - -@st.deferred -def reusable(): - return st.one_of( - st.sampled_from(base_reusable_strategies), - st.builds( - st.floats, - min_value=st.none() | st.floats(), - max_value=st.none() | st.floats(), - allow_infinity=st.booleans(), - allow_nan=st.booleans(), - ), - st.builds(st.just, st.builds(list)), - st.builds(st.sampled_from, st.lists(st.builds(list), min_size=1)), - st.lists(reusable).map(st.one_of), - st.lists(reusable).map(lambda ls: st.tuples(*ls)), - ) - - -assert not reusable.is_empty - - -@example(st.integers(min_value=1)) -@given(reusable) -def test_reusable_strategies_are_all_reusable(s): - try: - s.validate() - except (InvalidArgument, HypothesisDeprecationWarning): - reject() - - assert s.has_reusable_values - - -for s in base_reusable_strategies: - test_reusable_strategies_are_all_reusable = example(s)( - test_reusable_strategies_are_all_reusable - ) - test_reusable_strategies_are_all_reusable = example(st.tuples(s))( - test_reusable_strategies_are_all_reusable - ) - - -def test_composing_breaks_reusability(): - s = st.integers() - assert s.has_reusable_values - assert not s.filter(lambda x: True).has_reusable_values - assert not s.map(lambda x: x).has_reusable_values - assert not s.flatmap(lambda x: st.just(x)).has_reusable_values - - -@pytest.mark.parametrize( - "strat", - [ - st.lists(st.booleans()), - st.sets(st.booleans()), - st.dictionaries(st.booleans(), st.booleans()), - ], -) -def test_mutable_collections_do_not_have_reusable_values(strat): - assert not strat.has_reusable_values - - -def test_recursion_does_not_break_reusability(): - x = st.deferred(lambda: st.none() | st.tuples(x)) - assert x.has_reusable_values diff --git a/hypothesis-python/tests/cover/test_runner_strategy.py b/hypothesis-python/tests/cover/test_runner_strategy.py index 8948a6536b..6fe5c2baa9 100644 --- a/hypothesis-python/tests/cover/test_runner_strategy.py +++ b/hypothesis-python/tests/cover/test_runner_strategy.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from unittest import TestCase diff --git a/hypothesis-python/tests/cover/test_sampled_from.py b/hypothesis-python/tests/cover/test_sampled_from.py index babcf8de74..0d8ba38751 100644 --- a/hypothesis-python/tests/cover/test_sampled_from.py +++ b/hypothesis-python/tests/cover/test_sampled_from.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import collections import enum @@ -61,7 +56,7 @@ def test_can_sample_enums(member): @fails_with(FailedHealthCheck) @given(sampled_from(range(10)).filter(lambda x: x < 0)) def test_unsat_filtered_sampling(x): - assert False + raise AssertionError @fails_with(Unsatisfiable) @@ -69,7 +64,7 @@ def test_unsat_filtered_sampling(x): def test_unsat_filtered_sampling_in_rejection_stage(x): # Rejecting all possible indices before we calculate the allowed indices # takes an early exit path, so we need this test to cover that branch. - assert False + raise AssertionError def test_easy_filtered_sampling(): @@ -132,8 +127,8 @@ def stupid_sampled_sets(draw): @given(stupid_sampled_sets()) def test_efficient_sets_of_samples_with_chained_transformations_slow_path(x): - # This exercises the standard .do_filtered_draw method, rather than the - # special logic in UniqueSampledListStrategy, to the same if slower effect. + # This deliberately exercises the standard filtering logic without going + # through the special-case handling of UniqueSampledListStrategy. assert x == {x * 2 for x in range(20) if x % 3} @@ -155,7 +150,7 @@ def test_transformed_just_strategy(): assert s.do_draw(data) == 2 sf = s.filter(lambda x: False) assert isinstance(sf, JustStrategy) - assert sf.do_filtered_draw(data, sf) == filter_not_satisfied + assert sf.do_filtered_draw(data) == filter_not_satisfied with pytest.raises(StopTest): sf.do_draw(data) @@ -186,3 +181,12 @@ def test_mutability_2(data): assert data.draw(s) != 2 x.append(2) assert data.draw(s) != 2 + + +class AnnotationsInsteadOfElements(enum.Enum): + a: "a" + + +def test_suggests_elements_instead_of_annotations(): + with pytest.raises(InvalidArgument, match="Cannot sample.*annotations.*dataclass"): + st.sampled_from(AnnotationsInsteadOfElements).example() diff --git a/hypothesis-python/tests/cover/test_searchstrategy.py b/hypothesis-python/tests/cover/test_searchstrategy.py index 4e0e77157e..ef93f0c47a 100644 --- a/hypothesis-python/tests/cover/test_searchstrategy.py +++ b/hypothesis-python/tests/cover/test_searchstrategy.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import functools from collections import namedtuple @@ -19,7 +14,8 @@ import pytest from hypothesis.errors import InvalidArgument -from hypothesis.strategies import booleans, integers, just, tuples +from hypothesis.internal.conjecture.data import ConjectureData +from hypothesis.strategies import booleans, integers, just, none, tuples from tests.common.debug import assert_no_examples @@ -48,6 +44,18 @@ def __repr__(self): assert repr(just(WeirdRepr())) == f"just({WeirdRepr()!r})" +def test_just_strategy_does_not_draw(): + data = ConjectureData.for_buffer(b"") + s = just("hello") + assert s.do_draw(data) == "hello" + + +def test_none_strategy_does_not_draw(): + data = ConjectureData.for_buffer(b"") + s = none() + assert s.do_draw(data) is None + + def test_can_map(): s = integers().map(pack=lambda t: "foo") assert s.example() == "foo" diff --git a/hypothesis-python/tests/cover/test_seed_printing.py b/hypothesis-python/tests/cover/test_seed_printing.py index 9a79ab66c9..8b65d79b7e 100644 --- a/hypothesis-python/tests/cover/test_seed_printing.py +++ b/hypothesis-python/tests/cover/test_seed_printing.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import time @@ -59,7 +54,7 @@ def test(i): assert seed is not None if fail_healthcheck and verbosity != Verbosity.quiet: assert f"@seed({seed})" in output - contains_pytest_instruction = (f"--hypothesis-seed={seed}") in output + contains_pytest_instruction = f"--hypothesis-seed={seed}" in output assert contains_pytest_instruction == in_pytest else: assert "@seed" not in output diff --git a/hypothesis-python/tests/cover/test_settings.py b/hypothesis-python/tests/cover/test_settings.py index 7da8895c3e..e3ca7376ce 100644 --- a/hypothesis-python/tests/cover/test_settings.py +++ b/hypothesis-python/tests/cover/test_settings.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import datetime import subprocess @@ -423,7 +418,7 @@ def test_assigning_to_settings_attribute_on_state_machine_raises_error(): class StateMachine(RuleBasedStateMachine): @rule(x=st.none()) - def a_rule(x): + def a_rule(self, x): assert x is None StateMachine.settings = settings() @@ -469,12 +464,12 @@ def __repr__(self): def test_show_changed(): - s = settings(max_examples=999, database=None) + s = settings(max_examples=999, database=None, phases=tuple(Phase)[:-1]) assert s.show_changed() == "database=None, max_examples=999" def test_note_deprecation_checks_date(): - with pytest.warns(None) as rec: + with pytest.warns(HypothesisDeprecationWarning) as rec: note_deprecation("This is bad", since="RELEASEDAY", has_codemod=False) assert len(rec) == 1 with pytest.raises(AssertionError): diff --git a/hypothesis-python/tests/cover/test_setup_teardown.py b/hypothesis-python/tests/cover/test_setup_teardown.py index 07515b15aa..c533e1606a 100644 --- a/hypothesis-python/tests/cover/test_setup_teardown.py +++ b/hypothesis-python/tests/cover/test_setup_teardown.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest @@ -37,7 +32,7 @@ def give_me_an_int(self, x): pass @given(text()) - def give_me_a_string(myself, x): + def give_me_a_string(self, x): pass @given(integers()) diff --git a/hypothesis-python/tests/cover/test_shrink_budgeting.py b/hypothesis-python/tests/cover/test_shrink_budgeting.py index 91d29b220d..b67e2e2d91 100644 --- a/hypothesis-python/tests/cover/test_shrink_budgeting.py +++ b/hypothesis-python/tests/cover/test_shrink_budgeting.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import math import sys diff --git a/hypothesis-python/tests/cover/test_simple_characters.py b/hypothesis-python/tests/cover/test_simple_characters.py index e336202347..d90a8b1d60 100644 --- a/hypothesis-python/tests/cover/test_simple_characters.py +++ b/hypothesis-python/tests/cover/test_simple_characters.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import unicodedata diff --git a/hypothesis-python/tests/cover/test_simple_collections.py b/hypothesis-python/tests/cover/test_simple_collections.py index 24178ba686..576f9f9ba2 100644 --- a/hypothesis-python/tests/cover/test_simple_collections.py +++ b/hypothesis-python/tests/cover/test_simple_collections.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from collections import OrderedDict from random import Random diff --git a/hypothesis-python/tests/cover/test_simple_strings.py b/hypothesis-python/tests/cover/test_simple_strings.py index 0113ce87f8..805484a919 100644 --- a/hypothesis-python/tests/cover/test_simple_strings.py +++ b/hypothesis-python/tests/cover/test_simple_strings.py @@ -1,23 +1,17 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import given from hypothesis.strategies import binary, characters, text, tuples from tests.common.debug import minimal -from tests.common.utils import fails_with def test_can_minimize_up_to_zero(): @@ -60,12 +54,6 @@ def test_finds_single_element_strings(): assert minimal(text(), bool) == "0" -@fails_with(AssertionError) -@given(binary()) -def test_binary_generates_large_examples(x): - assert len(x) <= 20 - - @given(binary(max_size=5)) def test_binary_respects_max_size(x): assert len(x) <= 5 @@ -93,7 +81,7 @@ def test_respects_alphabet_if_string(xs): @given(text()) def test_can_encode_as_utf8(s): - s.encode("utf-8") + s.encode() @given(text(characters(blacklist_characters="\n"))) @@ -118,6 +106,6 @@ def test_fixed_size_bytes_just_draw_bytes(): assert x.draw(binary(min_size=3, max_size=3)) == b"foo" -@given(text(max_size=10 ** 6)) +@given(text(max_size=10**6)) def test_can_set_max_size_large(s): pass diff --git a/hypothesis-python/tests/cover/test_slices.py b/hypothesis-python/tests/cover/test_slices.py index 8241a9493a..a3bf3da018 100644 --- a/hypothesis-python/tests/cover/test_slices.py +++ b/hypothesis-python/tests/cover/test_slices.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest @@ -33,8 +28,8 @@ def test_stop_stays_within_bounds(size): @use_several_sizes def test_start_stay_within_bounds(size): assert_all_examples( - st.slices(size), - lambda x: x.start is None or (x.start >= -size and x.start <= size), + st.slices(size).filter(lambda x: x.start is not None), + lambda x: range(size)[x.start] or True, # no IndexError raised ) @@ -64,30 +59,32 @@ def test_slices_will_shrink(size): sliced = minimal(st.slices(size)) assert sliced.start == 0 or sliced.start is None assert sliced.stop == 0 or sliced.stop is None - assert sliced.step == 1 + assert sliced.step is None @given(st.integers(1, 1000)) @settings(deadline=None) def test_step_will_be_negative(size): - find_any(st.slices(size), lambda x: x.step < 0, settings(max_examples=10 ** 6)) + find_any( + st.slices(size), lambda x: (x.step or 1) < 0, settings(max_examples=10**6) + ) @given(st.integers(1, 1000)) @settings(deadline=None) def test_step_will_be_positive(size): - find_any(st.slices(size), lambda x: x.step > 0) + find_any(st.slices(size), lambda x: (x.step or 1) > 0) @pytest.mark.parametrize("size", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) def test_stop_will_equal_size(size): - find_any(st.slices(size), lambda x: x.stop == size, settings(max_examples=10 ** 6)) + find_any(st.slices(size), lambda x: x.stop == size, settings(max_examples=10**6)) @pytest.mark.parametrize("size", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) def test_start_will_equal_size(size): find_any( - st.slices(size), lambda x: x.start == size - 1, settings(max_examples=10 ** 6) + st.slices(size), lambda x: x.start == size - 1, settings(max_examples=10**6) ) diff --git a/hypothesis-python/tests/cover/test_slippage.py b/hypothesis-python/tests/cover/test_slippage.py index a30e20b4aa..4561000fcf 100644 --- a/hypothesis-python/tests/cover/test_slippage.py +++ b/hypothesis-python/tests/cover/test_slippage.py @@ -1,23 +1,19 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest -from hypothesis import Phase, assume, given, settings, strategies as st +from hypothesis import Phase, assume, given, settings, strategies as st, target from hypothesis.database import InMemoryExampleDatabase -from hypothesis.errors import Flaky, MultipleFailures +from hypothesis.errors import Flaky +from hypothesis.internal.compat import ExceptionGroup from hypothesis.internal.conjecture.engine import MIN_TEST_CALLS from tests.common.utils import ( @@ -27,6 +23,16 @@ ) +def capture_reports(test): + with capture_out() as o, pytest.raises(ExceptionGroup) as err: + test() + + return o.getvalue() + "\n\n".join( + f"{e!r}\n" + "\n".join(getattr(e, "__notes__", [])) + for e in (err.value,) + err.value.exceptions + ) + + def test_raises_multiple_failures_with_varying_type(): target = [None] @@ -43,12 +49,20 @@ def test(i): exc_class = TypeError if target[0] == i else ValueError raise exc_class() - with capture_out() as o: - with pytest.raises(MultipleFailures): - test() + output = capture_reports(test) + assert "TypeError" in output + assert "ValueError" in output - assert "TypeError" in o.getvalue() - assert "ValueError" in o.getvalue() + +def test_shows_target_scores_with_multiple_failures(): + @settings(database=None, max_examples=100) + @given(st.integers()) + def test(i): + target(i) + assert i > 0 + assert i < 0 + + assert "Highest target score:" in capture_reports(test) def test_raises_multiple_failures_when_position_varies(): @@ -66,11 +80,9 @@ def test(i): else: raise ValueError("loc 2") - with capture_out() as o: - with pytest.raises(MultipleFailures): - test() - assert "loc 1" in o.getvalue() - assert "loc 2" in o.getvalue() + output = capture_reports(test) + assert "loc 1" in output + assert "loc 2" in output def test_replays_both_failing_values(): @@ -86,10 +98,10 @@ def test(i): exc_class = TypeError if target[0] == i else ValueError raise exc_class() - with pytest.raises(MultipleFailures): + with pytest.raises(ExceptionGroup): test() - with pytest.raises(MultipleFailures): + with pytest.raises(ExceptionGroup): test() @@ -116,7 +128,7 @@ def test(i): if i == target[1]: raise ValueError() - with pytest.raises(MultipleFailures): + with pytest.raises(ExceptionGroup): test() bug_fixed = True @@ -147,7 +159,7 @@ def test(i): if i == target[1]: raise ValueError() - with pytest.raises(MultipleFailures): + with pytest.raises(ExceptionGroup): test() bug_fixed = True @@ -173,7 +185,7 @@ def test_shrinks_both_failures(): def test(i): if i >= 10000: first_has_failed[0] = True - assert False + raise AssertionError assert i < 10000 if first_has_failed[0]: if second_target[0] is None: @@ -185,11 +197,7 @@ def test(i): else: duds.add(i) - with capture_out() as o: - with pytest.raises(MultipleFailures): - test() - - output = o.getvalue() + output = capture_reports(test) assert_output_contains_failure(output, test, i=10000) assert_output_contains_failure(output, test, i=second_target[0]) @@ -217,13 +225,19 @@ def test(i): flaky_failed_once[0] = True raise ValueError() - with pytest.raises(Flaky): + try: test() + raise AssertionError("Expected test() to raise an error") + except ExceptionGroup as err: + assert any(isinstance(e, Flaky) for e in err.exceptions) flaky_fixed = True - with pytest.raises(MultipleFailures): + try: test() + raise AssertionError("Expected test() to raise an error") + except ExceptionGroup as err: + assert not any(isinstance(e, Flaky) for e in err.exceptions) @pytest.mark.parametrize("allow_multi", [True, False]) @@ -242,7 +256,7 @@ def test(i): seen.add(ValueError) raise ValueError - with pytest.raises(MultipleFailures if allow_multi else TypeError): + with pytest.raises(ExceptionGroup if allow_multi else TypeError): test() assert seen == {TypeError, ValueError} @@ -251,7 +265,7 @@ def test_finds_multiple_failures_in_generation(): special = [] seen = set() - @settings(phases=[Phase.generate], max_examples=100) + @settings(phases=[Phase.generate, Phase.shrink], max_examples=100) @given(st.integers(min_value=0)) def test(x): """Constructs a test so the 10th largeish example we've seen is a @@ -268,7 +282,7 @@ def test(x): assert x in seen or (x <= special[0]) assert x not in special - with pytest.raises(MultipleFailures): + with pytest.raises(ExceptionGroup): test() @@ -279,7 +293,7 @@ def test_stops_immediately_if_not_report_multiple_bugs(): @given(st.integers()) def test(x): seen.add(x) - assert False + raise AssertionError with pytest.raises(AssertionError): test() diff --git a/hypothesis-python/tests/cover/test_stateful.py b/hypothesis-python/tests/cover/test_stateful.py index 9a6c9872e8..ce6f60b55c 100644 --- a/hypothesis-python/tests/cover/test_stateful.py +++ b/hypothesis-python/tests/cover/test_stateful.py @@ -1,28 +1,27 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import base64 -from collections import defaultdict, namedtuple +import sys +from collections import defaultdict import pytest from _pytest.outcomes import Failed, Skipped +from pytest import raises from hypothesis import __version__, reproduce_failure, seed, settings as Settings from hypothesis.control import current_build_context from hypothesis.database import ExampleDatabase from hypothesis.errors import DidNotReproduce, Flaky, InvalidArgument, InvalidDefinition +from hypothesis.internal.compat import PYPY +from hypothesis.internal.entropy import deterministic_PRNG from hypothesis.stateful import ( Bundle, RuleBasedStateMachine, @@ -34,66 +33,13 @@ rule, run_state_machine_as_test, ) -from hypothesis.strategies import binary, booleans, data, integers, just, lists +from hypothesis.strategies import binary, data, integers, just, lists -from tests.common.utils import capture_out, raises +from tests.common.utils import capture_out, validate_deprecation +from tests.nocover.test_stateful import DepthMachine NO_BLOB_SETTINGS = Settings(print_blob=False) -Leaf = namedtuple("Leaf", ("label",)) -Split = namedtuple("Split", ("left", "right")) - - -class BalancedTrees(RuleBasedStateMachine): - trees = Bundle("BinaryTree") - - @rule(target=trees, x=booleans()) - def leaf(self, x): - return Leaf(x) - - @rule(target=trees, left=trees, right=trees) - def split(self, left, right): - return Split(left, right) - - @rule(tree=trees) - def test_is_balanced(self, tree): - if isinstance(tree, Leaf): - return - else: - assert abs(self.size(tree.left) - self.size(tree.right)) <= 1 - self.test_is_balanced(tree.left) - self.test_is_balanced(tree.right) - - def size(self, tree): - if isinstance(tree, Leaf): - return 1 - else: - return 1 + self.size(tree.left) + self.size(tree.right) - - -class DepthCharge: - def __init__(self, value): - if value is None: - self.depth = 0 - else: - self.depth = value.depth + 1 - - -class DepthMachine(RuleBasedStateMachine): - charges = Bundle("charges") - - @rule(targets=(charges,), child=charges) - def charge(self, child): - return DepthCharge(child) - - @rule(targets=(charges,)) - def none_charge(self): - return DepthCharge(None) - - @rule(check=charges) - def is_not_too_deep(self, check): - assert check.depth < 3 - class MultipleRulesSameFuncMachine(RuleBasedStateMachine): def myfunc(self, data): @@ -125,125 +71,12 @@ def div_by_precondition_before(self, num): self.num = num / self.num -class RoseTreeStateMachine(RuleBasedStateMachine): - nodes = Bundle("nodes") - - @rule(target=nodes, source=lists(nodes)) - def bunch(self, source): - return source - - @rule(source=nodes) - def shallow(self, source): - def d(ls): - if not ls: - return 0 - else: - return 1 + max(map(d, ls)) - - assert d(source) <= 5 - - -class NotTheLastMachine(RuleBasedStateMachine): - stuff = Bundle("stuff") - - def __init__(self): - super().__init__() - self.last = None - self.bye_called = False - - @rule(target=stuff) - def hi(self): - result = object() - self.last = result - return result - - @precondition(lambda self: not self.bye_called) - @rule(v=stuff) - def bye(self, v): - assert v == self.last - self.bye_called = True - - -class PopulateMultipleTargets(RuleBasedStateMachine): - b1 = Bundle("b1") - b2 = Bundle("b2") - - @rule(targets=(b1, b2)) - def populate(self): - return 1 - - @rule(x=b1, y=b2) - def fail(self, x, y): - assert False - - -class CanSwarm(RuleBasedStateMachine): - """This test will essentially never pass if you choose rules uniformly at - random, because every time the snake rule fires we return to the beginning, - so we will tend to undo progress well before we make enough progress for - the test to fail. - - This tests our swarm testing functionality in stateful testing by ensuring - that we can sometimes generate long runs of steps which exclude a - particular rule. - """ - - def __init__(self): - super().__init__() - self.seen = set() - - # The reason this rule takes a parameter is that it ensures that we do not - # achieve "swarming" by by just restricting the alphabet for single byte - # decisions, which is a thing the underlying conjecture engine will - # happily do on its own without knowledge of the rule structure. - @rule(move=integers(0, 255)) - def ladder(self, move): - self.seen.add(move) - assert len(self.seen) <= 15 - - @rule() - def snake(self): - self.seen.clear() - - -bad_machines = ( - BalancedTrees, - DepthMachine, - RoseTreeStateMachine, - NotTheLastMachine, - PopulateMultipleTargets, - CanSwarm, -) - -for m in bad_machines: - m.TestCase.settings = Settings(m.TestCase.settings, max_examples=1000) - - -cheap_bad_machines = list(bad_machines) -cheap_bad_machines.remove(BalancedTrees) - - -with_cheap_bad_machines = pytest.mark.parametrize( - "machine", cheap_bad_machines, ids=[t.__name__ for t in cheap_bad_machines] -) +TestPrecondition = PreconditionMachine.TestCase +TestPrecondition.settings = Settings(TestPrecondition.settings, max_examples=10) -@pytest.mark.parametrize( - "machine", bad_machines, ids=[t.__name__ for t in bad_machines] -) -def test_bad_machines_fail(machine): - test_class = machine.TestCase - try: - with capture_out() as o: - with raises(AssertionError): - test_class().runTest() - except Exception: - print(o.getvalue()) - raise - v = o.getvalue() - print(v) - steps = [l for l in v.splitlines() if "Step " in l or "state." in l] - assert 1 <= len(steps) <= 50 +def test_picks_up_settings_at_first_use_of_testcase(): + assert TestPrecondition.settings.max_examples == 10 def test_multiple_rules_same_func(): @@ -294,7 +127,7 @@ def action(self, d): FlakyRatchettingMachine.ratchet += 1 n = FlakyRatchettingMachine.ratchet d.draw(lists(integers(), min_size=n, max_size=n)) - assert False + raise AssertionError class MachineWithConsumingRule(RuleBasedStateMachine): @@ -336,7 +169,7 @@ def test_multiple(): none = multiple() some = multiple(1, 2.01, "3", b"4", 5) assert len(none.values) == 0 and len(some.values) == 5 - assert all(value in some.values for value in (1, 2.01, "3", b"4", 5)) + assert set(some.values) == {1, 2.01, "3", b"4", 5} class MachineUsingMultiple(RuleBasedStateMachine): @@ -373,18 +206,15 @@ def populate_bundle(self): @rule() def fail_fast(self): - assert False + raise AssertionError - with capture_out() as o: - # The state machine must raise an exception for the - # falsifying example to be printed. - with raises(AssertionError): - run_state_machine_as_test(ProducesMultiple) + with raises(AssertionError) as err: + run_state_machine_as_test(ProducesMultiple) # This is tightly coupled to the output format of the step printing. # The first line is "Falsifying Example:..." the second is creating # the state machine, the third is calling the "initialize" method. - assignment_line = o.getvalue().split("\n")[2] + assignment_line = err.value.__notes__[2] # 'populate_bundle()' returns 2 values, so should be # expanded to 2 variables. assert assignment_line == "v1, v2 = state.populate_bundle()" @@ -397,6 +227,32 @@ def fail_fast(self): state.fail_fast() +def test_multiple_variables_printed_single_element(): + # https://github.com/HypothesisWorks/hypothesis/issues/3236 + class ProducesMultiple(RuleBasedStateMachine): + b = Bundle("b") + + @initialize(target=b) + def populate_bundle(self): + return multiple(1) + + @rule(b=b) + def fail_fast(self, b): + assert b != 1 + + with raises(AssertionError) as err: + run_state_machine_as_test(ProducesMultiple) + + assignment_line = err.value.__notes__[2] + assert assignment_line == "(v1,) = state.populate_bundle()" + + state = ProducesMultiple() + (v1,) = state.populate_bundle() + state.fail_fast((v1,)) # passes if tuple not unpacked + with raises(AssertionError): + state.fail_fast(v1) + + def test_no_variables_printed(): class ProducesNoVariables(RuleBasedStateMachine): b = Bundle("b") @@ -407,18 +263,15 @@ def populate_bundle(self): @rule() def fail_fast(self): - assert False + raise AssertionError - with capture_out() as o: - # The state machine must raise an exception for the - # falsifying example to be printed. - with raises(AssertionError): - run_state_machine_as_test(ProducesNoVariables) + with raises(AssertionError) as err: + run_state_machine_as_test(ProducesNoVariables) # This is tightly coupled to the output format of the step printing. # The first line is "Falsifying Example:..." the second is creating # the state machine, the third is calling the "initialize" method. - assignment_line = o.getvalue().split("\n")[2] + assignment_line = err.value.__notes__[2] # 'populate_bundle()' returns 0 values, so there should be no # variable assignment. assert assignment_line == "state.populate_bundle()" @@ -452,53 +305,6 @@ def bye(self, hi): NonTerminalMachine.TestCase().runTest() -class DynamicMachine(RuleBasedStateMachine): - @rule(value=Bundle("hi")) - def test_stuff(x): - pass - - -DynamicMachine.define_rule(targets=(), function=lambda self: 1, arguments={}) - - -class IntAdder(RuleBasedStateMachine): - pass - - -IntAdder.define_rule( - targets=("ints",), function=lambda self, x: x, arguments={"x": integers()} -) - -IntAdder.define_rule( - targets=("ints",), - function=lambda self, x, y: x, - arguments={"x": integers(), "y": Bundle("ints")}, -) - - -TestDynamicMachine = DynamicMachine.TestCase -TestIntAdder = IntAdder.TestCase -TestPrecondition = PreconditionMachine.TestCase - - -for test_case in (TestDynamicMachine, TestIntAdder, TestPrecondition): - test_case.settings = Settings(test_case.settings, max_examples=10) - - -def test_picks_up_settings_at_first_use_of_testcase(): - assert TestDynamicMachine.settings.max_examples == 10 - - -def test_new_rules_are_picked_up_before_and_after_rules_call(): - class Foo(RuleBasedStateMachine): - pass - - Foo.define_rule(targets=(), function=lambda self: 1, arguments={}) - assert len(Foo.rules()) == 1 - Foo.define_rule(targets=(), function=lambda self: 2, arguments={}) - assert len(Foo.rules()) == 2 - - def test_minimizes_errors_in_teardown(): counter = [0] @@ -596,8 +402,10 @@ def test_saves_failing_example_in_database(): def test_can_run_with_no_db(): - with raises(AssertionError): - run_state_machine_as_test(DepthMachine, settings=Settings(database=None)) + with deterministic_PRNG(), raises(AssertionError): + run_state_machine_as_test( + DepthMachine, settings=Settings(database=None, max_examples=10_000) + ) def test_stateful_double_rule_is_forbidden(recwarn): @@ -693,6 +501,32 @@ def do_stuff(self): run_state_machine_as_test(Invariant) +@pytest.mark.parametrize( + "decorators", + [ + (invariant(), rule()), + (rule(), invariant()), + (invariant(), initialize()), + (initialize(), invariant()), + (invariant(), precondition(lambda self: True), rule()), + (rule(), precondition(lambda self: True), invariant()), + (precondition(lambda self: True), invariant(), rule()), + (precondition(lambda self: True), rule(), invariant()), + ], + ids=lambda x: "-".join(f.__qualname__.split(".")[0] for f in x), +) +def test_invariant_and_rule_are_incompatible(decorators): + """It's an error to apply @invariant and @rule to the same method.""" + + def method(self): + pass + + for d in decorators[:-1]: + method = d(method) + with pytest.raises(InvalidDefinition): + decorators[-1](method) + + def test_invalid_rule_argument(): """Rule kwargs that are not a Strategy are expected to raise an InvalidArgument error.""" with pytest.raises(InvalidArgument): @@ -759,7 +593,7 @@ def test_foo(self): run_state_machine_as_test(BadPrecondition) -def test_invariant_checks_initial_state(): +def test_invariant_checks_initial_state_if_no_initialize_rules(): """Invariants are checked before any rules run.""" class BadPrecondition(RuleBasedStateMachine): @@ -780,6 +614,97 @@ def test_foo(self): run_state_machine_as_test(BadPrecondition) +def test_invariant_failling_present_in_falsifying_example(): + @Settings(print_blob=False) + class BadInvariant(RuleBasedStateMachine): + @initialize() + def initialize_1(self): + pass + + @invariant() + def invariant_1(self): + raise ValueError() + + @rule() + def rule_1(self): + pass + + with pytest.raises(ValueError) as err: + run_state_machine_as_test(BadInvariant) + + result = "\n".join(err.value.__notes__) + assert ( + result + == """ +Falsifying example: +state = BadInvariant() +state.initialize_1() +state.invariant_1() +state.teardown() +""".strip() + ) + + +def test_invariant_present_in_falsifying_example(): + @Settings(print_blob=False) + class BadRuleWithGoodInvariants(RuleBasedStateMachine): + def __init__(self): + super().__init__() + self.num = 0 + + @initialize() + def initialize_1(self): + pass + + @invariant(check_during_init=True) + def invariant_1(self): + pass + + @invariant(check_during_init=False) + def invariant_2(self): + pass + + @precondition(lambda self: self.num > 0) + @invariant() + def invariant_3(self): + pass + + @rule() + def rule_1(self): + self.num += 1 + if self.num == 2: + raise ValueError() + + with pytest.raises(ValueError) as err: + run_state_machine_as_test(BadRuleWithGoodInvariants) + + expected = """\ +Falsifying example: +state = BadRuleWithGoodInvariants() +state.invariant_1() +state.initialize_1() +state.invariant_1() +state.invariant_2() +state.rule_1() +state.invariant_1() +state.invariant_2() +state.invariant_3() +state.rule_1() +state.teardown()""" + + if PYPY or sys.gettrace(): # explain mode disabled in these cases + result = "\n".join(err.value.__notes__) + else: + # Non-PyPy runs include explain mode, but we skip the final line because + # it includes the absolute path, which of course varies between machines. + expected += """ +Explanation: + These lines were always and only run by failing examples:""" + result = "\n".join(err.value.__notes__[:-1]) + + assert expected == result + + def test_always_runs_at_least_one_step(): class CountSteps(RuleBasedStateMachine): def __init__(self): @@ -839,12 +764,12 @@ def delete(self, k, v): def values_agree(self, k): assert not self.__deleted[k] - with capture_out() as o: - with pytest.raises(AssertionError): - run_state_machine_as_test(IncorrectDeletion) + with pytest.raises(AssertionError) as err: + run_state_machine_as_test(IncorrectDeletion) - assert o.getvalue().count(" = state.k(") == 1 - assert o.getvalue().count(" = state.v(") == 1 + result = "\n".join(err.value.__notes__) + assert result.count(" = state.k(") == 1 + assert result.count(" = state.v(") == 1 def test_prints_equal_values_with_correct_variable_name(): @@ -863,13 +788,12 @@ def transfer(self, source): @rule(source=b2) def fail(self, source): - assert False + raise AssertionError - with capture_out() as o: - with pytest.raises(AssertionError): - run_state_machine_as_test(MovesBetweenBundles) + with pytest.raises(AssertionError) as err: + run_state_machine_as_test(MovesBetweenBundles) - result = o.getvalue() + result = "\n".join(err.value.__notes__) for m in ["create", "transfer", "fail"]: assert result.count("state." + m) == 1 assert "v1 = state.create()" in result @@ -896,14 +820,13 @@ def initialize_c(self): @rule() def fail_fast(self): - assert False + raise AssertionError - with capture_out() as o: - with pytest.raises(AssertionError): - run_state_machine_as_test(WithInitializeRules) + with pytest.raises(AssertionError) as err: + run_state_machine_as_test(WithInitializeRules) assert set(WithInitializeRules.initialized[-3:]) == {"a", "b", "c"} - result = o.getvalue().splitlines()[1:] + result = err.value.__notes__[1:] assert result[0] == "state = WithInitializeRules()" # Initialize rules call order is shuffled assert {result[1], result[2], result[3]} == { @@ -925,23 +848,22 @@ def initialize_a(self, dep): @rule(param=a) def fail_fast(self, param): - assert False + raise AssertionError WithInitializeBundleRules.TestCase.settings = NO_BLOB_SETTINGS - with capture_out() as o: - with pytest.raises(AssertionError): - run_state_machine_as_test(WithInitializeBundleRules) + with pytest.raises(AssertionError) as err: + run_state_machine_as_test(WithInitializeBundleRules) - result = o.getvalue() + result = "\n".join(err.value.__notes__) assert ( result - == """\ + == """ Falsifying example: state = WithInitializeBundleRules() v1 = state.initialize_a(dep='dep') state.fail_fast(param=v1) state.teardown() -""" +""".strip() ) @@ -1008,14 +930,13 @@ def initialize_b(self): @rule() def fail_fast(self): - assert False + raise AssertionError - with capture_out() as o: - with pytest.raises(AssertionError): - run_state_machine_as_test(ChildStateMachine) + with pytest.raises(AssertionError) as err: + run_state_machine_as_test(ChildStateMachine) assert set(ChildStateMachine.initialized[-2:]) == {"a", "b"} - result = o.getvalue().splitlines()[1:] + result = err.value.__notes__[1:] assert result[0] == "state = ChildStateMachine()" # Initialize rules call order is shuffled assert {result[1], result[2]} == {"state.initialize_a()", "state.initialize_b()"} @@ -1030,59 +951,47 @@ class StateMachine(RuleBasedStateMachine): @initialize() def initialize(self): self.initialize_called_counter += 1 - return self.initialize_called_counter @rule() def fail_eventually(self): - assert self.initialize() <= 2 + self.initialize() + assert self.initialize_called_counter <= 2 StateMachine.TestCase.settings = NO_BLOB_SETTINGS - with capture_out() as o: - with pytest.raises(AssertionError): - run_state_machine_as_test(StateMachine) + with pytest.raises(AssertionError) as err: + run_state_machine_as_test(StateMachine) - result = o.getvalue() + result = "\n".join(err.value.__notes__) assert ( result - == """\ + == """ Falsifying example: state = StateMachine() state.initialize() state.fail_eventually() state.fail_eventually() state.teardown() -""" +""".strip() ) -def test_new_initialize_rules_are_picked_up_before_and_after_rules_call(): - class Foo(RuleBasedStateMachine): - pass - - Foo.define_initialize_rule(targets=(), function=lambda self: 1, arguments={}) - assert len(Foo.initialize_rules()) == 1 - Foo.define_initialize_rule(targets=(), function=lambda self: 2, arguments={}) - assert len(Foo.initialize_rules()) == 2 - - -def test_steps_printed_despite_pytest_fail(capsys): +def test_steps_printed_despite_pytest_fail(): # Test for https://github.com/HypothesisWorks/hypothesis/issues/1372 + @Settings(print_blob=False) class RaisesProblem(RuleBasedStateMachine): @rule() def oops(self): pytest.fail() - with pytest.raises(Failed): + with pytest.raises(Failed) as err: run_state_machine_as_test(RaisesProblem) - out, _ = capsys.readouterr() assert ( - """\ + "\n".join(err.value.__notes__).strip() + == """ Falsifying example: state = RaisesProblem() state.oops() -state.teardown() -""" - in out +state.teardown()""".strip() ) @@ -1127,7 +1036,7 @@ def test_uses_seed(capsys): class TrivialMachine(RuleBasedStateMachine): @rule() def oops(self): - assert False + raise AssertionError with pytest.raises(AssertionError): run_state_machine_as_test(TrivialMachine) @@ -1140,7 +1049,7 @@ def test_reproduce_failure_works(): class TrivialMachine(RuleBasedStateMachine): @rule() def oops(self): - assert False + raise AssertionError with pytest.raises(AssertionError): run_state_machine_as_test(TrivialMachine, settings=Settings(print_blob=True)) @@ -1151,7 +1060,7 @@ def test_reproduce_failure_fails_if_no_error(): class TrivialMachine(RuleBasedStateMachine): @rule() def ok(self): - assert True + pass with pytest.raises(DidNotReproduce): run_state_machine_as_test(TrivialMachine, settings=Settings(print_blob=True)) @@ -1175,9 +1084,111 @@ def init_data(self, value): def mostly_fails(self, d): assert d == 42 - with capture_out() as o: - with pytest.raises(AssertionError): - run_state_machine_as_test(TrickyPrintingMachine) - output = o.getvalue() - assert "v1 = state.init_data(value=0)" in output - assert "v1 = state.init_data(value=v1)" not in output + with pytest.raises(AssertionError) as err: + run_state_machine_as_test(TrickyPrintingMachine) + assert "v1 = state.init_data(value=0)" in err.value.__notes__ + assert "v1 = state.init_data(value=v1)" not in err.value.__notes__ + + +def test_multiple_precondition_bug(): + # See https://github.com/HypothesisWorks/hypothesis/issues/2861 + class MultiplePreconditionMachine(RuleBasedStateMachine): + @rule(x=integers()) + def good_method(self, x): + pass + + @precondition(lambda self: True) + @precondition(lambda self: False) + @rule(x=integers()) + def bad_method_a(self, x): + raise AssertionError("This rule runs, even though it shouldn't.") + + @precondition(lambda self: False) + @precondition(lambda self: True) + @rule(x=integers()) + def bad_method_b(self, x): + raise AssertionError("This rule might be skipped for the wrong reason.") + + @precondition(lambda self: True) + @rule(x=integers()) + @precondition(lambda self: False) + def bad_method_c(self, x): + raise AssertionError("This rule runs, even though it shouldn't.") + + @rule(x=integers()) + @precondition(lambda self: True) + @precondition(lambda self: False) + def bad_method_d(self, x): + raise AssertionError("This rule runs, even though it shouldn't.") + + @precondition(lambda self: True) + @precondition(lambda self: False) + @invariant() + def bad_invariant_a(self): + raise AssertionError("This invariant runs, even though it shouldn't.") + + @precondition(lambda self: False) + @precondition(lambda self: True) + @invariant() + def bad_invariant_b(self): + raise AssertionError("This invariant runs, even though it shouldn't.") + + @precondition(lambda self: True) + @invariant() + @precondition(lambda self: False) + def bad_invariant_c(self): + raise AssertionError("This invariant runs, even though it shouldn't.") + + @invariant() + @precondition(lambda self: True) + @precondition(lambda self: False) + def bad_invariant_d(self): + raise AssertionError("This invariant runs, even though it shouldn't.") + + run_state_machine_as_test(MultiplePreconditionMachine) + + +class TrickyInitMachine(RuleBasedStateMachine): + @initialize() + def init_a(self): + self.a = 0 + + @rule() + def inc(self): + self.a += 1 + + @invariant() + def check_a_positive(self): + # This will fail if run before the init_a method, but without + # @invariant(check_during_init=True) it will only run afterwards. + assert self.a >= 0 + + +def test_invariants_are_checked_after_init_steps(): + run_state_machine_as_test(TrickyInitMachine) + + +def test_invariants_can_be_checked_during_init_steps(): + class UndefinedMachine(TrickyInitMachine): + @invariant(check_during_init=True) + def check_a_defined(self): + # This will fail because `a` is undefined before the init rule. + self.a + + with pytest.raises(AttributeError): + run_state_machine_as_test(UndefinedMachine) + + +def test_check_during_init_must_be_boolean(): + invariant(check_during_init=False) + invariant(check_during_init=True) + with pytest.raises(InvalidArgument): + invariant(check_during_init="not a bool") + + +def test_deprecated_target_consumes_bundle(): + # It would be nicer to raise this error at runtime, but the internals make + # this sadly impractical. Most InvalidDefinition errors happen at, well, + # definition-time already anyway, so it's not *worse* than the status quo. + with validate_deprecation(): + rule(target=consumes(Bundle("b"))) diff --git a/hypothesis-python/tests/cover/test_statistical_events.py b/hypothesis-python/tests/cover/test_statistical_events.py index 5e8eef93f5..0230e431dc 100644 --- a/hypothesis-python/tests/cover/test_statistical_events.py +++ b/hypothesis-python/tests/cover/test_statistical_events.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import time import traceback @@ -142,7 +137,7 @@ def test(i): if first[0]: first[0] = False print("Hi") - assert False + raise AssertionError stats = call_for_statistics(test) assert stats["stopped-because"] == "test was flaky" @@ -157,6 +152,7 @@ def test_draw_time_percentage(draw_delay, test_delay): def s(draw): if draw_delay: time.sleep(0.05) + draw(st.integers()) @given(s()) def test(_): @@ -259,3 +255,8 @@ def test(value): stats = describe_statistics(call_for_statistics(test)) assert "- Events:" in stats assert "- Highest target score: " in stats + + +@given(st.booleans()) +def test_event_with_non_weakrefable_keys(b): + event((b,)) diff --git a/hypothesis-python/tests/cover/test_subnormal_floats.py b/hypothesis-python/tests/cover/test_subnormal_floats.py new file mode 100644 index 0000000000..4877ce8ed5 --- /dev/null +++ b/hypothesis-python/tests/cover/test_subnormal_floats.py @@ -0,0 +1,97 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +from sys import float_info + +import pytest + +from hypothesis.errors import InvalidArgument +from hypothesis.internal.floats import next_down, next_up +from hypothesis.strategies import floats +from hypothesis.strategies._internal.numbers import next_down_normal, next_up_normal + +from tests.common.debug import assert_no_examples, find_any +from tests.common.utils import PYTHON_FTZ + +pytestmark = [pytest.mark.skipif(PYTHON_FTZ, reason="broken by unsafe compiler flags")] + + +def kw(marks=(), **kwargs): + id_ = ", ".join(f"{k}={v!r}" for k, v in kwargs.items()) + return pytest.param(kwargs, id=id_, marks=marks) + + +@pytest.mark.parametrize( + "kwargs", + [ + kw(min_value=1), + kw(min_value=1), + kw(max_value=-1), + kw(min_value=float_info.min), + kw(min_value=next_down(float_info.min), exclude_min=True), + kw(max_value=-float_info.min), + kw(min_value=next_up(-float_info.min), exclude_max=True), + ], +) +def test_subnormal_validation(kwargs): + strat = floats(**kwargs, allow_subnormal=True) + with pytest.raises(InvalidArgument): + strat.example() + + +@pytest.mark.parametrize( + "kwargs", + [ + # min value + kw(allow_subnormal=False, min_value=1), + kw(allow_subnormal=False, min_value=float_info.min), + kw(allow_subnormal=True, min_value=-1), + kw(allow_subnormal=True, min_value=next_down(float_info.min)), + # max value + kw(allow_subnormal=False, max_value=-1), + kw(allow_subnormal=False, max_value=-float_info.min), + kw(allow_subnormal=True, max_value=1), + kw(allow_subnormal=True, max_value=next_up(-float_info.min)), + # min/max values + kw(allow_subnormal=True, min_value=-1, max_value=1), + kw( + allow_subnormal=True, + min_value=next_down(float_info.min), + max_value=float_info.min, + ), + kw( + allow_subnormal=True, + min_value=-float_info.min, + max_value=next_up(-float_info.min), + ), + kw(allow_subnormal=False, min_value=-1, max_value=-float_info.min), + kw(allow_subnormal=False, min_value=float_info.min, max_value=1), + ], +) +def test_allow_subnormal_defaults_correctly(kwargs): + allow_subnormal = kwargs.pop("allow_subnormal") + strat = floats(**kwargs).filter(lambda x: x != 0) + if allow_subnormal: + find_any(strat, lambda x: -float_info.min < x < float_info.min) + else: + assert_no_examples(strat, lambda x: -float_info.min < x < float_info.min) + + +@pytest.mark.parametrize( + "func, val, expected", + [ + (next_up_normal, -float_info.min, -0.0), + (next_up_normal, +0.0, float_info.min), + (next_down_normal, float_info.min, +0.0), + (next_down_normal, -0.0, -float_info.min), + ], +) +def test_next_float_normal(func, val, expected): + assert func(value=val, width=64, allow_subnormal=False) == expected diff --git a/hypothesis-python/tests/cover/test_targeting.py b/hypothesis-python/tests/cover/test_targeting.py index 6532cfc101..c645643f9d 100644 --- a/hypothesis-python/tests/cover/test_targeting.py +++ b/hypothesis-python/tests/cover/test_targeting.py @@ -1,23 +1,18 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from operator import itemgetter import pytest -from hypothesis import Phase, example, given, seed, settings, strategies as st, target +from hypothesis import example, given, strategies as st, target from hypothesis.errors import InvalidArgument @@ -68,7 +63,7 @@ def test_respects_max_pool_size(observations): def everything_except(type_): - # Note: we would usually stick to fater traditional or parametrized + # Note: we would usually stick to faster traditional or parametrized # tests to check that invalid inputs are rejected, but for `target()` # we need to use `@given` (to validate arguments instead of context) # so we might as well apply this neat recipe. @@ -125,36 +120,3 @@ def test_targeting_with_many_empty(_): # This exercises some logic in the optimiser that prevents it from trying # to mutate empty examples in the middle of the test case. target(1.0) - - -def test_targeting_can_be_disabled(): - strat = st.lists(st.integers(0, 255)) - - def score(enabled): - result = [0] - phases = [Phase.generate] - if enabled: - phases.append(Phase.target) - - @seed(0) - @settings(database=None, max_examples=200, phases=phases) - @given(strat) - def test(ls): - score = float(sum(ls)) - result[0] = max(result[0], score) - target(score) - - test() - return result[0] - - assert score(enabled=True) > score(enabled=False) - - -def test_issue_2395_regression(): - @given(d=st.floats().filter(lambda x: abs(x) < 1000)) - @settings(max_examples=1000, database=None) - @seed(93962505385993024185959759429298090872) - def test_targeting_square_loss(d): - target(-((d - 42.5) ** 2.0)) - - test_targeting_square_loss() diff --git a/hypothesis-python/tests/cover/test_testdecorators.py b/hypothesis-python/tests/cover/test_testdecorators.py index f1ff2ade4b..2bd583e658 100644 --- a/hypothesis-python/tests/cover/test_testdecorators.py +++ b/hypothesis-python/tests/cover/test_testdecorators.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import functools import threading @@ -43,6 +38,10 @@ raises, ) +# This particular test file is run under both pytest and nose, so it can't +# rely on pytest-specific helpers like `pytest.raises` unless we define a +# fallback in tests.common.utils. + @given(integers(), integers()) def test_int_addition_is_commutative(x, y): @@ -50,14 +49,17 @@ def test_int_addition_is_commutative(x, y): @fails -@given(text(), text()) +@given(text(min_size=1), text(min_size=1)) def test_str_addition_is_commutative(x, y): assert x + y == y + x @fails -@given(binary(), binary()) +@given(binary(min_size=1), binary(min_size=1)) def test_bytes_addition_is_commutative(x, y): + # We enforce min_size=1 here to avoid a rare flakiness, where the + # test passes if x and/or y are b"" for every generated example. + # (the internal implementation makes this less rare for bytes) assert x + y == y + x @@ -85,11 +87,9 @@ def is_not_too_large(x): if x >= 10: raise ValueError(f"No, {x} is just too large. Sorry") - with raises(ValueError) as exinfo: + with raises(ValueError, match=" 10 "): is_not_too_large() - assert " 10 " in exinfo.value.args[0] - @given(integers()) def test_integer_division_shrinks_positive_integers(n): @@ -114,7 +114,7 @@ def test_abs_non_negative_varargs_kwargs(self, *args, **kw): assert isinstance(self, TestCases) @given(x=integers()) - def test_abs_non_negative_varargs_kwargs_only(*args, **kw): + def test_abs_non_negative_varargs_kwargs_only(*args, **kw): # noqa: B902 assert abs(kw["x"]) >= 0 assert isinstance(args[0], TestCases) @@ -147,18 +147,6 @@ def test_is_the_answer(x): assert x == 42 -@fails -@given(text(), text()) -def test_text_addition_is_not_commutative(x, y): - assert x + y == y + x - - -@fails -@given(binary(), binary()) -def test_binary_addition_is_not_commutative(x, y): - assert x + y == y + x - - @given(integers(1, 10)) def test_integers_are_in_range(x): assert 1 <= x <= 10 @@ -290,19 +278,21 @@ def test_is_ascii(x): def test_is_not_ascii(x): try: x.encode("ascii") - assert False + raise AssertionError except UnicodeEncodeError: pass @fails -@given(text()) +@given(text(min_size=2)) +@settings(max_examples=100, derandomize=True) def test_can_find_string_with_duplicates(s): assert len(set(s)) == len(s) @fails -@given(text()) +@given(text(min_size=1)) +@settings(derandomize=True) def test_has_ascii(x): if not x: return @@ -334,7 +324,7 @@ def test_can_run_without_database(): @given(integers()) @settings(database=None) def test_blah(x): - assert False + raise AssertionError with raises(AssertionError): test_blah() @@ -414,15 +404,14 @@ def foo(x): if x > 11: note("Lo") failing.append(x) - assert False + raise AssertionError - with raises(AssertionError): - with capture_out() as out: - foo() + with raises(AssertionError) as err: + foo() assert len(failing) == 2 assert len(set(failing)) == 1 - assert "Falsifying example" in out.getvalue() - assert "Lo" in out.getvalue() + assert "Falsifying example" in "\n".join(err.value.__notes__) + assert "Lo" in err.value.__notes__ @given(integers().filter(lambda x: x % 4 == 0)) @@ -476,14 +465,17 @@ def test(xs): if sum(xs) <= 100: raise ValueError() - with capture_out() as out: - with reporting.with_reporter(reporting.default): - with raises(ValueError): - test() - lines = out.getvalue().strip().splitlines() - assert lines.count("Hi there") == 1 + with raises(ValueError) as err: + test() + assert err.value.__notes__.count("Hi there") == 1 @given(lists(integers(), max_size=0)) def test_empty_lists(xs): assert xs == [] + + +def test_given_usable_inline_on_lambdas(): + xs = [] + given(booleans())(lambda x: xs.append(x))() + assert len(xs) == 2 and set(xs) == {False, True} diff --git a/hypothesis-python/tests/cover/test_text.py b/hypothesis-python/tests/cover/test_text.py index be5e35f590..964db142f3 100644 --- a/hypothesis-python/tests/cover/test_text.py +++ b/hypothesis-python/tests/cover/test_text.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis.strategies._internal.strings import OneCharStringStrategy diff --git a/hypothesis-python/tests/cover/test_traceback_elision.py b/hypothesis-python/tests/cover/test_traceback_elision.py index 6d687b585d..31106de46b 100644 --- a/hypothesis-python/tests/cover/test_traceback_elision.py +++ b/hypothesis-python/tests/cover/test_traceback_elision.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import traceback diff --git a/hypothesis-python/tests/cover/test_type_lookup.py b/hypothesis-python/tests/cover/test_type_lookup.py index 397683d831..14460d9df6 100644 --- a/hypothesis-python/tests/cover/test_type_lookup.py +++ b/hypothesis-python/tests/cover/test_type_lookup.py @@ -1,57 +1,64 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER +import abc import enum -import sys -from typing import Callable, Generic, List, Sequence, TypeVar, Union +from inspect import Parameter as P, Signature +from typing import Callable, Dict, Generic, List, Sequence, TypeVar, Union import pytest -from hypothesis import given, infer, strategies as st +from hypothesis import given, infer, settings, strategies as st from hypothesis.errors import ( HypothesisDeprecationWarning, InvalidArgument, ResolutionFailed, ) +from hypothesis.internal.compat import get_type_hints +from hypothesis.internal.reflection import get_pretty_function_description from hypothesis.strategies._internal import types -from hypothesis.strategies._internal.core import _strategies +from hypothesis.strategies._internal.core import _from_type from hypothesis.strategies._internal.types import _global_type_lookup +from hypothesis.strategies._internal.utils import _strategies from tests.common.debug import assert_all_examples, find_any from tests.common.utils import fails_with, temp_registered # Build a set of all types output by core strategies -blacklist = [ +blocklist = { "builds", + "data", + "deferred", "from_regex", "from_type", "ip_addresses", "iterables", + "just", + "nothing", + "one_of", "permutations", "random_module", "randoms", + "recursive", "runner", "sampled_from", + "shared", "timezone_keys", "timezones", -] +} +assert set(_strategies).issuperset(blocklist), blocklist.difference(_strategies) types_with_core_strat = set() for thing in ( getattr(st, name) for name in sorted(_strategies) - if name in dir(st) and name not in blacklist + if name in dir(st) and name not in blocklist ): for n in range(3): try: @@ -167,13 +174,14 @@ def test_pulic_interface_works(): fails.example() -def test_given_can_infer_from_manual_annotations(): +@pytest.mark.parametrize("infer_token", [infer, ...]) +def test_given_can_infer_from_manual_annotations(infer_token): # Editing annotations before decorating is hilariously awkward, but works! def inner(a): - pass + assert isinstance(a, int) inner.__annotations__ = {"a": int} - given(a=infer)(inner)() + given(a=infer_token)(inner)() class EmptyEnum(enum.Enum): @@ -200,8 +208,13 @@ def test_uninspectable_from_type(): st.from_type(BrokenClass).example() +def _check_instances(t): + # See https://github.com/samuelcolvin/pydantic/discussions/2508 + return t.__module__ != "typing" and not t.__module__.startswith("pydantic") + + @pytest.mark.parametrize( - "typ", sorted((x for x in _global_type_lookup if x.__module__ != "typing"), key=str) + "typ", sorted((x for x in _global_type_lookup if _check_instances(x)), key=str) ) @given(data=st.data()) def test_can_generate_from_all_registered_types(data, typ): @@ -217,6 +230,18 @@ def __init__(self, arg: T) -> None: self.arg = arg +class Lines(Sequence[str]): + """Represent a sequence of text lines. + + It turns out that resolving a class which inherits from a parametrised generic + type is... tricky. See https://github.com/HypothesisWorks/hypothesis/issues/2951 + """ + + +class SpecificDict(Dict[int, int]): + pass + + def using_generic(instance: MyGeneric[T]) -> T: return instance.arg @@ -230,7 +255,22 @@ def test_generic_origin_empty(): find_any(st.builds(using_generic)) -_skip_callables_mark = pytest.mark.skipif(sys.version_info[:2] < (3, 7), reason="old") +def test_issue_2951_regression(): + lines_strat = st.builds(Lines, lines=st.lists(st.text())) + with temp_registered(Lines, lines_strat): + assert st.from_type(Lines) == lines_strat + # Now let's test that the strategy for ``Sequence[int]`` did not + # change just because we registered a strategy for ``Lines``: + expected = "one_of(binary(), lists(integers()))" + assert repr(st.from_type(Sequence[int])) == expected + + +def test_issue_2951_regression_two_params(): + map_strat = st.builds(SpecificDict, st.dictionaries(st.integers(), st.integers())) + expected = repr(st.from_type(Dict[int, int])) + with temp_registered(SpecificDict, map_strat): + assert st.from_type(SpecificDict) == map_strat + assert expected == repr(st.from_type(Dict[int, int])) @pytest.mark.parametrize( @@ -239,10 +279,8 @@ def test_generic_origin_empty(): Union[str, int], Sequence[Sequence[int]], MyGeneric[str], - # On Python <= 3.6, we always trigger the multi-registration guard clause - # and raise InvalidArgument on the first attempted registration. - pytest.param(Callable[..., str], marks=_skip_callables_mark), - pytest.param(Callable[[int], str], marks=_skip_callables_mark), + Callable[..., str], + Callable[[int], str], ), ids=repr, ) @@ -272,13 +310,20 @@ def test_generic_origin_without_type_args(generic): pass -def test_generic_origin_from_type(): +@pytest.mark.parametrize( + "strat, type_", + [ + (st.from_type, MyGeneric[T]), + (st.from_type, MyGeneric[int]), + (st.from_type, MyGeneric), + (st.builds, using_generic), + (st.builds, using_concrete_generic), + ], + ids=get_pretty_function_description, +) +def test_generic_origin_from_type(strat, type_): with temp_registered(MyGeneric, st.builds(MyGeneric)): - find_any(st.from_type(MyGeneric[T])) - find_any(st.from_type(MyGeneric[int])) - find_any(st.from_type(MyGeneric)) - find_any(st.builds(using_generic)) - find_any(st.builds(using_concrete_generic)) + find_any(strat(type_)) def test_generic_origin_concrete_builds(): @@ -286,3 +331,98 @@ def test_generic_origin_concrete_builds(): assert_all_examples( st.builds(using_generic), lambda example: isinstance(example, int) ) + + +class AbstractFoo(abc.ABC): + def __init__(self, x): + pass + + @abc.abstractmethod + def qux(self): + pass + + +class ConcreteFoo1(AbstractFoo): + # Can't resolve this one due to unannotated `x` param + def qux(self): + pass + + +class ConcreteFoo2(AbstractFoo): + def __init__(self, x: int): + pass + + def qux(self): + pass + + +@given(st.from_type(AbstractFoo)) +def test_gen_abstract(foo): + # This requires that we correctly checked which of the subclasses + # could be resolved, rather than unconditionally using all of them. + assert isinstance(foo, ConcreteFoo2) + + +class AbstractBar(abc.ABC): + def __init__(self, x): + pass + + @abc.abstractmethod + def qux(self): + pass + + +class ConcreteBar(AbstractBar): + def qux(self): + pass + + +def test_abstract_resolver_fallback(): + # We create our distinct strategies for abstract and concrete types + gen_abstractbar = _from_type(AbstractBar) + gen_concretebar = st.builds(ConcreteBar, x=st.none()) + assert gen_abstractbar != gen_concretebar + + # And trying to generate an instance of the abstract type fails, + # UNLESS the concrete type is currently resolvable + with pytest.raises(ResolutionFailed): + gen_abstractbar.example() + with temp_registered(ConcreteBar, gen_concretebar): + gen = gen_abstractbar.example() + with pytest.raises(ResolutionFailed): + gen_abstractbar.example() + + # which in turn means we resolve to the concrete subtype. + assert isinstance(gen, ConcreteBar) + + +def _one_arg(x: int): + assert isinstance(x, int) + + +def _multi_arg(x: int, y: str): + assert isinstance(x, int) + assert isinstance(y, str) + + +def _kwd_only(*, y: str): + assert isinstance(y, str) + + +def _pos_and_kwd_only(x: int, *, y: str): + assert isinstance(x, int) + assert isinstance(y, str) + + +@pytest.mark.parametrize("func", [_one_arg, _multi_arg, _kwd_only, _pos_and_kwd_only]) +def test_infer_all(func): + # tests @given(...) against various signatures + settings(max_examples=1)(given(...))(func)() + + +def test_does_not_add_param_empty_to_type_hints(): + def f(x): + pass + + f.__signature__ = Signature([P("y", P.KEYWORD_ONLY)], return_annotation=None) + assert get_type_hints(f) == {} diff --git a/hypothesis-python/tests/cover/test_type_lookup_forward_ref.py b/hypothesis-python/tests/cover/test_type_lookup_forward_ref.py index 6f867eb629..488023b6c7 100644 --- a/hypothesis-python/tests/cover/test_type_lookup_forward_ref.py +++ b/hypothesis-python/tests/cover/test_type_lookup_forward_ref.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """ We need these test to make sure ``TypeVar('X', bound='MyType')`` works correctly. @@ -19,35 +14,26 @@ There was a problem previously that ``bound='MyType'`` was resolved as ``ForwardRef('MyType')`` which is not a real type. And ``hypothesis`` was not able to generate any meaningful values out of it. -Right here we test different possible outcomes for different Python versions (excluding ``3.5``): +Right here we test different possible outcomes for different Python versions: - Regular case, when ``'MyType'`` can be imported - Alias case, when we use type aliases for ``'MyType'`` - ``if TYPE_CHECKING:`` case, when ``'MyType'`` only exists during type checking and is not importable at all - Dot access case, like ``'module.MyType'`` - Missing case, when there's no ``'MyType'`` at all - -We also separate how ``3.6`` works, because it has its limitations. -Basically, ``TypeVar`` has no information about the module it was defined at. """ -import sys -from typing import TYPE_CHECKING, TypeVar +from typing import TYPE_CHECKING, ForwardRef, TypeVar import pytest from hypothesis import given, strategies as st from hypothesis.errors import ResolutionFailed -from hypothesis.internal.compat import ForwardRef from tests.common import utils if TYPE_CHECKING: from tests.common.utils import ExcInfo # we just need any type # noqa: F401 -skip_before_python37 = pytest.mark.skipif( - sys.version_info[:2] < (3, 7), reason="typing module was broken" -) - # Correct: _Correct = TypeVar("_Correct", bound="CustomType") @@ -62,26 +48,12 @@ def __init__(self, arg: int) -> None: self.arg = arg -@skip_before_python37 @given(st.builds(correct_fun)) def test_bound_correct_forward_ref(built): - """Correct resolution of existing type in ``python3.7+`` codebase.""" + """Correct resolution of existing type codebase.""" assert isinstance(built, int) -@pytest.mark.skipif( - sys.version_info[:2] != (3, 6), reason="typing in python3.6 is partially working" -) -def test_bound_correct_forward_ref_python36(): - """ - Very special case for ``python3.6`` where we have this feature partially suported. - - Due to ``TypeVar`` module definition bug. - """ - with pytest.raises(ResolutionFailed): - st.builds(correct_fun).example() - - # Alises: _Alias = TypeVar("_Alias ", bound="OurAlias") @@ -94,17 +66,16 @@ def alias_fun(thing: _Alias) -> int: OurAlias = CustomType -@skip_before_python37 @given(st.builds(alias_fun)) def test_bound_alias_forward_ref(built): - """Correct resolution of type aliases in ``python3.7+``.""" + """Correct resolution of type aliases.""" assert isinstance(built, int) # Dot access: _CorrectDotAccess = TypeVar("_CorrectDotAccess", bound="utils.ExcInfo") -_WrongDotAccess = TypeVar("_WrongDotAccess", bound="wrong.ExcInfo") +_WrongDotAccess = TypeVar("_WrongDotAccess", bound="wrong.ExcInfo") # noqa _MissingDotAccess = TypeVar("_MissingDotAccess", bound="utils.MissingType") @@ -120,31 +91,28 @@ def missing_dot_access_fun(thing: _MissingDotAccess) -> int: return 1 -@skip_before_python37 @given(st.builds(correct_dot_access_fun)) def test_bound_correct_dot_access_forward_ref(built): - """Correct resolution of dot access types in ``python3.7+``.""" + """Correct resolution of dot access types.""" assert isinstance(built, int) -@skip_before_python37 @pytest.mark.parametrize("function", [wrong_dot_access_fun, missing_dot_access_fun]) def test_bound_missing_dot_access_forward_ref(function): - """Resolution of missing type in dot access in ``python3.7+``.""" + """Resolution of missing type in dot access.""" with pytest.raises(ResolutionFailed): st.builds(function).example() # Missing: -_Missing = TypeVar("_Missing", bound="MissingType") +_Missing = TypeVar("_Missing", bound="MissingType") # noqa def missing_fun(thing: _Missing) -> int: return 1 -@pytest.mark.skipif(sys.version_info[:2] < (3, 6), reason="typing module was strange") def test_bound_missing_forward_ref(): """We should raise proper errors on missing types.""" with pytest.raises(ResolutionFailed): @@ -160,14 +128,12 @@ def typechecking_only_fun(thing: _TypeChecking) -> int: return 1 -@skip_before_python37 def test_bound_type_cheking_only_forward_ref(): """We should fallback to registering explicit ``ForwardRef`` when we have to.""" with utils.temp_registered(ForwardRef("ExcInfo"), st.just(1)): st.builds(typechecking_only_fun).example() -@skip_before_python37 def test_bound_type_cheking_only_forward_ref_wrong_type(): """We should check ``ForwardRef`` parameter name correctly.""" with utils.temp_registered(ForwardRef("WrongType"), st.just(1)): diff --git a/hypothesis-python/tests/cover/test_unicode_identifiers.py b/hypothesis-python/tests/cover/test_unicode_identifiers.py index 731654fb1d..ce88bff974 100644 --- a/hypothesis-python/tests/cover/test_unicode_identifiers.py +++ b/hypothesis-python/tests/cover/test_unicode_identifiers.py @@ -1,23 +1,18 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import given, strategies as st from hypothesis.internal.reflection import get_pretty_function_description, proxies -def test_can_copy_argspec_of_unicode_args(): +def test_can_copy_signature_of_unicode_args(): def foo(μ): return μ @@ -28,7 +23,7 @@ def bar(μ): assert bar(1) == 1 -def test_can_copy_argspec_of_unicode_name(): +def test_can_copy_signature_of_unicode_name(): def ā(): return 1 diff --git a/hypothesis-python/tests/cover/test_unittest.py b/hypothesis-python/tests/cover/test_unittest.py index 513d3b0899..af638045a0 100644 --- a/hypothesis-python/tests/cover/test_unittest.py +++ b/hypothesis-python/tests/cover/test_unittest.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import io import sys @@ -56,10 +51,11 @@ def test(self): SUBTEST_SUITE = """ import unittest -from hypothesis import given, strategies as st +from hypothesis import given, settings, strategies as st class MyTest(unittest.TestCase): @given(s=st.text()) + @settings(deadline=None) def test_subtest(self, s): with self.subTest(text=s): self.assertIsInstance(s, str) diff --git a/hypothesis-python/tests/cover/test_uuids.py b/hypothesis-python/tests/cover/test_uuids.py new file mode 100644 index 0000000000..3103439186 --- /dev/null +++ b/hypothesis-python/tests/cover/test_uuids.py @@ -0,0 +1,34 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import uuid + +import pytest + +from hypothesis import strategies as st +from hypothesis.errors import InvalidArgument + +from tests.common.debug import assert_no_examples, find_any + + +def test_no_nil_uuid_by_default(): + assert_no_examples(st.uuids(), lambda x: x == uuid.UUID(int=0)) + + +def test_can_generate_nil_uuid(): + find_any(st.uuids(allow_nil=True), lambda x: x == uuid.UUID(int=0)) + + +def test_can_only_allow_nil_uuid_with_none_version(): + st.uuids(version=None, allow_nil=True).example() + with pytest.raises(InvalidArgument): + st.uuids(version=4, allow_nil=True).example() + with pytest.raises(InvalidArgument): + st.uuids(version=None, allow_nil="not a bool").example() diff --git a/hypothesis-python/tests/cover/test_validation.py b/hypothesis-python/tests/cover/test_validation.py index 30f42cf694..ffc52e4678 100644 --- a/hypothesis-python/tests/cover/test_validation.py +++ b/hypothesis-python/tests/cover/test_validation.py @@ -1,19 +1,15 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import functools +import sys import pytest @@ -279,3 +275,13 @@ def test_check_strategy_might_suggest_sampled_from(): with pytest.raises(InvalidArgument, match="such as st.sampled_from"): check_strategy_((1, 2, 3)) check_strategy_(integers(), "passes for our custom coverage check") + + +@pytest.mark.skipif(sys.version_info[:2] >= (3, 8), reason="only on 3.7") +def test_explicit_error_from_array_api_before_py38(): + with pytest.raises( + RuntimeError, match=r"The Array API standard requires Python 3\.8 or later" + ): + from hypothesis.extra import array_api + + assert array_api # avoid unused-name lint error diff --git a/hypothesis-python/tests/cover/test_verbosity.py b/hypothesis-python/tests/cover/test_verbosity.py index 3622220630..f26ce8b22c 100644 --- a/hypothesis-python/tests/cover/test_verbosity.py +++ b/hypothesis-python/tests/cover/test_verbosity.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from contextlib import contextmanager @@ -50,7 +45,7 @@ def test_does_not_log_in_quiet_mode(): @settings(verbosity=Verbosity.quiet, print_blob=False) @given(integers()) def test_foo(x): - assert False + raise AssertionError test_foo() assert not o.getvalue() @@ -66,7 +61,6 @@ def test_includes_progress_in_verbose_mode(): out = o.getvalue() assert out assert "Trying example: " in out - assert "Falsifying example: " in out def test_prints_initial_attempts_on_find(): diff --git a/hypothesis-python/tests/datetime/__init__.py b/hypothesis-python/tests/datetime/__init__.py index e61e3c2872..fcb1ac6538 100644 --- a/hypothesis-python/tests/datetime/__init__.py +++ b/hypothesis-python/tests/datetime/__init__.py @@ -1,14 +1,9 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER diff --git a/hypothesis-python/tests/datetime/test_dateutil_timezones.py b/hypothesis-python/tests/datetime/test_dateutil_timezones.py index 73c2d9921c..0f6181831f 100644 --- a/hypothesis-python/tests/datetime/test_dateutil_timezones.py +++ b/hypothesis-python/tests/datetime/test_dateutil_timezones.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import datetime as dt @@ -127,4 +122,4 @@ def test_datetimes_can_exclude_imaginary(): def test_non_imaginary_datetimes_at_boundary(val): # This is expected to fail because Australia/Sydney is UTC+10, # and the filter logic overflows when checking for round-trips. - assert False + raise AssertionError diff --git a/hypothesis-python/tests/datetime/test_pytz_timezones.py b/hypothesis-python/tests/datetime/test_pytz_timezones.py index f62fbf0699..1023401d10 100644 --- a/hypothesis-python/tests/datetime/test_pytz_timezones.py +++ b/hypothesis-python/tests/datetime/test_pytz_timezones.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import datetime as dt @@ -23,6 +18,7 @@ from hypothesis.errors import InvalidArgument from hypothesis.extra.pytz import timezones from hypothesis.strategies import data, datetimes, just, sampled_from, times +from hypothesis.strategies._internal.datetime import datetime_does_not_exist from tests.common.debug import ( assert_all_examples, @@ -124,36 +120,37 @@ def test_datetimes_stay_within_naive_bounds(data, lo, hi): assert lo <= out.replace(tzinfo=None) <= hi -@pytest.mark.xfail(reason="is_dst not equivalent to fold when DST offset is negative") -def test_datetimes_can_exclude_imaginary(): - # The day of a spring-forward transition; 2am is imaginary - australia = { - "min_value": dt.datetime(2020, 10, 4), - "max_value": dt.datetime(2020, 10, 5), - "timezones": just(pytz.timezone("Australia/Sydney")), - } - # Ireland uses *negative* offset DST, which means that our sloppy interpretation - # of "is_dst=not fold" bypasses the filter for imaginary times. This is basically - # unfixable without redesigning pytz per PEP-495, and it's much more likely to be - # replaced by dateutil or PEP-615 zoneinfo in the standard library instead. - # (we use both so an optimistic `is_dst=bool(fold)` also fails the test) - ireland = { - "min_value": dt.datetime(2019, 3, 31), - "max_value": dt.datetime(2019, 4, 1), - "timezones": just(pytz.timezone("Europe/Dublin")), - } +@pytest.mark.parametrize( + "kw", + [ + # Ireland uses *negative* offset DST, which means that our sloppy interpretation + # of "is_dst=not fold" bypasses the filter for imaginary times. This is basically + # unfixable without redesigning pytz per PEP-495, and it's much more likely to be + # replaced by dateutil or PEP-615 zoneinfo in the standard library instead. + { + "min_value": dt.datetime(2019, 3, 31), + "max_value": dt.datetime(2019, 4, 1), + "timezones": just(pytz.timezone("Europe/Dublin")), + }, + # The day of a spring-forward transition in Australia; 2am is imaginary + # (the common case so an optimistic `is_dst=bool(fold)` also fails the test) + { + "min_value": dt.datetime(2020, 10, 4), + "max_value": dt.datetime(2020, 10, 5), + "timezones": just(pytz.timezone("Australia/Sydney")), + }, + ], +) +def test_datetimes_can_exclude_imaginary(kw): # Sanity check: fail unless those days contain an imaginary hour to filter out - find_any( - datetimes(**australia, allow_imaginary=True), - lambda x: not datetime_exists(x), - ) - find_any( - datetimes(**ireland, allow_imaginary=True), - lambda x: not datetime_exists(x), - ) + find_any(datetimes(**kw, allow_imaginary=True), lambda x: not datetime_exists(x)) + # Assert that with allow_imaginary=False we only generate existing datetimes. - assert_all_examples( - datetimes(**australia, allow_imaginary=False) - | datetimes(**ireland, allow_imaginary=False), - datetime_exists, - ) + assert_all_examples(datetimes(**kw, allow_imaginary=False), datetime_exists) + + +def test_really_weird_tzinfo_case(): + x = dt.datetime(2019, 3, 31, 2, 30, tzinfo=pytz.timezone("Europe/Dublin")) + assert x.tzinfo is not x.astimezone(dt.timezone.utc).astimezone(x.tzinfo) + # And that weird case exercises the rare branch in our helper: + assert datetime_does_not_exist(x) diff --git a/hypothesis-python/tests/datetime/test_zoneinfo_timezones.py b/hypothesis-python/tests/datetime/test_zoneinfo_timezones.py index a5d201d6f7..4832d4de83 100644 --- a/hypothesis-python/tests/datetime/test_zoneinfo_timezones.py +++ b/hypothesis-python/tests/datetime/test_zoneinfo_timezones.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import platform diff --git a/hypothesis-python/tests/django/__init__.py b/hypothesis-python/tests/django/__init__.py index e61e3c2872..fcb1ac6538 100644 --- a/hypothesis-python/tests/django/__init__.py +++ b/hypothesis-python/tests/django/__init__.py @@ -1,14 +1,9 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER diff --git a/hypothesis-python/tests/django/manage.py b/hypothesis-python/tests/django/manage.py index ba14383266..be364ea9a3 100755 --- a/hypothesis-python/tests/django/manage.py +++ b/hypothesis-python/tests/django/manage.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import os import sys @@ -38,4 +33,11 @@ warnings.simplefilter("ignore", category=DeprecationWarning) from django.core.management import execute_from_command_line - execute_from_command_line(sys.argv) + try: + from django.utils.deprecation import RemovedInDjango50Warning + except ImportError: + RemovedInDjango50Warning = () + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=RemovedInDjango50Warning) + execute_from_command_line(sys.argv) diff --git a/hypothesis-python/tests/django/toys/__init__.py b/hypothesis-python/tests/django/toys/__init__.py index e61e3c2872..fcb1ac6538 100644 --- a/hypothesis-python/tests/django/toys/__init__.py +++ b/hypothesis-python/tests/django/toys/__init__.py @@ -1,14 +1,9 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER diff --git a/hypothesis-python/tests/django/toys/settings.py b/hypothesis-python/tests/django/toys/settings.py index e0ada39479..f892444bd8 100644 --- a/hypothesis-python/tests/django/toys/settings.py +++ b/hypothesis-python/tests/django/toys/settings.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """Django settings for toys project. diff --git a/hypothesis-python/tests/django/toys/urls.py b/hypothesis-python/tests/django/toys/urls.py index 0a55de7cb0..69ef75396a 100644 --- a/hypothesis-python/tests/django/toys/urls.py +++ b/hypothesis-python/tests/django/toys/urls.py @@ -1,20 +1,15 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER -from django.conf.urls import include, re_path from django.contrib import admin +from django.urls import include, re_path patterns, namespace, name = admin.site.urls diff --git a/hypothesis-python/tests/django/toys/wsgi.py b/hypothesis-python/tests/django/toys/wsgi.py index 6a269ce490..1f4e75d6e2 100644 --- a/hypothesis-python/tests/django/toys/wsgi.py +++ b/hypothesis-python/tests/django/toys/wsgi.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """WSGI config for toys project. diff --git a/hypothesis-python/tests/django/toystore/__init__.py b/hypothesis-python/tests/django/toystore/__init__.py index e61e3c2872..fcb1ac6538 100644 --- a/hypothesis-python/tests/django/toystore/__init__.py +++ b/hypothesis-python/tests/django/toystore/__init__.py @@ -1,14 +1,9 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER diff --git a/hypothesis-python/tests/django/toystore/admin.py b/hypothesis-python/tests/django/toystore/admin.py index e61e3c2872..fcb1ac6538 100644 --- a/hypothesis-python/tests/django/toystore/admin.py +++ b/hypothesis-python/tests/django/toystore/admin.py @@ -1,14 +1,9 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER diff --git a/hypothesis-python/tests/django/toystore/forms.py b/hypothesis-python/tests/django/toystore/forms.py index 947ddfb40b..7b072d267a 100644 --- a/hypothesis-python/tests/django/toystore/forms.py +++ b/hypothesis-python/tests/django/toystore/forms.py @@ -1,19 +1,15 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from django import forms +from django.contrib.auth.forms import ReadOnlyPasswordHashField, UsernameField from django.core.validators import ( MaxLengthValidator, MaxValueValidator, @@ -206,7 +202,7 @@ def __init__(self, subfield_count=12, **kwargs): super().__init__(fields=subfields, widget=widget) def compress(self, values): - return "::".join([str(x) for x in values]) + return "::".join(str(x) for x in values) class ManyMultiValueForm(ReprForm): @@ -217,3 +213,11 @@ def __init__(self, subfield_count=12, **kwargs): class ShortStringForm(ReprForm): _not_too_long = forms.CharField(max_length=20, required=False) + + +class UsernameForm(ReprForm): + username = UsernameField() + + +class ReadOnlyPasswordHashFieldForm(ReprForm): + password = ReadOnlyPasswordHashField() diff --git a/hypothesis-python/tests/django/toystore/models.py b/hypothesis-python/tests/django/toystore/models.py index 1ee0864adb..270f7d6f8e 100644 --- a/hypothesis-python/tests/django/toystore/models.py +++ b/hypothesis-python/tests/django/toystore/models.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from django.core.exceptions import ValidationError from django.db import models diff --git a/hypothesis-python/tests/django/toystore/test_basic_configuration.py b/hypothesis-python/tests/django/toystore/test_basic_configuration.py index 0da42ecadc..1e08417be4 100644 --- a/hypothesis-python/tests/django/toystore/test_basic_configuration.py +++ b/hypothesis-python/tests/django/toystore/test_basic_configuration.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from unittest import TestCase as VanillaTestCase @@ -81,7 +76,7 @@ def test_given_needs_hypothesis_test_case(self): class LocalTest(DjangoTestCase): @given(integers()) def tst(self, i): - assert False, "InvalidArgument should be raised in @given" + raise AssertionError("InvalidArgument should be raised in @given") with pytest.raises(InvalidArgument): LocalTest("tst").tst() diff --git a/hypothesis-python/tests/django/toystore/test_given_forms.py b/hypothesis-python/tests/django/toystore/test_given_forms.py index 0821010054..9925a52657 100644 --- a/hypothesis-python/tests/django/toystore/test_given_forms.py +++ b/hypothesis-python/tests/django/toystore/test_given_forms.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import given from hypothesis.extra.django import TestCase, from_form, register_field_strategy @@ -34,6 +29,7 @@ SlugFieldForm, TemporalFieldForm, URLFieldForm, + UsernameForm, UUIDFieldForm, WithValidatorsForm, ) @@ -123,3 +119,11 @@ def test_tight_validators_form(self, x): self.assertTrue(1 <= x.data["_decimal_one_to_five"] <= 5) self.assertTrue(1 <= x.data["_float_one_to_five"] <= 5) self.assertTrue(5 <= len(x.data["_string_five_to_ten"]) <= 10) + + @given(from_form(UsernameForm)) + def test_username_form(self, username_form): + self.assertTrue(username_form.is_valid()) + + @given(from_form(UsernameForm)) + def test_read_only_password_hash_field_form(self, password_form): + self.assertTrue(password_form.is_valid()) diff --git a/hypothesis-python/tests/django/toystore/test_given_models.py b/hypothesis-python/tests/django/toystore/test_given_models.py index efe661254c..f8e571a258 100644 --- a/hypothesis-python/tests/django/toystore/test_given_models.py +++ b/hypothesis-python/tests/django/toystore/test_given_models.py @@ -1,25 +1,21 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import datetime as dt +import sys from uuid import UUID from django.conf import settings as django_settings from django.contrib.auth.models import User -from hypothesis import HealthCheck, assume, given, infer, settings +from hypothesis import HealthCheck, assume, given, settings from hypothesis.control import reject from hypothesis.errors import HypothesisException, InvalidArgument from hypothesis.extra.django import ( @@ -115,11 +111,11 @@ def test_mandatory_computed_fields_may_not_be_provided(self): mc = from_model(MandatoryComputed, company=from_model(Company)) self.assertRaises(RuntimeError, mc.example) - @given(from_model(CustomishDefault, customish=infer)) + @given(from_model(CustomishDefault, customish=...)) def test_customish_default_overridden_by_infer(self, x): assert x.customish == "a" - @given(from_model(CustomishDefault, customish=infer)) + @given(from_model(CustomishDefault, customish=...)) def test_customish_infer_uses_registered_instead_of_default(self, x): assert x.customish == "a" @@ -196,7 +192,12 @@ class TestPosOnlyArg(TestCase): def test_user_issue_2369_regression(self, val): pass - def test_from_model_argspec(self): - self.assertRaises(TypeError, from_model().example) - self.assertRaises(TypeError, from_model(Car, None).example) - self.assertRaises(TypeError, from_model(model=Customer).example) + def test_from_model_signature(self): + if sys.version_info[:2] <= (3, 7): + self.assertRaises(TypeError, from_model().example) + self.assertRaises(TypeError, from_model(Car, None).example) + self.assertRaises(TypeError, from_model(model=Customer).example) + else: + self.assertRaises(TypeError, from_model) + self.assertRaises(TypeError, from_model, Car, None) + self.assertRaises(TypeError, from_model, model=Customer) diff --git a/hypothesis-python/tests/django/toystore/views.py b/hypothesis-python/tests/django/toystore/views.py index e61e3c2872..fcb1ac6538 100644 --- a/hypothesis-python/tests/django/toystore/views.py +++ b/hypothesis-python/tests/django/toystore/views.py @@ -1,14 +1,9 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER diff --git a/hypothesis-python/tests/dpcontracts/__init__.py b/hypothesis-python/tests/dpcontracts/__init__.py index e61e3c2872..fcb1ac6538 100644 --- a/hypothesis-python/tests/dpcontracts/__init__.py +++ b/hypothesis-python/tests/dpcontracts/__init__.py @@ -1,14 +1,9 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER diff --git a/hypothesis-python/tests/dpcontracts/test_contracts.py b/hypothesis-python/tests/dpcontracts/test_contracts.py index 54e1e6a345..983438ad2c 100644 --- a/hypothesis-python/tests/dpcontracts/test_contracts.py +++ b/hypothesis-python/tests/dpcontracts/test_contracts.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest from dpcontracts import require diff --git a/hypothesis-python/tests/ghostwriter/recorded/division_binop_error_handler.txt b/hypothesis-python/tests/ghostwriter/recorded/division_binop_error_handler.txt new file mode 100644 index 0000000000..869a579892 --- /dev/null +++ b/hypothesis-python/tests/ghostwriter/recorded/division_binop_error_handler.txt @@ -0,0 +1,26 @@ +# This test code was written by the `hypothesis.extra.ghostwriter` module +# and is provided under the Creative Commons Zero public domain dedication. + +import test_expected_output +from hypothesis import given, strategies as st + +divide_operands = st.integers() + + +@given(a=divide_operands, b=divide_operands, c=divide_operands) +def test_associative_binary_operation_divide(a, b, c): + left = test_expected_output.divide(a=a, b=test_expected_output.divide(a=b, b=c)) + right = test_expected_output.divide(a=test_expected_output.divide(a=a, b=b), b=c) + assert left == right, (left, right) + + +@given(a=divide_operands, b=divide_operands) +def test_commutative_binary_operation_divide(a, b): + left = test_expected_output.divide(a=a, b=b) + right = test_expected_output.divide(a=b, b=a) + assert left == right, (left, right) + + +@given(a=divide_operands) +def test_identity_binary_operation_divide(a): + assert a == test_expected_output.divide(a=a, b=1) diff --git a/hypothesis-python/tests/ghostwriter/recorded/division_fuzz_error_handler.txt b/hypothesis-python/tests/ghostwriter/recorded/division_fuzz_error_handler.txt new file mode 100644 index 0000000000..43bf350d09 --- /dev/null +++ b/hypothesis-python/tests/ghostwriter/recorded/division_fuzz_error_handler.txt @@ -0,0 +1,13 @@ +# This test code was written by the `hypothesis.extra.ghostwriter` module +# and is provided under the Creative Commons Zero public domain dedication. + +import test_expected_output +from hypothesis import given, reject, strategies as st + + +@given(a=st.integers(), b=st.integers()) +def test_fuzz_divide(a, b): + try: + test_expected_output.divide(a=a, b=b) + except ZeroDivisionError: + reject() diff --git a/hypothesis-python/tests/ghostwriter/recorded/division_roundtrip_arithmeticerror_handler.txt b/hypothesis-python/tests/ghostwriter/recorded/division_roundtrip_arithmeticerror_handler.txt new file mode 100644 index 0000000000..825f64294e --- /dev/null +++ b/hypothesis-python/tests/ghostwriter/recorded/division_roundtrip_arithmeticerror_handler.txt @@ -0,0 +1,16 @@ +# This test code was written by the `hypothesis.extra.ghostwriter` module +# and is provided under the Creative Commons Zero public domain dedication. + +import _operator +import test_expected_output +from hypothesis import given, reject, strategies as st + + +@given(a=st.integers(), b=st.integers()) +def test_roundtrip_divide_mul(a, b): + try: + value0 = test_expected_output.divide(a=a, b=b) + value1 = _operator.mul(value0, b) + except ArithmeticError: + reject() + assert a == value1, (a, value1) diff --git a/hypothesis-python/tests/ghostwriter/recorded/division_roundtrip_error_handler.txt b/hypothesis-python/tests/ghostwriter/recorded/division_roundtrip_error_handler.txt new file mode 100644 index 0000000000..719d9067aa --- /dev/null +++ b/hypothesis-python/tests/ghostwriter/recorded/division_roundtrip_error_handler.txt @@ -0,0 +1,16 @@ +# This test code was written by the `hypothesis.extra.ghostwriter` module +# and is provided under the Creative Commons Zero public domain dedication. + +import _operator +import test_expected_output +from hypothesis import given, reject, strategies as st + + +@given(a=st.integers(), b=st.integers()) +def test_roundtrip_divide_mul(a, b): + try: + value0 = test_expected_output.divide(a=a, b=b) + except ZeroDivisionError: + reject() + value1 = _operator.mul(value0, b) + assert a == value1, (a, value1) diff --git a/hypothesis-python/tests/ghostwriter/recorded/division_roundtrip_typeerror_handler.txt b/hypothesis-python/tests/ghostwriter/recorded/division_roundtrip_typeerror_handler.txt new file mode 100644 index 0000000000..99c4ac40cd --- /dev/null +++ b/hypothesis-python/tests/ghostwriter/recorded/division_roundtrip_typeerror_handler.txt @@ -0,0 +1,19 @@ +# This test code was written by the `hypothesis.extra.ghostwriter` module +# and is provided under the Creative Commons Zero public domain dedication. + +import _operator +import test_expected_output +from hypothesis import given, reject, strategies as st + + +@given(a=st.integers(), b=st.integers()) +def test_roundtrip_divide_mul(a, b): + try: + try: + value0 = test_expected_output.divide(a=a, b=b) + except ZeroDivisionError: + reject() + value1 = _operator.mul(value0, b) + except TypeError: + reject() + assert a == value1, (a, value1) diff --git a/hypothesis-python/tests/ghostwriter/recorded/fuzz_staticmethod.txt b/hypothesis-python/tests/ghostwriter/recorded/fuzz_staticmethod.txt new file mode 100644 index 0000000000..1ae8f7920e --- /dev/null +++ b/hypothesis-python/tests/ghostwriter/recorded/fuzz_staticmethod.txt @@ -0,0 +1,10 @@ +# This test code was written by the `hypothesis.extra.ghostwriter` module +# and is provided under the Creative Commons Zero public domain dedication. + +import test_expected_output +from hypothesis import given, strategies as st + + +@given(arg=st.integers()) +def test_fuzz_A_Class_a_staticmethod(arg): + test_expected_output.A_Class.a_staticmethod(arg=arg) diff --git a/hypothesis-python/tests/ghostwriter/recorded/fuzz_with_docstring.txt b/hypothesis-python/tests/ghostwriter/recorded/fuzz_with_docstring.txt new file mode 100644 index 0000000000..29f60c8b19 --- /dev/null +++ b/hypothesis-python/tests/ghostwriter/recorded/fuzz_with_docstring.txt @@ -0,0 +1,16 @@ +# This test code was written by the `hypothesis.extra.ghostwriter` module +# and is provided under the Creative Commons Zero public domain dedication. + +import test_expected_output +from hypothesis import given, strategies as st + + +@given( + a=st.lists(st.integers()), + b=st.one_of(st.none(), st.builds(list), st.builds(tuple)), + c=st.sampled_from(["foo", "bar", None]), + d=st.just(int), + e=st.just(lambda x: f"xx{x}xx"), +) +def test_fuzz_with_docstring(a, b, c, d, e): + test_expected_output.with_docstring(a=a, b=b, c=c, d=d, e=e) diff --git a/hypothesis-python/tests/ghostwriter/recorded/hypothesis_module_magic.txt b/hypothesis-python/tests/ghostwriter/recorded/hypothesis_module_magic.txt index 043a0d844a..4b514590ca 100644 --- a/hypothesis-python/tests/ghostwriter/recorded/hypothesis_module_magic.txt +++ b/hypothesis-python/tests/ghostwriter/recorded/hypothesis_module_magic.txt @@ -4,6 +4,7 @@ import hypothesis import typing from hypothesis import given, settings, strategies as st +from hypothesis.strategies._internal.strategies import Ex from random import Random @@ -19,7 +20,7 @@ def test_fuzz_event(value): @given( specifier=st.from_type(hypothesis.strategies.SearchStrategy), - condition=st.functions(like=lambda *a: ..., returns=st.booleans()), + condition=st.functions(like=lambda *a, **k: None, returns=st.booleans()), settings=st.one_of(st.none(), st.builds(settings)), random=st.one_of(st.none(), st.builds(Random)), database_key=st.one_of(st.none(), st.binary()), @@ -95,6 +96,21 @@ def test_fuzz_settings( ) +@given(name=st.text()) +def test_fuzz_settings_get_profile(name): + hypothesis.settings.get_profile(name=name) + + +@given(name=st.text()) +def test_fuzz_settings_load_profile(name): + hypothesis.settings.load_profile(name=name) + + +@given(name=st.text(), parent=st.one_of(st.none(), st.builds(settings))) +def test_fuzz_settings_register_profile(name, parent): + hypothesis.settings.register_profile(name=name, parent=parent) + + @given(observation=st.one_of(st.floats(), st.integers()), label=st.text()) def test_fuzz_target(observation, label): hypothesis.target(observation=observation, label=label) diff --git a/hypothesis-python/tests/ghostwriter/recorded/magic_base64_roundtrip.txt b/hypothesis-python/tests/ghostwriter/recorded/magic_base64_roundtrip.txt new file mode 100644 index 0000000000..db1888544b --- /dev/null +++ b/hypothesis-python/tests/ghostwriter/recorded/magic_base64_roundtrip.txt @@ -0,0 +1,14 @@ +# This test code was written by the `hypothesis.extra.ghostwriter` module +# and is provided under the Creative Commons Zero public domain dedication. + +import base64 +from hypothesis import given, strategies as st + +# TODO: replace st.nothing() with an appropriate strategy + + +@given(altchars=st.none(), s=st.nothing(), validate=st.booleans()) +def test_roundtrip_b64encode_b64decode(altchars, s, validate): + value0 = base64.b64encode(s=s, altchars=altchars) + value1 = base64.b64decode(s=value0, altchars=altchars, validate=validate) + assert s == value1, (s, value1) diff --git a/hypothesis-python/tests/ghostwriter/recorded/magic_builtins.txt b/hypothesis-python/tests/ghostwriter/recorded/magic_builtins.txt new file mode 100644 index 0000000000..767eb421dd --- /dev/null +++ b/hypothesis-python/tests/ghostwriter/recorded/magic_builtins.txt @@ -0,0 +1,306 @@ +# This test code was written by the `hypothesis.extra.ghostwriter` module +# and is provided under the Creative Commons Zero public domain dedication. + +from hypothesis import given, strategies as st + +# TODO: replace st.nothing() with appropriate strategies + + +@given(x=st.nothing()) +def test_fuzz_abs(x): + abs(x) + + +@given(iterable=st.one_of(st.iterables(st.integers()), st.iterables(st.text()))) +def test_fuzz_all(iterable): + all(iterable) + + +@given(iterable=st.one_of(st.iterables(st.integers()), st.iterables(st.text()))) +def test_fuzz_any(iterable): + any(iterable) + + +@given(obj=st.nothing()) +def test_fuzz_ascii(obj): + ascii(obj) + + +@given(number=st.one_of(st.integers(), st.floats())) +def test_fuzz_bin(number): + bin(number) + + +@given(frm=st.nothing(), to=st.nothing()) +def test_fuzz_bytearray_maketrans(frm, to): + bytearray.maketrans(frm, to) + + +@given(frm=st.nothing(), to=st.nothing()) +def test_fuzz_bytes_maketrans(frm, to): + bytes.maketrans(frm, to) + + +@given(obj=st.nothing()) +def test_fuzz_callable(obj): + callable(obj) + + +@given(i=st.nothing()) +def test_fuzz_chr(i): + chr(i) + + +@given( + source=st.nothing(), + filename=st.nothing(), + mode=st.nothing(), + flags=st.just(0), + dont_inherit=st.booleans(), + optimize=st.just(-1), + _feature_version=st.just(-1), +) +def test_fuzz_compile( + source, filename, mode, flags, dont_inherit, optimize, _feature_version +): + compile( + source=source, + filename=filename, + mode=mode, + flags=flags, + dont_inherit=dont_inherit, + optimize=optimize, + _feature_version=_feature_version, + ) + + +@given(real=st.just(0), imag=st.just(0)) +def test_fuzz_complex(real, imag): + complex(real=real, imag=imag) + + +@given(obj=st.nothing(), name=st.text()) +def test_fuzz_delattr(obj, name): + delattr(obj, name) + + +@given(object=st.builds(object)) +def test_fuzz_dir(object): + dir(object) + + +@given(x=st.nothing(), y=st.nothing()) +def test_fuzz_divmod(x, y): + divmod(x, y) + + +@given( + iterable=st.one_of(st.iterables(st.integers()), st.iterables(st.text())), + start=st.just(0), +) +def test_fuzz_enumerate(iterable, start): + enumerate(iterable=iterable, start=start) + + +@given(source=st.nothing(), globals=st.none(), locals=st.none()) +def test_fuzz_eval(source, globals, locals): + eval(source, globals, locals) + + +@given(source=st.nothing(), globals=st.none(), locals=st.none()) +def test_fuzz_exec(source, globals, locals): + exec(source, globals, locals) + + +@given(x=st.just(0)) +def test_fuzz_float(x): + float(x) + + +@given(value=st.nothing(), format_spec=st.just("")) +def test_fuzz_format(value, format_spec): + format(value, format_spec) + + +@given(object=st.builds(object), name=st.text(), default=st.nothing()) +def test_fuzz_getattr(object, name, default): + getattr(object, name, default) + + +@given(obj=st.nothing(), name=st.text()) +def test_fuzz_hasattr(obj, name): + hasattr(obj, name) + + +@given(obj=st.nothing()) +def test_fuzz_hash(obj): + hash(obj) + + +@given(number=st.one_of(st.integers(), st.floats())) +def test_fuzz_hex(number): + hex(number) + + +@given(obj=st.nothing()) +def test_fuzz_id(obj): + id(obj) + + +@given(prompt=st.none()) +def test_fuzz_input(prompt): + input(prompt) + + +@given(obj=st.nothing(), class_or_tuple=st.nothing()) +def test_fuzz_isinstance(obj, class_or_tuple): + isinstance(obj, class_or_tuple) + + +@given(cls=st.nothing(), class_or_tuple=st.nothing()) +def test_fuzz_issubclass(cls, class_or_tuple): + issubclass(cls, class_or_tuple) + + +@given(iterable=st.one_of(st.iterables(st.integers()), st.iterables(st.text()))) +def test_fuzz_iter(iterable): + iter(iterable) + + +@given(obj=st.nothing()) +def test_fuzz_len(obj): + len(obj) + + +@given(iterable=st.just(())) +def test_fuzz_list(iterable): + list(iterable) + + +@given( + iterable=st.one_of(st.iterables(st.integers()), st.iterables(st.text())), + default=st.nothing(), + key=st.nothing(), +) +def test_fuzz_max(iterable, default, key): + max(iterable, default=default, key=key) + + +@given(object=st.builds(object)) +def test_fuzz_memoryview(object): + memoryview(object=object) + + +@given( + iterable=st.one_of(st.iterables(st.integers()), st.iterables(st.text())), + default=st.nothing(), + key=st.nothing(), +) +def test_fuzz_min(iterable, default, key): + min(iterable, default=default, key=key) + + +@given(iterator=st.nothing(), default=st.nothing()) +def test_fuzz_next(iterator, default): + next(iterator, default) + + +@given(number=st.one_of(st.integers(), st.floats())) +def test_fuzz_oct(number): + oct(number) + + +@given( + file=st.nothing(), + mode=st.just("r"), + buffering=st.just(-1), + encoding=st.none(), + errors=st.none(), + newline=st.none(), + closefd=st.booleans(), + opener=st.none(), +) +def test_fuzz_open(file, mode, buffering, encoding, errors, newline, closefd, opener): + open( + file=file, + mode=mode, + buffering=buffering, + encoding=encoding, + errors=errors, + newline=newline, + closefd=closefd, + opener=opener, + ) + + +@given(c=st.nothing()) +def test_fuzz_ord(c): + ord(c) + + +@given(base=st.nothing(), exp=st.nothing(), mod=st.none()) +def test_fuzz_pow(base, exp, mod): + pow(base=base, exp=exp, mod=mod) + + +@given( + value=st.nothing(), + sep=st.text(), + end=st.nothing(), + file=st.nothing(), + flush=st.nothing(), +) +def test_fuzz_print(value, sep, end, file, flush): + print(value, sep=sep, end=end, file=file, flush=flush) + + +@given(fget=st.none(), fset=st.none(), fdel=st.none(), doc=st.none()) +def test_fuzz_property(fget, fset, fdel, doc): + property(fget=fget, fset=fset, fdel=fdel, doc=doc) + + +@given(obj=st.nothing()) +def test_fuzz_repr(obj): + repr(obj) + + +@given(sequence=st.nothing()) +def test_fuzz_reversed(sequence): + reversed(sequence) + + +@given(number=st.one_of(st.integers(), st.floats()), ndigits=st.none()) +def test_fuzz_round(number, ndigits): + round(number=number, ndigits=ndigits) + + +@given(obj=st.nothing(), name=st.text(), value=st.nothing()) +def test_fuzz_setattr(obj, name, value): + setattr(obj, name, value) + + +@given( + iterable=st.one_of(st.iterables(st.integers()), st.iterables(st.text())), + key=st.none(), + reverse=st.booleans(), +) +def test_fuzz_sorted(iterable, key, reverse): + sorted(iterable, key=key, reverse=reverse) + + +@given( + iterable=st.one_of(st.iterables(st.integers()), st.iterables(st.text())), + start=st.just(0), +) +def test_fuzz_sum(iterable, start): + sum(iterable, start=start) + + +@given(iterable=st.just(())) +def test_fuzz_tuple(iterable): + tuple(iterable) + + +@given(object=st.builds(object)) +def test_fuzz_vars(object): + vars(object) diff --git a/hypothesis-python/tests/ghostwriter/recorded/magic_class.txt b/hypothesis-python/tests/ghostwriter/recorded/magic_class.txt new file mode 100644 index 0000000000..ad2c6302f5 --- /dev/null +++ b/hypothesis-python/tests/ghostwriter/recorded/magic_class.txt @@ -0,0 +1,20 @@ +# This test code was written by the `hypothesis.extra.ghostwriter` module +# and is provided under the Creative Commons Zero public domain dedication. + +import test_expected_output +from hypothesis import given, strategies as st + + +@given() +def test_fuzz_A_Class(): + test_expected_output.A_Class() + + +@given(arg=st.integers()) +def test_fuzz_A_Class_a_classmethod(arg): + test_expected_output.A_Class.a_classmethod(arg=arg) + + +@given(arg=st.integers()) +def test_fuzz_A_Class_a_staticmethod(arg): + test_expected_output.A_Class.a_staticmethod(arg=arg) diff --git a/hypothesis-python/tests/ghostwriter/recorded/magic_gufunc.txt b/hypothesis-python/tests/ghostwriter/recorded/magic_gufunc.txt index 0bb6a4e1ab..e264046a71 100644 --- a/hypothesis-python/tests/ghostwriter/recorded/magic_gufunc.txt +++ b/hypothesis-python/tests/ghostwriter/recorded/magic_gufunc.txt @@ -1,22 +1,25 @@ # This test code was written by the `hypothesis.extra.ghostwriter` module # and is provided under the Creative Commons Zero public domain dedication. -import hypothesis.extra.numpy as npst import numpy from hypothesis import given, strategies as st +from hypothesis.extra.numpy import arrays, mutually_broadcastable_shapes @given( data=st.data(), - shapes=npst.mutually_broadcastable_shapes(signature="(n?,k),(k,m?)->(n?,m?)"), - types=st.sampled_from(numpy.matmul.types).filter(lambda sig: "O" not in sig), + shapes=mutually_broadcastable_shapes(signature="(n?,k),(k,m?)->(n?,m?)"), + types=st.sampled_from([sig for sig in numpy.matmul.types if "O" not in sig]), ) def test_gufunc_matmul(data, shapes, types): input_shapes, expected_shape = shapes input_dtypes, expected_dtype = types.split("->") - array_st = [npst.arrays(d, s) for d, s in zip(input_dtypes, input_shapes)] + array_strats = [ + arrays(dtype=dtp, shape=shp, elements={"allow_nan": True}) + for dtp, shp in zip(input_dtypes, input_shapes) + ] - a, b = data.draw(st.tuples(*array_st)) + a, b = data.draw(st.tuples(*array_strats)) result = numpy.matmul(a, b) assert result.shape == expected_shape diff --git a/hypothesis-python/tests/ghostwriter/recorded/re_compile.txt b/hypothesis-python/tests/ghostwriter/recorded/re_compile.txt index 81497e6c3c..a8b943ceee 100644 --- a/hypothesis-python/tests/ghostwriter/recorded/re_compile.txt +++ b/hypothesis-python/tests/ghostwriter/recorded/re_compile.txt @@ -4,9 +4,7 @@ import re from hypothesis import given, strategies as st -# TODO: replace st.nothing() with an appropriate strategy - -@given(pattern=st.nothing(), flags=st.just(0)) +@given(pattern=st.text(), flags=st.just(0)) def test_fuzz_compile(pattern, flags): re.compile(pattern=pattern, flags=flags) diff --git a/hypothesis-python/tests/ghostwriter/recorded/re_compile_except.txt b/hypothesis-python/tests/ghostwriter/recorded/re_compile_except.txt index e266fc8f00..fed9c0f3d4 100644 --- a/hypothesis-python/tests/ghostwriter/recorded/re_compile_except.txt +++ b/hypothesis-python/tests/ghostwriter/recorded/re_compile_except.txt @@ -4,10 +4,8 @@ import re from hypothesis import given, reject, strategies as st -# TODO: replace st.nothing() with an appropriate strategy - -@given(pattern=st.nothing(), flags=st.just(0)) +@given(pattern=st.text(), flags=st.just(0)) def test_fuzz_compile(pattern, flags): try: re.compile(pattern=pattern, flags=flags) diff --git a/hypothesis-python/tests/ghostwriter/recorded/re_compile_unittest.txt b/hypothesis-python/tests/ghostwriter/recorded/re_compile_unittest.txt index 396190b716..955fec7456 100644 --- a/hypothesis-python/tests/ghostwriter/recorded/re_compile_unittest.txt +++ b/hypothesis-python/tests/ghostwriter/recorded/re_compile_unittest.txt @@ -5,10 +5,8 @@ import re import unittest from hypothesis import given, strategies as st -# TODO: replace st.nothing() with an appropriate strategy - class TestFuzzCompile(unittest.TestCase): - @given(pattern=st.nothing(), flags=st.just(0)) + @given(pattern=st.text(), flags=st.just(0)) def test_fuzz_compile(self, pattern, flags): re.compile(pattern=pattern, flags=flags) diff --git a/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_1error.txt b/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_1error.txt new file mode 100644 index 0000000000..eea2a26513 --- /dev/null +++ b/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_1error.txt @@ -0,0 +1,28 @@ +# This test code was written by the `hypothesis.extra.ghostwriter` module +# and is provided under the Creative Commons Zero public domain dedication. + +import pytest +from hypothesis import given, reject, strategies as st, target + + +@given( + iterable=st.one_of(st.iterables(st.integers()), st.iterables(st.text())), + key=st.none(), + reverse=st.booleans(), +) +def test_equivalent_sorted_sorted(iterable, key, reverse): + try: + result_0_sorted = sorted(iterable, key=key, reverse=reverse) + exc_type = None + target(1, label="input was valid") + except ValueError: + reject() + except Exception as exc: + exc_type = type(exc) + + if exc_type: + with pytest.raises(exc_type): + sorted(iterable, key=key, reverse=reverse) + else: + result_1_sorted = sorted(iterable, key=key, reverse=reverse) + assert result_0_sorted == result_1_sorted, (result_0_sorted, result_1_sorted) diff --git a/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_2error_unittest.txt b/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_2error_unittest.txt new file mode 100644 index 0000000000..ce5086b083 --- /dev/null +++ b/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_2error_unittest.txt @@ -0,0 +1,29 @@ +# This test code was written by the `hypothesis.extra.ghostwriter` module +# and is provided under the Creative Commons Zero public domain dedication. + +import unittest +from hypothesis import given, reject, strategies as st, target + + +class TestEquivalentSortedSorted(unittest.TestCase): + @given( + iterable=st.one_of(st.iterables(st.integers()), st.iterables(st.text())), + key=st.none(), + reverse=st.booleans(), + ) + def test_equivalent_sorted_sorted(self, iterable, key, reverse): + try: + result_0_sorted = sorted(iterable, key=key, reverse=reverse) + exc_type = None + target(1, label="input was valid") + except (TypeError, ValueError): + reject() + except Exception as exc: + exc_type = type(exc) + + if exc_type: + with self.assertRaises(exc_type): + sorted(iterable, key=key, reverse=reverse) + else: + result_1_sorted = sorted(iterable, key=key, reverse=reverse) + self.assertEqual(result_0_sorted, result_1_sorted) diff --git a/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_simple.txt b/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_simple.txt new file mode 100644 index 0000000000..254964dd2e --- /dev/null +++ b/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_simple.txt @@ -0,0 +1,26 @@ +# This test code was written by the `hypothesis.extra.ghostwriter` module +# and is provided under the Creative Commons Zero public domain dedication. + +import pytest +from hypothesis import given, strategies as st, target + + +@given( + iterable=st.one_of(st.iterables(st.integers()), st.iterables(st.text())), + key=st.none(), + reverse=st.booleans(), +) +def test_equivalent_sorted_sorted(iterable, key, reverse): + try: + result_0_sorted = sorted(iterable, key=key, reverse=reverse) + exc_type = None + target(1, label="input was valid") + except Exception as exc: + exc_type = type(exc) + + if exc_type: + with pytest.raises(exc_type): + sorted(iterable, key=key, reverse=reverse) + else: + result_1_sorted = sorted(iterable, key=key, reverse=reverse) + assert result_0_sorted == result_1_sorted, (result_0_sorted, result_1_sorted) diff --git a/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_threefuncs.txt b/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_threefuncs.txt new file mode 100644 index 0000000000..34c7b05ef5 --- /dev/null +++ b/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_threefuncs.txt @@ -0,0 +1,33 @@ +# This test code was written by the `hypothesis.extra.ghostwriter` module +# and is provided under the Creative Commons Zero public domain dedication. + +import pytest +from hypothesis import given, strategies as st, target + + +@given( + iterable=st.one_of(st.iterables(st.integers()), st.iterables(st.text())), + key=st.none(), + reverse=st.booleans(), +) +def test_equivalent_sorted_sorted_sorted(iterable, key, reverse): + try: + result_0_sorted = sorted(iterable, key=key, reverse=reverse) + exc_type = None + target(1, label="input was valid") + except Exception as exc: + exc_type = type(exc) + + if exc_type: + with pytest.raises(exc_type): + sorted(iterable, key=key, reverse=reverse) + else: + result_1_sorted = sorted(iterable, key=key, reverse=reverse) + assert result_0_sorted == result_1_sorted, (result_0_sorted, result_1_sorted) + + if exc_type: + with pytest.raises(exc_type): + sorted(iterable, key=key, reverse=reverse) + else: + result_2_sorted = sorted(iterable, key=key, reverse=reverse) + assert result_0_sorted == result_2_sorted, (result_0_sorted, result_2_sorted) diff --git a/hypothesis-python/tests/ghostwriter/recorded/timsort_idempotent_asserts.txt b/hypothesis-python/tests/ghostwriter/recorded/timsort_idempotent_asserts.txt new file mode 100644 index 0000000000..a3c6c1d5be --- /dev/null +++ b/hypothesis-python/tests/ghostwriter/recorded/timsort_idempotent_asserts.txt @@ -0,0 +1,15 @@ +# This test code was written by the `hypothesis.extra.ghostwriter` module +# and is provided under the Creative Commons Zero public domain dedication. + +import test_expected_output +from hypothesis import given, reject, strategies as st + + +@given(seq=st.one_of(st.binary(), st.lists(st.integers()))) +def test_idempotent_timsort(seq): + try: + result = test_expected_output.timsort(seq=seq) + repeat = test_expected_output.timsort(seq=result) + except AssertionError: + reject() + assert result == repeat, (result, repeat) diff --git a/hypothesis-python/tests/ghostwriter/test_expected_output.py b/hypothesis-python/tests/ghostwriter/test_expected_output.py index 6cc9294e32..f694e86c07 100644 --- a/hypothesis-python/tests/ghostwriter/test_expected_output.py +++ b/hypothesis-python/tests/ghostwriter/test_expected_output.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """ 'Golden master' tests for the ghostwriter. @@ -21,6 +16,7 @@ import ast import base64 +import builtins import operator import pathlib import re @@ -51,25 +47,53 @@ def timsort(seq: Sequence[int]) -> Sequence[int]: return sorted(seq) +def with_docstring(a, b, c, d=int, e=lambda x: f"xx{x}xx") -> None: + """Demonstrates parsing params from the docstring + + :param a: sphinx docstring style + :type a: sequence of integers + + b (list, tuple, or None): Google docstring style + + c : {"foo", "bar", or None} + Numpy docstring style + """ + + class A_Class: @classmethod def a_classmethod(cls, arg: int): pass + @staticmethod + def a_staticmethod(arg: int): + pass + def add(a: float, b: float) -> float: return a + b +def divide(a: int, b: int) -> float: + """This is a RST-style docstring for `divide`. + + :raises ZeroDivisionError: if b == 0 + """ + return a / b + + # Note: for some of the `expected` outputs, we replace away some small # parts which vary between minor versions of Python. @pytest.mark.parametrize( "data", [ ("fuzz_sorted", ghostwriter.fuzz(sorted)), + ("fuzz_with_docstring", ghostwriter.fuzz(with_docstring)), ("fuzz_classmethod", ghostwriter.fuzz(A_Class.a_classmethod)), + ("fuzz_staticmethod", ghostwriter.fuzz(A_Class.a_staticmethod)), ("fuzz_ufunc", ghostwriter.fuzz(numpy.add)), ("magic_gufunc", ghostwriter.magic(numpy.matmul)), + ("magic_base64_roundtrip", ghostwriter.magic(base64.b64encode)), ("re_compile", ghostwriter.fuzz(re.compile)), ( "re_compile_except", @@ -78,13 +102,37 @@ def add(a: float, b: float) -> float: .replace("import sre_constants\n", "").replace("sre_constants.", "re."), ), ("re_compile_unittest", ghostwriter.fuzz(re.compile, style="unittest")), - ("base64_magic", ghostwriter.magic(base64)), + pytest.param( + ("base64_magic", ghostwriter.magic(base64)), + marks=pytest.mark.skipif("sys.version_info[:2] >= (3, 10)"), + ), ("sorted_idempotent", ghostwriter.idempotent(sorted)), ("timsort_idempotent", ghostwriter.idempotent(timsort)), + ( + "timsort_idempotent_asserts", + ghostwriter.idempotent(timsort, except_=AssertionError), + ), ("eval_equivalent", ghostwriter.equivalent(eval, ast.literal_eval)), ("sorted_self_equivalent", ghostwriter.equivalent(sorted, sorted, sorted)), ("addition_op_magic", ghostwriter.magic(add)), ("addition_op_multimagic", ghostwriter.magic(add, operator.add, numpy.add)), + ("division_fuzz_error_handler", ghostwriter.fuzz(divide)), + ( + "division_binop_error_handler", + ghostwriter.binary_operation(divide, identity=1), + ), + ( + "division_roundtrip_error_handler", + ghostwriter.roundtrip(divide, operator.mul), + ), + ( + "division_roundtrip_arithmeticerror_handler", + ghostwriter.roundtrip(divide, operator.mul, except_=ArithmeticError), + ), + ( + "division_roundtrip_typeerror_handler", + ghostwriter.roundtrip(divide, operator.mul, except_=TypeError), + ), ( "division_operator", ghostwriter.binary_operation( @@ -106,6 +154,43 @@ def add(a: float, b: float) -> float: style="unittest", ), ), + ( + "sorted_self_error_equivalent_simple", + ghostwriter.equivalent(sorted, sorted, allow_same_errors=True), + ), + ( + "sorted_self_error_equivalent_threefuncs", + ghostwriter.equivalent(sorted, sorted, sorted, allow_same_errors=True), + ), + ( + "sorted_self_error_equivalent_1error", + ghostwriter.equivalent( + sorted, + sorted, + allow_same_errors=True, + except_=ValueError, + ), + ), + ( + "sorted_self_error_equivalent_2error_unittest", + ghostwriter.equivalent( + sorted, + sorted, + allow_same_errors=True, + except_=(TypeError, ValueError), + style="unittest", + ), + ), + ("magic_class", ghostwriter.magic(A_Class)), + pytest.param( + ("magic_builtins", ghostwriter.magic(builtins)), + marks=[ + pytest.mark.skipif( + sys.version_info[:2] not in [(3, 8), (3, 9)], + reason="compile arg new in 3.8, aiter and anext new in 3.10", + ) + ], + ), ], ids=lambda x: x[0], ) @@ -119,8 +204,6 @@ def test_ghostwriter_example_outputs(update_recorded_outputs, data): def test_ghostwriter_on_hypothesis(update_recorded_outputs): actual = ghostwriter.magic(hypothesis).replace("Strategy[+Ex]", "Strategy") expected = get_recorded("hypothesis_module_magic", actual * update_recorded_outputs) - # The py36 typing module has some different handling of generics (SearchStrategy) - # and contents (collections.abc vs typing), but we can still check the code works. - if sys.version_info[:2] > (3, 6): + if sys.version_info[:2] < (3, 10): assert actual == expected exec(expected, {"not_set": not_set}) diff --git a/hypothesis-python/tests/ghostwriter/test_ghostwriter.py b/hypothesis-python/tests/ghostwriter/test_ghostwriter.py index c50b6b74e7..93b13c2e23 100644 --- a/hypothesis-python/tests/ghostwriter/test_ghostwriter.py +++ b/hypothesis-python/tests/ghostwriter/test_ghostwriter.py @@ -1,33 +1,47 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import ast import enum import json import re +import socket import unittest import unittest.mock from decimal import Decimal -from types import ModuleType -from typing import Any, List, Sequence, Set, Union +from pathlib import Path +from textwrap import dedent +from types import FunctionType, ModuleType +from typing import ( + Any, + FrozenSet, + KeysView, + List, + Match, + Pattern, + Sequence, + Set, + Sized, + Union, + ValuesView, +) +import attr +import click import pytest -from hypothesis.errors import InvalidArgument, MultipleFailures, Unsatisfiable -from hypothesis.extra import ghostwriter -from hypothesis.strategies import builds, from_type, just +from hypothesis import assume +from hypothesis.errors import InvalidArgument, Unsatisfiable +from hypothesis.extra import cli, ghostwriter +from hypothesis.internal.compat import BaseExceptionGroup +from hypothesis.strategies import builds, from_type, just, lists from hypothesis.strategies._internal.lazy import LazyStrategy varied_excepts = pytest.mark.parametrize("ex", [(), ValueError, (TypeError, re.error)]) @@ -38,7 +52,11 @@ def get_test_function(source_code): # Note that this also tests that the module is syntatically-valid, # AND free from undefined names, import problems, and so on. namespace = {} - exec(source_code, namespace) + try: + exec(source_code, namespace) + except Exception: + print(f"************\n{source_code}\n************") + raise tests = [ v for k, v in namespace.items() @@ -123,6 +141,39 @@ def test_flattens_one_of_repr(): assert ghostwriter._valid_syntax_repr(strat)[1].count("one_of(") == 1 +def takes_keys(x: KeysView[int]) -> None: + pass + + +def takes_values(x: ValuesView[int]) -> None: + pass + + +def takes_match(x: Match[bytes]) -> None: + pass + + +def takes_pattern(x: Pattern[str]) -> None: + pass + + +def takes_sized(x: Sized) -> None: + pass + + +def takes_frozensets(a: FrozenSet[int], b: FrozenSet[int]) -> None: + pass + + +@attr.s() +class Foo: + foo: str = attr.ib() + + +def takes_attrs_class(x: Foo) -> None: + pass + + @varied_excepts @pytest.mark.parametrize( "func", @@ -136,6 +187,13 @@ def test_flattens_one_of_repr(): annotated_any, space_in_name, non_resolvable_arg, + takes_keys, + takes_values, + takes_match, + takes_pattern, + takes_sized, + takes_frozensets, + takes_attrs_class, ], ) def test_ghostwriter_fuzz(func, ex): @@ -143,6 +201,18 @@ def test_ghostwriter_fuzz(func, ex): get_test_function(source_code) +def test_socket_module(): + source_code = ghostwriter.magic(socket) + exec(source_code, {}) + + +def test_binary_op_also_handles_frozensets(): + # Using str.replace in a loop would convert `frozensets()` into + # `st.frozenst.sets()` instead of `st.frozensets()`; fixed with re.sub. + source_code = ghostwriter.binary_operation(takes_frozensets) + exec(source_code, {}) + + @varied_excepts @pytest.mark.parametrize( "func", [re.compile, json.loads, json.dump, timsort, ast.literal_eval] @@ -192,6 +262,34 @@ def test_invalid_func_inputs(gw, args): gw(*args) +class A: + @classmethod + def to_json(cls, obj: Union[dict, list]) -> str: + return json.dumps(obj) + + @classmethod + def from_json(cls, obj: str) -> Union[dict, list]: + return json.loads(obj) + + @staticmethod + def static_sorter(seq: Sequence[int]) -> List[int]: + return sorted(seq) + + +@pytest.mark.parametrize( + "gw,args", + [ + (ghostwriter.fuzz, [A.static_sorter]), + (ghostwriter.idempotent, [A.static_sorter]), + (ghostwriter.roundtrip, [A.to_json, A.from_json]), + (ghostwriter.equivalent, [A.to_json, json.dumps]), + ], +) +def test_class_methods_inputs(gw, args): + source_code = gw(*args) + get_test_function(source_code)() + + def test_run_ghostwriter_fuzz(): # Our strategy-guessing code works for all the arguments to sorted, # and we handle positional-only arguments in calls correctly too. @@ -240,7 +338,7 @@ def test_run_ghostwriter_roundtrip(): ) try: get_test_function(source_code)() - except (AssertionError, ValueError, MultipleFailures): + except (AssertionError, ValueError, BaseExceptionGroup): pass # Finally, restricting ourselves to finite floats makes the test pass! @@ -323,7 +421,88 @@ def test_unrepr_identity_elem(): builds(enum.Enum, builds(Decimal), kw=builds(re.compile)), {("enum", "Enum"), ("decimal", "Decimal"), ("re", "compile")}, ), + # lists recurse on imports + ( + lists(builds(Decimal)), + {("decimal", "Decimal")}, + ), ], ) def test_get_imports_for_strategy(strategy, imports): assert ghostwriter._imports_for_strategy(strategy) == imports + + +@pytest.fixture +def temp_script_file(): + """Fixture to yield a Path to a temporary file in the local directory. File name will end + in .py and will include an importable function. + """ + p = Path("my_temp_script.py") + if p.exists(): + raise FileExistsError(f"Did not expect {p} to exist during testing") + p.write_text( + dedent( + """ + def say_hello(): + print("Hello world!") + """ + ) + ) + yield p + p.unlink() + + +@pytest.fixture +def temp_script_file_with_py_function(): + """Fixture to yield a Path to a temporary file in the local directory. File name will end + in .py and will include an importable function named "py" + """ + p = Path("my_temp_script_with_py_function.py") + if p.exists(): + raise FileExistsError(f"Did not expect {p} to exist during testing") + p.write_text( + dedent( + """ + def py(): + print('A function named "py" has been called') + """ + ) + ) + yield p + p.unlink() + + +def test_obj_name(temp_script_file, temp_script_file_with_py_function): + # Module paths (strings including a "/") should raise a meaningful UsageError + with pytest.raises(click.exceptions.UsageError) as e: + cli.obj_name("mydirectory/myscript.py") + assert e.match( + "Remember that the ghostwriter should be passed the name of a module, not a path." + ) + # Windows paths (strings including a "\") should also raise a meaningful UsageError + with pytest.raises(click.exceptions.UsageError) as e: + cli.obj_name("mydirectory\\myscript.py") + assert e.match( + "Remember that the ghostwriter should be passed the name of a module, not a path." + ) + # File names of modules (strings ending in ".py") should raise a meaningful UsageError + with pytest.raises(click.exceptions.UsageError) as e: + cli.obj_name("myscript.py") + assert e.match( + "Remember that the ghostwriter should be passed the name of a module, not a file." + ) + # File names of modules (strings ending in ".py") that exist should get a suggestion + with pytest.raises(click.exceptions.UsageError) as e: + cli.obj_name(str(temp_script_file)) + assert e.match( + "Remember that the ghostwriter should be passed the name of a module, not a file." + + f"\n\tTry: hypothesis write {temp_script_file.stem}" + ) + # File names of modules (strings ending in ".py") that define a py function should succeed + assert isinstance( + cli.obj_name(str(temp_script_file_with_py_function)), FunctionType + ) + + +def test_gets_public_location_not_impl_location(): + assert ghostwriter._get_module(assume) == "hypothesis" # not "hypothesis.control" diff --git a/hypothesis-python/tests/ghostwriter/test_ghostwriter_cli.py b/hypothesis-python/tests/ghostwriter/test_ghostwriter_cli.py index 8676d7ba11..dddd434a57 100644 --- a/hypothesis-python/tests/ghostwriter/test_ghostwriter_cli.py +++ b/hypothesis-python/tests/ghostwriter/test_ghostwriter_cli.py @@ -1,27 +1,30 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import ast +import itertools import json +import operator import re import subprocess import pytest from hypothesis.errors import StopTest -from hypothesis.extra.ghostwriter import equivalent, fuzz, idempotent, roundtrip +from hypothesis.extra.ghostwriter import ( + binary_operation, + equivalent, + fuzz, + idempotent, + roundtrip, +) @pytest.mark.parametrize( @@ -41,21 +44,24 @@ ), # Imports submodule (importlib.import_module passes; __import__ fails) ("hypothesis.errors.StopTest", lambda: fuzz(StopTest)), + # We can write tests for classes even without classmethods or staticmethods + ("hypothesis.errors.StopTest", lambda: fuzz(StopTest)), + # Search for identity element does not print e.g. "You can use @seed ..." + ("--binary-op operator.add", lambda: binary_operation(operator.add)), ], ) def test_cli_python_equivalence(cli, code): result = subprocess.run( "hypothesis write " + cli, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, + capture_output=True, shell=True, - universal_newlines=True, + text=True, ) + result.check_returncode() cli_output = result.stdout.strip() assert not result.stderr code_output = code().strip() assert code_output == cli_output - result.check_returncode() @pytest.mark.parametrize( @@ -83,10 +89,9 @@ def test_cli_too_many_functions(cli, err_msg): # Supplying multiple functions to writers that only cope with one result = subprocess.run( "hypothesis write " + cli, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, + capture_output=True, shell=True, - universal_newlines=True, + text=True, ) assert result.returncode == 2 assert "Error: " + err_msg in result.stderr @@ -105,24 +110,144 @@ def test_can_import_from_scripts_in_working_dir(tmpdir): (tmpdir / "mycode.py").write(CODE_TO_TEST) result = subprocess.run( "hypothesis write mycode.sorter", - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, + capture_output=True, + shell=True, + text=True, + cwd=tmpdir, + ) + assert result.returncode == 0 + assert "Error: " not in result.stderr + + +CLASS_CODE_TO_TEST = """ +from typing import Sequence, List + +def my_func(seq: Sequence[int]) -> List[int]: + return sorted(seq) + +class MyClass: + + @staticmethod + def my_staticmethod(seq: Sequence[int]) -> List[int]: + return sorted(seq) + + @classmethod + def my_classmethod(cls, seq: Sequence[int]) -> List[int]: + return sorted(seq) +""" + + +@pytest.mark.parametrize("func", ["my_staticmethod", "my_classmethod"]) +def test_can_import_from_class(tmpdir, func): + (tmpdir / "mycode.py").write(CLASS_CODE_TO_TEST) + result = subprocess.run( + f"hypothesis write mycode.MyClass.{func}", + capture_output=True, shell=True, - universal_newlines=True, + text=True, cwd=tmpdir, ) assert result.returncode == 0 assert "Error: " not in result.stderr +@pytest.mark.parametrize( + "classname,thing,kind", + [ + ("XX", "", "class"), + ("MyClass", " and 'MyClass' class", "attribute"), + ("my_func", " and 'my_func' attribute", "attribute"), + ], +) +def test_error_import_from_class(tmpdir, classname, thing, kind): + (tmpdir / "mycode.py").write(CLASS_CODE_TO_TEST) + result = subprocess.run( + f"hypothesis write mycode.{classname}.XX", + capture_output=True, + shell=True, + text=True, + cwd=tmpdir, + ) + msg = f"Error: Found the 'mycode' module{thing}, but it doesn't have a 'XX' {kind}." + assert result.returncode == 2 + assert msg in result.stderr + + +def test_magic_discovery_from_module(tmpdir): + (tmpdir / "mycode.py").write(CLASS_CODE_TO_TEST) + result = subprocess.run( + f"hypothesis write mycode", + capture_output=True, + shell=True, + text=True, + cwd=tmpdir, + ) + assert result.returncode == 0 + assert "my_func" in result.stdout + assert "MyClass.my_staticmethod" in result.stdout + assert "MyClass.my_classmethod" in result.stdout + + +ROUNDTRIP_CODE_TO_TEST = """ +from typing import Union +import json + +def to_json(json: Union[dict,list]) -> str: + return json.dumps(json) + +def from_json(json: str) -> Union[dict,list]: + return json.loads(json) + +class MyClass: + + @staticmethod + def to_json(json: Union[dict,list]) -> str: + return json.dumps(json) + + @staticmethod + def from_json(json: str) -> Union[dict,list]: + return json.loads(json) + +class OtherClass: + + @classmethod + def to_json(cls, json: Union[dict,list]) -> str: + return json.dumps(json) + + @classmethod + def from_json(cls, json: str) -> Union[dict,list]: + return json.loads(json) +""" + + +def test_roundtrip_correct_pairs(tmpdir): + (tmpdir / "mycode.py").write(ROUNDTRIP_CODE_TO_TEST) + result = subprocess.run( + f"hypothesis write mycode", + capture_output=True, + shell=True, + text=True, + cwd=tmpdir, + ) + assert result.returncode == 0 + for scope1, scope2 in itertools.product( + ["mycode.MyClass", "mycode.OtherClass", "mycode"], repeat=2 + ): + round_trip_code = f"""value0 = {scope1}.to_json(json=json) + value1 = {scope2}.from_json(json=value0)""" + if scope1 == scope2: + assert round_trip_code in result.stdout + else: + assert round_trip_code not in result.stdout + + def test_empty_module_is_not_error(tmpdir): (tmpdir / "mycode.py").write("# Nothing to see here\n") result = subprocess.run( "hypothesis write mycode", - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, + capture_output=True, shell=True, - universal_newlines=True, + text=True, cwd=tmpdir, ) assert result.returncode == 0 diff --git a/hypothesis-python/tests/ghostwriter/try-writing-for-installed.py b/hypothesis-python/tests/ghostwriter/try-writing-for-installed.py index 285bf3daf4..2c0ba6cd53 100644 --- a/hypothesis-python/tests/ghostwriter/try-writing-for-installed.py +++ b/hypothesis-python/tests/ghostwriter/try-writing-for-installed.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """Try running `hypothesis write ...` on all available modules. @@ -55,10 +50,9 @@ def write_for(mod): subprocess.run( ["hypothesis", "write", mod], check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + capture_output=True, timeout=10, - universal_newlines=True, + text=True, ) except subprocess.SubprocessError as e: # Only report the error if we could load _but not process_ the module diff --git a/hypothesis-python/tests/lark/__init__.py b/hypothesis-python/tests/lark/__init__.py index e61e3c2872..fcb1ac6538 100644 --- a/hypothesis-python/tests/lark/__init__.py +++ b/hypothesis-python/tests/lark/__init__.py @@ -1,14 +1,9 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER diff --git a/hypothesis-python/tests/lark/test_grammar.py b/hypothesis-python/tests/lark/test_grammar.py index f2cbc814c7..cd770d4b9e 100644 --- a/hypothesis-python/tests/lark/test_grammar.py +++ b/hypothesis-python/tests/lark/test_grammar.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import json @@ -85,7 +80,7 @@ def test_can_generate_ignored_tokens(): %ignore WS """ strategy = from_lark(Lark(list_grammar, start="list")) - # A JSON list of strings in canoncial form which does not round-trip, + # A JSON list of strings in canonical form which does not round-trip, # must contain ignorable whitespace in the initial string. find_any(strategy, lambda s: "\t" in s) diff --git a/hypothesis-python/tests/nocover/__init__.py b/hypothesis-python/tests/nocover/__init__.py index e61e3c2872..fcb1ac6538 100644 --- a/hypothesis-python/tests/nocover/__init__.py +++ b/hypothesis-python/tests/nocover/__init__.py @@ -1,14 +1,9 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER diff --git a/hypothesis-python/tests/nocover/test_argument_validation.py b/hypothesis-python/tests/nocover/test_argument_validation.py index 88fe6b5bac..9a43c7534f 100644 --- a/hypothesis-python/tests/nocover/test_argument_validation.py +++ b/hypothesis-python/tests/nocover/test_argument_validation.py @@ -1,24 +1,19 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from inspect import Parameter import pytest from hypothesis import strategies as st -from hypothesis.strategies._internal.core import _strategies +from hypothesis.strategies._internal.utils import _strategies from tests.common.arguments import argument_validation_test, e diff --git a/hypothesis-python/tests/nocover/test_bad_repr.py b/hypothesis-python/tests/nocover/test_bad_repr.py index 70fc49bc3f..4f2909d689 100644 --- a/hypothesis-python/tests/nocover/test_bad_repr.py +++ b/hypothesis-python/tests/nocover/test_bad_repr.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import given, strategies as st diff --git a/hypothesis-python/tests/cover/test_baseexception.py b/hypothesis-python/tests/nocover/test_baseexception.py similarity index 62% rename from hypothesis-python/tests/cover/test_baseexception.py rename to hypothesis-python/tests/nocover/test_baseexception.py index 8ac7655fd8..3b4e6c96da 100644 --- a/hypothesis-python/tests/cover/test_baseexception.py +++ b/hypothesis-python/tests/nocover/test_baseexception.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest @@ -39,6 +34,9 @@ def test_exception_propagates_fine_from_strategy(e): @composite def interrupt_eventually(draw): raise e + # this line will not be executed, but must be here + # to pass draw function static reference check + return draw(st.none()) @given(interrupt_eventually()) def test_do_nothing(x): @@ -48,9 +46,7 @@ def test_do_nothing(x): test_do_nothing() -@pytest.mark.parametrize( - "e", [KeyboardInterrupt, SystemExit, GeneratorExit, ValueError] -) +@pytest.mark.parametrize("e", [KeyboardInterrupt, ValueError]) def test_baseexception_no_rerun_no_flaky(e): runs = [0] interrupt = 3 @@ -76,13 +72,14 @@ def test_raise_baseexception(x): "e", [KeyboardInterrupt, SystemExit, GeneratorExit, ValueError] ) def test_baseexception_in_strategy_no_rerun_no_flaky(e): - runs = [0] + runs = 0 interrupt = 3 @composite def interrupt_eventually(draw): - runs[0] += 1 - if runs[0] == interrupt: + nonlocal runs + runs += 1 + if runs == interrupt: raise e return draw(integers()) @@ -94,9 +91,40 @@ def test_do_nothing(x): with pytest.raises(e): test_do_nothing() - assert runs[0] == interrupt + assert runs == interrupt else: # Now SystemExit and GeneratorExit are caught like other exceptions with pytest.raises(Flaky): test_do_nothing() + + +TEMPLATE = """ +from hypothesis import given, note, strategies as st + +@st.composite +def things(draw): + raise {exception} + # this line will not be executed, but must be here + # to pass draw function static reference check + return draw(st.none()) + + +@given(st.data(), st.integers()) +def test(data, x): + if x > 100: + data.draw({strategy}) + raise {exception} +""" + + +@pytest.mark.parametrize("exc_name", ["SystemExit", "GeneratorExit"]) +@pytest.mark.parametrize("use_composite", [True, False]) +def test_explanations(testdir, exc_name, use_composite): + code = TEMPLATE.format( + exception=exc_name, strategy="things()" if use_composite else "st.none()" + ) + test_file = str(testdir.makepyfile(code)) + pytest_stdout = str(testdir.runpytest_inprocess(test_file, "--tb=native").stdout) + assert "x=101" in pytest_stdout + assert exc_name in pytest_stdout diff --git a/hypothesis-python/tests/nocover/test_boundary_exploration.py b/hypothesis-python/tests/nocover/test_boundary_exploration.py index 8337217c63..083e920449 100644 --- a/hypothesis-python/tests/nocover/test_boundary_exploration.py +++ b/hypothesis-python/tests/nocover/test_boundary_exploration.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest diff --git a/hypothesis-python/tests/nocover/test_build_signature.py b/hypothesis-python/tests/nocover/test_build_signature.py index e962cc892a..c92136e8b1 100644 --- a/hypothesis-python/tests/nocover/test_build_signature.py +++ b/hypothesis-python/tests/nocover/test_build_signature.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from inspect import signature from typing import List, get_type_hints @@ -110,7 +105,7 @@ def test_build_with_non_types_in_signature(val): class UnconventionalSignature: - def __init__(x: int = 0, self: bool = True): + def __init__(x: int = 0, self: bool = True): # noqa: B902 assert not isinstance(x, int) x.self = self diff --git a/hypothesis-python/tests/nocover/test_cache_implementation.py b/hypothesis-python/tests/nocover/test_cache_implementation.py index 8d264eb0a4..7175d63dee 100644 --- a/hypothesis-python/tests/nocover/test_cache_implementation.py +++ b/hypothesis-python/tests/nocover/test_cache_implementation.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from collections import Counter diff --git a/hypothesis-python/tests/nocover/test_cacheable.py b/hypothesis-python/tests/nocover/test_cacheable.py index 710f4ae0d5..23e9831379 100644 --- a/hypothesis-python/tests/nocover/test_cacheable.py +++ b/hypothesis-python/tests/nocover/test_cacheable.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import gc import weakref diff --git a/hypothesis-python/tests/nocover/test_characters.py b/hypothesis-python/tests/nocover/test_characters.py index c02dc16a67..7835670332 100644 --- a/hypothesis-python/tests/nocover/test_characters.py +++ b/hypothesis-python/tests/nocover/test_characters.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import string diff --git a/hypothesis-python/tests/nocover/test_collective_minimization.py b/hypothesis-python/tests/nocover/test_collective_minimization.py index db1f2443cb..bd51b2099c 100644 --- a/hypothesis-python/tests/nocover/test_collective_minimization.py +++ b/hypothesis-python/tests/nocover/test_collective_minimization.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest diff --git a/hypothesis-python/tests/nocover/test_compat.py b/hypothesis-python/tests/nocover/test_compat.py index a895d9b98d..b96dc87930 100644 --- a/hypothesis-python/tests/nocover/test_compat.py +++ b/hypothesis-python/tests/nocover/test_compat.py @@ -1,37 +1,17 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER - -from hypothesis import given, strategies as st -from hypothesis.internal.compat import ( - ceil, - floor, - int_from_bytes, - int_to_bytes, - qualname, -) - -class Foo: - def bar(self): - pass +import math - -def test_qualname(): - assert qualname(Foo.bar) == "Foo.bar" - assert qualname(Foo().bar) == "Foo.bar" - assert qualname(qualname) == "qualname" +from hypothesis import given, strategies as st +from hypothesis.internal.compat import ceil, floor, int_from_bytes, int_to_bytes @given(st.binary()) @@ -49,7 +29,7 @@ def test_to_int_in_big_endian_order(x, y): assert 0 <= int_from_bytes(x) <= int_from_bytes(y) -ints8 = st.integers(min_value=0, max_value=2 ** 63 - 1) +ints8 = st.integers(min_value=0, max_value=2**63 - 1) @given(ints8, ints8) @@ -60,17 +40,13 @@ def test_to_bytes_in_big_endian_order(x, y): @given(st.fractions()) def test_ceil(x): - """The compat ceil function always has the Python 3 semantics. - - Under Python 2, math.ceil returns a float, which cannot represent large - integers - for example, `float(2**53) == float(2**53 + 1)` - and this - is obviously incorrect for unlimited-precision integer operations. - """ assert isinstance(ceil(x), int) assert x <= ceil(x) < x + 1 + assert ceil(x) == math.ceil(x) @given(st.fractions()) def test_floor(x): assert isinstance(floor(x), int) assert x - 1 < floor(x) <= x + assert floor(x) == math.floor(x) diff --git a/hypothesis-python/tests/nocover/test_completion.py b/hypothesis-python/tests/nocover/test_completion.py index 07da38c201..630f0cae0e 100644 --- a/hypothesis-python/tests/nocover/test_completion.py +++ b/hypothesis-python/tests/nocover/test_completion.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import given, strategies as st diff --git a/hypothesis-python/tests/nocover/test_conjecture_engine.py b/hypothesis-python/tests/nocover/test_conjecture_engine.py index 554375b30f..3d6008d907 100644 --- a/hypothesis-python/tests/nocover/test_conjecture_engine.py +++ b/hypothesis-python/tests/nocover/test_conjecture_engine.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import given, settings, strategies as st from hypothesis.database import InMemoryExampleDatabase diff --git a/hypothesis-python/tests/nocover/test_conjecture_int_list.py b/hypothesis-python/tests/nocover/test_conjecture_int_list.py index 954b4e86de..10919cf2fa 100644 --- a/hypothesis-python/tests/nocover/test_conjecture_int_list.py +++ b/hypothesis-python/tests/nocover/test_conjecture_int_list.py @@ -1,23 +1,18 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import strategies as st from hypothesis.internal.conjecture.junkdrawer import IntList from hypothesis.stateful import RuleBasedStateMachine, initialize, invariant, rule -INTEGERS = st.integers(0, 2 ** 68) +INTEGERS = st.integers(0, 2**68) @st.composite diff --git a/hypothesis-python/tests/nocover/test_conjecture_utils.py b/hypothesis-python/tests/nocover/test_conjecture_utils.py index 166744f746..83d495e953 100644 --- a/hypothesis-python/tests/nocover/test_conjecture_utils.py +++ b/hypothesis-python/tests/nocover/test_conjecture_utils.py @@ -1,23 +1,21 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER +import sys from fractions import Fraction +from hypothesis import assume, example, given, strategies as st, target from hypothesis.internal.compat import int_to_bytes from hypothesis.internal.conjecture import utils as cu from hypothesis.internal.conjecture.data import ConjectureData, StopTest +from hypothesis.internal.conjecture.engine import BUFFER_SIZE def test_gives_the_correct_probabilities(): @@ -32,15 +30,43 @@ def test_gives_the_correct_probabilities(): counts = [0] * len(weights) i = 0 - while i < 2 ** 16: + while i < 2**16: data = ConjectureData.for_buffer(int_to_bytes(i, 2)) try: c = sampler.sample(data) counts[c] += 1 - assert probabilities[c] >= Fraction(counts[c], 2 ** 16) + assert probabilities[c] >= Fraction(counts[c], 2**16) except StopTest: pass if 1 in data.forced_indices: i += 256 else: i += 1 + + +@example(0, 1) +@example(0, float("inf")) +@example(cu.SMALLEST_POSITIVE_FLOAT, 2 * cu.SMALLEST_POSITIVE_FLOAT) +@example(cu.SMALLEST_POSITIVE_FLOAT, 1) +@example(cu.SMALLEST_POSITIVE_FLOAT, float("inf")) +@example(sys.float_info.min, 1) +@example(sys.float_info.min, float("inf")) +@example(10, 10) +@example(10, float("inf")) +# BUFFER_SIZE divided by (2bytes coin + 0byte element) gives the +# maximum number of elements that we would ever be able to generate. +@given(st.floats(0, BUFFER_SIZE // 2), st.integers(0, BUFFER_SIZE // 2)) +def test_p_continue(average_size, max_size): + assume(average_size <= max_size) + p = cu._calc_p_continue(average_size, max_size) + assert 0 <= target(p, label="p") <= 1 + assert 0 < target(p, label="-p") or average_size < 1e-5 + abs_err = abs(average_size - cu._p_continue_to_avg(p, max_size)) + assert target(abs_err, label="abs_err") < 0.01 + + +@example(1.1, 10) +@given(st.floats(0, 1), st.integers(0, BUFFER_SIZE // 2)) +def test_p_continue_to_average(p_continue, max_size): + average = cu._p_continue_to_avg(p_continue, max_size) + assert 0 <= average <= max_size diff --git a/hypothesis-python/tests/nocover/test_conventions.py b/hypothesis-python/tests/nocover/test_conventions.py index d53ba9b156..cba93fb623 100644 --- a/hypothesis-python/tests/nocover/test_conventions.py +++ b/hypothesis-python/tests/nocover/test_conventions.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis.utils.conventions import UniqueIdentifier diff --git a/hypothesis-python/tests/nocover/test_database_agreement.py b/hypothesis-python/tests/nocover/test_database_agreement.py index b5968e7bc0..1ab4aef758 100644 --- a/hypothesis-python/tests/nocover/test_database_agreement.py +++ b/hypothesis-python/tests/nocover/test_database_agreement.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import os import shutil diff --git a/hypothesis-python/tests/nocover/test_database_usage.py b/hypothesis-python/tests/nocover/test_database_usage.py index ec5a9acda2..dcd8fafdda 100644 --- a/hypothesis-python/tests/nocover/test_database_usage.py +++ b/hypothesis-python/tests/nocover/test_database_usage.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import os.path @@ -32,7 +27,7 @@ def test_saves_incremental_steps_in_database(): database = InMemoryExampleDatabase() find( st.binary(min_size=10), - lambda x: has_a_non_zero_byte(x), + has_a_non_zero_byte, settings=settings(database=database), database_key=key, ) @@ -68,30 +63,41 @@ def stuff(): if not keys: break else: - assert False + raise AssertionError def test_trashes_invalid_examples(): key = b"a database key" database = InMemoryExampleDatabase() - finicky = False + + invalid = set() def stuff(): try: - find( - st.binary(min_size=100), - lambda x: assume(not finicky) and has_a_non_zero_byte(x), + + def condition(x): + assume(x not in invalid) + return not invalid and has_a_non_zero_byte(x) + + return find( + st.binary(min_size=5), + condition, settings=settings(database=database), database_key=key, ) - except Unsatisfiable: + except (Unsatisfiable, NoSuchExample): pass - stuff() + with deterministic_PRNG(): + value = stuff() + original = len(all_values(database)) assert original > 1 - finicky = True - stuff() + + invalid.add(value) + with deterministic_PRNG(): + stuff() + assert len(all_values(database)) < original diff --git a/hypothesis-python/tests/nocover/test_deferred_errors.py b/hypothesis-python/tests/nocover/test_deferred_errors.py index 58f0fe4293..1d871babe2 100644 --- a/hypothesis-python/tests/nocover/test_deferred_errors.py +++ b/hypothesis-python/tests/nocover/test_deferred_errors.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest diff --git a/hypothesis-python/tests/nocover/test_drypython_returns.py b/hypothesis-python/tests/nocover/test_drypython_returns.py new file mode 100644 index 0000000000..08167667c0 --- /dev/null +++ b/hypothesis-python/tests/nocover/test_drypython_returns.py @@ -0,0 +1,202 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +from typing import Generic, TypeVar + +import pytest + +from hypothesis import given, strategies as st +from hypothesis.errors import ResolutionFailed + +from tests.common.debug import find_any +from tests.common.utils import temp_registered + +# Primitives: +# =========== + +_InstanceType = TypeVar("_InstanceType", covariant=True) +_TypeArgType1 = TypeVar("_TypeArgType1", covariant=True) +_FirstType = TypeVar("_FirstType") +_LawType = TypeVar("_LawType") + + +class KindN(Generic[_InstanceType, _TypeArgType1]): + pass + + +class Lawful(Generic[_LawType]): + """This type defines law-related operations.""" + + +class MappableN(Generic[_FirstType], Lawful["MappableN[_FirstType]"]): + """Behaves like a functor.""" + + +# End definition: +# =============== + +_ValueType = TypeVar("_ValueType") + + +class MyFunctor(KindN["MyFunctor", _ValueType], MappableN[_ValueType]): + def __init__(self, inner_value: _ValueType) -> None: + self.inner_value = inner_value + + +# Testing part: +# ============= + + +def target_func(mappable: "MappableN[_FirstType]") -> bool: + return isinstance(mappable, MappableN) + + +@given(st.data()) +def test_my_mappable(source: st.DataObject) -> None: + """ + Checks that complex types with multiple inheritance levels and strings are fine. + + Regression test for https://github.com/HypothesisWorks/hypothesis/issues/3060 + """ + # In `returns` we register all types in `__mro__` + # to be this exact type at the moment. But here, we only need `Mappable`. + # Current `__mro__` is `MyFunctor / Kind / Mappable`: + assert MyFunctor.__mro__[2] is MappableN + with temp_registered( + MyFunctor.__mro__[2], + st.builds(MyFunctor), + ): + assert source.draw(st.builds(target_func)) is True + + +A = TypeVar("A") +B = TypeVar("B") +C = TypeVar("C") +D = TypeVar("D") + + +class _FirstBase(Generic[A, B]): + pass + + +class _SecondBase(Generic[C, D]): + pass + + +# To be tested: + + +class TwoGenericBases1(_FirstBase[A, B], _SecondBase[C, D]): + pass + + +class TwoGenericBases2(_FirstBase[C, D], _SecondBase[A, B]): + pass + + +class OneGenericOneConrete1(_FirstBase[int, str], _SecondBase[A, B]): + pass + + +class OneGenericOneConrete2(_FirstBase[A, B], _SecondBase[float, bool]): + pass + + +class MixedGenerics1(_FirstBase[int, B], _SecondBase[C, bool]): + pass + + +class MixedGenerics2(_FirstBase[A, str], _SecondBase[float, D]): + pass + + +class AllConcrete(_FirstBase[int, str], _SecondBase[float, bool]): + pass + + +_generic_test_types = ( + TwoGenericBases1, + TwoGenericBases2, + OneGenericOneConrete1, + OneGenericOneConrete2, + MixedGenerics1, + MixedGenerics2, + AllConcrete, +) + + +@pytest.mark.parametrize("type_", _generic_test_types) +def test_several_generic_bases(type_): + with temp_registered(_FirstBase, st.builds(type_)): + find_any(st.builds(_FirstBase)) + + with temp_registered(_SecondBase, st.builds(type_)): + find_any(st.builds(_SecondBase)) + + +def var_generic_func1(obj: _FirstBase[A, B]): + pass + + +def var_generic_func2(obj: _SecondBase[A, B]): + pass + + +def concrete_generic_func1(obj: _FirstBase[int, str]): + pass + + +def concrete_generic_func2(obj: _SecondBase[float, bool]): + pass + + +def mixed_generic_func1(obj: _FirstBase[A, str]): + pass + + +def mixed_generic_func2(obj: _SecondBase[float, D]): + pass + + +@pytest.mark.parametrize("type_", _generic_test_types) +@pytest.mark.parametrize( + "func", + [ + var_generic_func1, + var_generic_func2, + concrete_generic_func1, + concrete_generic_func2, + mixed_generic_func1, + mixed_generic_func2, + ], +) +def test_several_generic_bases_functions(type_, func): + with temp_registered(_FirstBase, st.builds(type_)), temp_registered( + _SecondBase, st.builds(type_) + ): + find_any(st.builds(func)) + + with temp_registered(type_, st.builds(type_)): + find_any(st.builds(func)) + + +def wrong_generic_func1(obj: _FirstBase[A, None]): + pass + + +def wrong_generic_func2(obj: _SecondBase[None, bool]): + pass + + +@pytest.mark.parametrize("func", [wrong_generic_func1, wrong_generic_func2]) +def test_several_generic_bases_wrong_functions(func): + with temp_registered(AllConcrete, st.builds(AllConcrete)): + with pytest.raises(ResolutionFailed): + st.builds(func).example() diff --git a/hypothesis-python/tests/nocover/test_duplication.py b/hypothesis-python/tests/nocover/test_duplication.py index fb15722b62..7d53f625f3 100644 --- a/hypothesis-python/tests/nocover/test_duplication.py +++ b/hypothesis-python/tests/nocover/test_duplication.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from collections import Counter diff --git a/hypothesis-python/tests/nocover/test_dynamic_variable.py b/hypothesis-python/tests/nocover/test_dynamic_variable.py index 44bf9fe05d..0771f1b477 100644 --- a/hypothesis-python/tests/nocover/test_dynamic_variable.py +++ b/hypothesis-python/tests/nocover/test_dynamic_variable.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis.utils.dynamicvariables import DynamicVariable diff --git a/hypothesis-python/tests/nocover/test_emails.py b/hypothesis-python/tests/nocover/test_emails.py index 7affd123ef..41b3818c38 100644 --- a/hypothesis-python/tests/nocover/test_emails.py +++ b/hypothesis-python/tests/nocover/test_emails.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import given from hypothesis.strategies import emails diff --git a/hypothesis-python/tests/nocover/test_eval_as_source.py b/hypothesis-python/tests/nocover/test_eval_as_source.py index d872f8ee52..31bc24a5ce 100644 --- a/hypothesis-python/tests/nocover/test_eval_as_source.py +++ b/hypothesis-python/tests/nocover/test_eval_as_source.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis.internal.reflection import source_exec_as_module diff --git a/hypothesis-python/tests/nocover/test_explore_arbitrary_languages.py b/hypothesis-python/tests/nocover/test_explore_arbitrary_languages.py index f704aa3c39..82632791e8 100644 --- a/hypothesis-python/tests/nocover/test_explore_arbitrary_languages.py +++ b/hypothesis-python/tests/nocover/test_explore_arbitrary_languages.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import random @@ -75,6 +70,12 @@ class Terminal: writes = st.builds(Write, value=st.binary(min_size=1), child=nodes) +# Remember what the default phases are with no test running, so that we can +# run an outer test with non-default phases and then restore the defaults for +# the inner test. +_default_phases = settings.default.phases + + def run_language_test_for(root, data, seed): random.seed(seed) @@ -106,7 +107,9 @@ def test(local_data): database=None, suppress_health_check=HealthCheck.all(), verbosity=Verbosity.quiet, - phases=list(Phase), + # Restore the global default phases, so that we don't inherit the + # phases setting from the outer test. + phases=_default_phases, ), ) try: @@ -120,12 +123,12 @@ def test(local_data): @settings( suppress_health_check=HealthCheck.all(), deadline=None, - phases=set(Phase) - {Phase.shrink}, + phases=set(settings.default.phases) - {Phase.shrink}, ) @given(st.data()) def test_explore_an_arbitrary_language(data): root = data.draw(writes | branches) - seed = data.draw(st.integers(0, 2 ** 64 - 1)) + seed = data.draw(st.integers(0, 2**64 - 1)) run_language_test_for(root, data, seed) diff --git a/hypothesis-python/tests/nocover/test_fancy_repr.py b/hypothesis-python/tests/nocover/test_fancy_repr.py index 4153503a61..c4b7b2190f 100644 --- a/hypothesis-python/tests/nocover/test_fancy_repr.py +++ b/hypothesis-python/tests/nocover/test_fancy_repr.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import strategies as st diff --git a/hypothesis-python/tests/nocover/test_filtering.py b/hypothesis-python/tests/nocover/test_filtering.py index 4728695ad4..554633a0a0 100644 --- a/hypothesis-python/tests/nocover/test_filtering.py +++ b/hypothesis-python/tests/nocover/test_filtering.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest diff --git a/hypothesis-python/tests/nocover/test_find.py b/hypothesis-python/tests/nocover/test_find.py index b06812335c..2fd6c9c3fb 100644 --- a/hypothesis-python/tests/nocover/test_find.py +++ b/hypothesis-python/tests/nocover/test_find.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import math diff --git a/hypothesis-python/tests/nocover/test_fixtures.py b/hypothesis-python/tests/nocover/test_fixtures.py index 0d71086ccb..c697a2cc67 100644 --- a/hypothesis-python/tests/nocover/test_fixtures.py +++ b/hypothesis-python/tests/nocover/test_fixtures.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import time diff --git a/hypothesis-python/tests/nocover/test_flatmap.py b/hypothesis-python/tests/nocover/test_flatmap.py index 3abb074af1..2843ddef02 100644 --- a/hypothesis-python/tests/nocover/test_flatmap.py +++ b/hypothesis-python/tests/nocover/test_flatmap.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from collections import Counter @@ -64,7 +59,7 @@ def test_flatmap_retrieve_from_db(): def record_and_test_size(xs): if sum(xs) >= 1: track.append(xs) - assert False + raise AssertionError with pytest.raises(AssertionError): record_and_test_size() diff --git a/hypothesis-python/tests/nocover/test_floating.py b/hypothesis-python/tests/nocover/test_floating.py index 2fdc567079..fd679c3429 100644 --- a/hypothesis-python/tests/nocover/test_floating.py +++ b/hypothesis-python/tests/nocover/test_floating.py @@ -1,26 +1,25 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """Tests for being able to generate weird and wonderful floating point numbers.""" import math import sys +import pytest + from hypothesis import HealthCheck, assume, given, settings +from hypothesis.internal.floats import float_to_int from hypothesis.strategies import data, floats, lists +from tests.common.debug import find_any from tests.common.utils import fails TRY_HARDER = settings( @@ -102,32 +101,6 @@ def test_is_in_exact_int_range(x): assert x + 1 != x -# Tests whether we can represent subnormal floating point numbers. -# This is essentially a function of how the python interpreter -# was compiled. -# Everything is terrible -if math.ldexp(0.25, -1022) > 0: - REALLY_SMALL_FLOAT = sys.float_info.min -else: - REALLY_SMALL_FLOAT = sys.float_info.min * 2 - - -@fails -@given(floats()) -@TRY_HARDER -def test_can_generate_really_small_positive_floats(x): - assume(x > 0) - assert x >= REALLY_SMALL_FLOAT - - -@fails -@given(floats()) -@TRY_HARDER -def test_can_generate_really_small_negative_floats(x): - assume(x < 0) - assert x <= -REALLY_SMALL_FLOAT - - @fails @given(floats()) @TRY_HARDER @@ -153,3 +126,16 @@ def test_floats_are_in_range(x, y, data): t = data.draw(floats(x, y)) assert x <= t <= y + + +@pytest.mark.parametrize("neg", [False, True]) +@pytest.mark.parametrize("snan", [False, True]) +def test_can_find_negative_and_signaling_nans(neg, snan): + find_any( + floats().filter(math.isnan), + lambda x: ( + snan is (float_to_int(abs(x)) != float_to_int(float("nan"))) + and neg is (math.copysign(1, x) == -1) + ), + settings=TRY_HARDER, + ) diff --git a/hypothesis-python/tests/nocover/test_from_type_recipe.py b/hypothesis-python/tests/nocover/test_from_type_recipe.py index 48b8472c31..4d4464f8a6 100644 --- a/hypothesis-python/tests/nocover/test_from_type_recipe.py +++ b/hypothesis-python/tests/nocover/test_from_type_recipe.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import given, strategies as st from hypothesis.strategies._internal.types import _global_type_lookup diff --git a/hypothesis-python/tests/nocover/test_given_error_conditions.py b/hypothesis-python/tests/nocover/test_given_error_conditions.py index 519f5b3ce6..7d88e8bacc 100644 --- a/hypothesis-python/tests/nocover/test_given_error_conditions.py +++ b/hypothesis-python/tests/nocover/test_given_error_conditions.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest diff --git a/hypothesis-python/tests/nocover/test_given_reuse.py b/hypothesis-python/tests/nocover/test_given_reuse.py index 7fb31d7421..2abd49764c 100644 --- a/hypothesis-python/tests/nocover/test_given_reuse.py +++ b/hypothesis-python/tests/nocover/test_given_reuse.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest @@ -36,7 +31,7 @@ def test_has_an_arg_named_y(y): def test_fail_independently(): @given_named_booleans def test_z1(z): - assert False + raise AssertionError @given_named_booleans def test_z2(z): diff --git a/hypothesis-python/tests/nocover/test_imports.py b/hypothesis-python/tests/nocover/test_imports.py index 9a673403e1..a6f66aff1c 100644 --- a/hypothesis-python/tests/nocover/test_imports.py +++ b/hypothesis-python/tests/nocover/test_imports.py @@ -1,26 +1,20 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import * from hypothesis.strategies import * def test_can_star_import_from_hypothesis(): - @given(lists(integers())) - @settings(max_examples=10000, verbosity=Verbosity.quiet) - def f(x): - pass - - f() + find( + lists(integers()), + lambda x: sum(x) > 1, + settings=settings(max_examples=10000, verbosity=Verbosity.quiet), + ) diff --git a/hypothesis-python/tests/nocover/test_integer_ranges.py b/hypothesis-python/tests/nocover/test_integer_ranges.py index 453e4de96f..f8c949e161 100644 --- a/hypothesis-python/tests/nocover/test_integer_ranges.py +++ b/hypothesis-python/tests/nocover/test_integer_ranges.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest @@ -33,15 +28,20 @@ def do_draw(self, data): return integer_range(data, self.lower, self.upper, center=self.center) -@pytest.mark.parametrize("inter", [(0, 5, 10), (-10, 10, 10), (0, 1, 1), (1, 1, 2)]) -def test_intervals_shrink_to_center(inter): - lower, center, upper = inter +@pytest.mark.parametrize( + "lower_center_upper", + [(0, 5, 10), (-10, 10, 10), (0, 1, 1), (1, 1, 2), (-10, 0, 10), (-10, 5, 10)], + ids=repr, +) +def test_intervals_shrink_to_center(lower_center_upper): + lower, center, upper = lower_center_upper s = interval(lower, upper, center) assert minimal(s, lambda x: True) == center if lower < center: assert minimal(s, lambda x: x < center) == center - 1 if center < upper: assert minimal(s, lambda x: x > center) == center + 1 + assert minimal(s, lambda x: x != center) == center + 1 def test_bounded_integers_distribution_of_bit_width_issue_1387_regression(): diff --git a/hypothesis-python/tests/nocover/test_interesting_origin.py b/hypothesis-python/tests/nocover/test_interesting_origin.py index 6c046590c8..caa50ae91e 100644 --- a/hypothesis-python/tests/nocover/test_interesting_origin.py +++ b/hypothesis-python/tests/nocover/test_interesting_origin.py @@ -1,22 +1,17 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest from hypothesis import given, settings, strategies as st -from hypothesis.errors import MultipleFailures +from hypothesis.internal.compat import ExceptionGroup from tests.common.utils import flaky @@ -28,7 +23,7 @@ def go_wrong_naive(a, b): except Exception: # Hiding the actual problem is terrible, but this pattern can make sense # if you're raising a library-specific or semantically meaningful error. - raise ValueError("Something went wrong") + raise ValueError("Something went wrong") from None def go_wrong_with_cause(a, b): @@ -57,9 +52,11 @@ def go_wrong_coverup(a, b): ) @flaky(max_runs=3, min_passes=1) def test_can_generate_specified_version(function): - try: - given(st.integers(), st.integers())(settings(database=None)(function))() - except MultipleFailures: - pass - else: - raise AssertionError("Expected MultipleFailures") + @given(st.integers(), st.integers()) + @settings(database=None) + def test_fn(x, y): + # Indirection to fix https://github.com/HypothesisWorks/hypothesis/issues/2888 + return function(x, y) + + with pytest.raises(ExceptionGroup): + test_fn() diff --git a/hypothesis-python/tests/nocover/test_labels.py b/hypothesis-python/tests/nocover/test_labels.py index ce215f2a2a..20de4aa41c 100644 --- a/hypothesis-python/tests/nocover/test_labels.py +++ b/hypothesis-python/tests/nocover/test_labels.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import strategies as st @@ -27,12 +22,12 @@ def test_labels_are_distinct(): @st.composite def foo(draw): - pass + return draw(st.none()) @st.composite def bar(draw): - pass + return draw(st.none()) def test_different_composites_have_different_labels(): diff --git a/hypothesis-python/tests/nocover/test_large_examples.py b/hypothesis-python/tests/nocover/test_large_examples.py index ed4d8b27c3..72c8bfce2f 100644 --- a/hypothesis-python/tests/nocover/test_large_examples.py +++ b/hypothesis-python/tests/nocover/test_large_examples.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import strategies as st diff --git a/hypothesis-python/tests/nocover/test_limits.py b/hypothesis-python/tests/nocover/test_limits.py index 2b650a2649..8b7762a778 100644 --- a/hypothesis-python/tests/nocover/test_limits.py +++ b/hypothesis-python/tests/nocover/test_limits.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import given, settings, strategies as st diff --git a/hypothesis-python/tests/nocover/test_lstar.py b/hypothesis-python/tests/nocover/test_lstar.py new file mode 100644 index 0000000000..dca40b4ae9 --- /dev/null +++ b/hypothesis-python/tests/nocover/test_lstar.py @@ -0,0 +1,35 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import hypothesis.strategies as st +from hypothesis import Phase, example, given, settings +from hypothesis.internal.conjecture.dfa.lstar import LStar + + +@st.composite +def byte_order(draw): + ls = draw(st.permutations(range(256))) + n = draw(st.integers(0, len(ls))) + return ls[:n] + + +@example({0}, [1]) +@given(st.sets(st.integers(0, 255)), byte_order()) +# This test doesn't even use targeting at all, but for some reason the +# pareto optimizer makes it much slower. +@settings(phases=set(settings.default.phases) - {Phase.target}) +def test_learning_always_changes_generation(chars, order): + learner = LStar(lambda s: len(s) == 1 and s[0] in chars) + for c in order: + prev = learner.generation + s = bytes([c]) + if learner.dfa.matches(s) != learner.member(s): + learner.learn(s) + assert learner.generation > prev diff --git a/hypothesis-python/tests/nocover/test_modify_inner_test.py b/hypothesis-python/tests/nocover/test_modify_inner_test.py index 8ca571cf3d..28006ac6a3 100644 --- a/hypothesis-python/tests/nocover/test_modify_inner_test.py +++ b/hypothesis-python/tests/nocover/test_modify_inner_test.py @@ -1,23 +1,19 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from functools import wraps import pytest from hypothesis import given, strategies as st +from hypothesis.errors import InvalidArgument def always_passes(*args, **kwargs): @@ -30,7 +26,7 @@ def always_passes(*args, **kwargs): @given(st.integers()) def test_can_replace_inner_test(x): - assert False, "This should be replaced" + raise AssertionError("This should be replaced") test_can_replace_inner_test.hypothesis.inner_test = always_passes @@ -49,7 +45,7 @@ def inner(*args, **kwargs): @decorator @given(st.integers()) def test_can_replace_when_decorated(x): - assert False, "This should be replaced" + raise AssertionError("This should be replaced") test_can_replace_when_decorated.hypothesis.inner_test = always_passes @@ -58,7 +54,51 @@ def test_can_replace_when_decorated(x): @pytest.mark.parametrize("x", [1, 2]) @given(y=st.integers()) def test_can_replace_when_parametrized(x, y): - assert False, "This should be replaced" + raise AssertionError("This should be replaced") test_can_replace_when_parametrized.hypothesis.inner_test = always_passes + + +def test_can_replace_when_original_is_invalid(): + # Invalid: @given with too many positional arguments + @given(st.integers(), st.integers()) + def invalid_test(x): + raise AssertionError + + invalid_test.hypothesis.inner_test = always_passes + + # Even after replacing the inner test, calling the wrapper should still + # fail. + with pytest.raises(InvalidArgument, match="Too many positional arguments"): + invalid_test() + + +def test_inner_is_original_even_when_invalid(): + def invalid_test(x): + raise AssertionError + + original = invalid_test + + # Invalid: @given with no arguments + invalid_test = given()(invalid_test) + + # Verify that the test is actually invalid + with pytest.raises( + InvalidArgument, + match="given must be called with at least one argument", + ): + invalid_test() + + assert invalid_test.hypothesis.inner_test == original + + +def test_invokes_inner_function_with_args_by_name(): + # Regression test for https://github.com/HypothesisWorks/hypothesis/issues/3245 + @given(st.integers()) + def test(x): + pass + + f = test.hypothesis.inner_test + test.hypothesis.inner_test = wraps(f)(lambda **kw: f(**kw)) + test() diff --git a/hypothesis-python/tests/nocover/test_nesting.py b/hypothesis-python/tests/nocover/test_nesting.py index 728011b813..4546fba07e 100644 --- a/hypothesis-python/tests/nocover/test_nesting.py +++ b/hypothesis-python/tests/nocover/test_nesting.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from pytest import raises diff --git a/hypothesis-python/tests/nocover/test_pretty_repr.py b/hypothesis-python/tests/nocover/test_pretty_repr.py index 5f2ac6170d..fc0170e21c 100644 --- a/hypothesis-python/tests/nocover/test_pretty_repr.py +++ b/hypothesis-python/tests/nocover/test_pretty_repr.py @@ -1,20 +1,17 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from collections import OrderedDict +import pytest + from hypothesis import given, strategies as st from hypothesis.control import reject from hypothesis.errors import HypothesisDeprecationWarning, InvalidArgument @@ -105,3 +102,15 @@ def test_repr_evals_to_thing_with_same_repr(strategy): via_eval = eval(r, strategy_globals) r2 = repr(via_eval) assert r == r2 + + +@pytest.mark.parametrize( + "r", + [ + "none().filter(foo).map(bar)", + "just(1).filter(foo).map(bar)", + "sampled_from([1, 2, 3]).filter(foo).map(bar)", + ], +) +def test_sampled_transform_reprs(r): + assert repr(eval(r, strategy_globals)) == r diff --git a/hypothesis-python/tests/nocover/test_randomization.py b/hypothesis-python/tests/nocover/test_randomization.py index 14ea9f6467..b44461fe4c 100644 --- a/hypothesis-python/tests/nocover/test_randomization.py +++ b/hypothesis-python/tests/nocover/test_randomization.py @@ -1,32 +1,25 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER - -import random from pytest import raises -from hypothesis import Verbosity, find, given, settings, strategies as st +from hypothesis import Verbosity, core, find, given, settings, strategies as st from tests.common.utils import no_shrink -def test_seeds_off_random(): +def test_seeds_off_internal_random(): s = settings(phases=no_shrink, database=None) - r = random.getstate() + r = core._hypothesis_global_random.getstate() x = find(st.integers(), lambda x: True, settings=s) - random.setstate(r) + core._hypothesis_global_random.setstate(r) y = find(st.integers(), lambda x: True, settings=s) assert x == y diff --git a/hypothesis-python/tests/nocover/test_recursive.py b/hypothesis-python/tests/nocover/test_recursive.py index c4fce937bf..244eaa9382 100644 --- a/hypothesis-python/tests/nocover/test_recursive.py +++ b/hypothesis-python/tests/nocover/test_recursive.py @@ -1,21 +1,16 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import threading -from hypothesis import given, settings, strategies as st +from hypothesis import HealthCheck, given, settings, strategies as st from tests.common.debug import find_any, minimal from tests.common.utils import flaky @@ -133,10 +128,15 @@ def test_can_form_sets_of_recursive_data(): def test_drawing_from_recursive_strategy_is_thread_safe(): - shared_strategy = st.recursive(st.integers(), lambda s: st.lists(s, max_size=3)) + shared_strategy = st.recursive( + st.integers(), lambda s: st.lists(s, max_size=2), max_leaves=20 + ) errors = [] + @settings( + database=None, deadline=None, suppress_health_check=[HealthCheck.too_slow] + ) @given(data=st.data()) def test(data): try: @@ -156,3 +156,15 @@ def test(data): thread.join() assert not errors + + +SELF_REF = st.recursive( + st.deferred(lambda: st.booleans() | SELF_REF), + lambda s: st.lists(s, min_size=1), +) + + +@given(SELF_REF) +def test_self_ref_regression(_): + # See https://github.com/HypothesisWorks/hypothesis/issues/2794 + pass diff --git a/hypothesis-python/tests/nocover/test_regex.py b/hypothesis-python/tests/nocover/test_regex.py index 29db8d703b..316b982adc 100644 --- a/hypothesis-python/tests/nocover/test_regex.py +++ b/hypothesis-python/tests/nocover/test_regex.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import re import string @@ -52,13 +47,14 @@ def conservative_regex(draw): assume(COMBINED_MATCHER.search(result) is None) control = sum(result.count(c) for c in "?+*") assume(control <= 3) + assume(I_WITH_DOT not in result) # known to be weird return result CONSERVATIVE_REGEX = conservative_regex() -FLAGS = st.sets(st.sampled_from([getattr(re, "A", 0), re.I, re.M, re.S])).map( - lambda flag_set: reduce(int.__or__, flag_set, 0) -) +FLAGS = st.sets( + st.sampled_from([re.ASCII, re.IGNORECASE, re.MULTILINE, re.DOTALL]) +).map(lambda flag_set: reduce(int.__or__, flag_set, 0)) @given(st.data()) @@ -102,7 +98,7 @@ def test_case_insensitive_not_literal_never_constructs_multichar_match(data): for _ in range(5): s = data.draw(strategy) assert pattern.fullmatch(s) is not None - # And to be on the safe side, we implment this stronger property: + # And to be on the safe side, we implement this stronger property: assert set(s).isdisjoint(I_WITH_DOT.swapcase()) diff --git a/hypothesis-python/tests/nocover/test_regressions.py b/hypothesis-python/tests/nocover/test_regressions.py index e023152ff1..b716fdf450 100644 --- a/hypothesis-python/tests/nocover/test_regressions.py +++ b/hypothesis-python/tests/nocover/test_regressions.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest diff --git a/hypothesis-python/tests/nocover/test_reusable_values.py b/hypothesis-python/tests/nocover/test_reusable_values.py new file mode 100644 index 0000000000..ce3701a389 --- /dev/null +++ b/hypothesis-python/tests/nocover/test_reusable_values.py @@ -0,0 +1,162 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import pytest + +from hypothesis import example, given, strategies as st +from hypothesis.errors import InvalidArgument + +# Be aware that tests in this file pass strategies as arguments to @example. +# That's normally a mistake, but for these tests it's intended. +# If one of these tests fails, Hypothesis will complain about the +# @example/strategy interaction, but it should be safe to ignore that +# error message and focus on the underlying failure instead. + +base_reusable_strategies = ( + st.text(), + st.binary(), + st.dates(), + st.times(), + st.timedeltas(), + st.booleans(), + st.complex_numbers(), + st.floats(), + st.floats(-1.0, 1.0), + st.integers(), + st.integers(1, 10), + st.integers(1), + # Note that `just` and `sampled_from` count as "reusable" even if their + # values are mutable, because the user has implicitly promised that they + # don't care about the same mutable value being returned by separate draws. + st.just([]), + st.sampled_from([[]]), + st.tuples(st.integers()), +) + + +@st.deferred +def reusable(): + """Meta-strategy that produces strategies that should have + ``.has_reusable_values == True``.""" + return st.one_of( + # This looks like it should be `one_of`, but `sampled_from` is correct + # because we want this meta-strategy to yield strategies as its values. + st.sampled_from(base_reusable_strategies), + # This sometimes produces invalid combinations of arguments, which + # we filter out further down with an explicit validation check. + st.builds( + st.floats, + min_value=st.none() | st.floats(allow_nan=False), + max_value=st.none() | st.floats(allow_nan=False), + allow_infinity=st.booleans(), + allow_nan=st.booleans(), + ), + st.builds(st.just, st.builds(list)), + st.builds(st.sampled_from, st.lists(st.builds(list), min_size=1)), + st.lists(reusable).map(st.one_of), + st.lists(reusable).map(lambda ls: st.tuples(*ls)), + ) + + +def is_valid(s): + try: + s.validate() + return True + except InvalidArgument: + return False + + +reusable = reusable.filter(is_valid) + +assert not reusable.is_empty + + +def many_examples(examples): + """Helper decorator to apply the ``@example`` decorator multiple times, + once for each given example.""" + + def accept(f): + for e in examples: + f = example(e)(f) + return f + + return accept + + +@many_examples(base_reusable_strategies) +@many_examples(st.tuples(s) for s in base_reusable_strategies) +@given(reusable) +def test_reusable_strategies_are_all_reusable(s): + assert s.has_reusable_values + + +@many_examples(base_reusable_strategies) +@given(reusable) +def test_filter_breaks_reusability(s): + cond = True + + def nontrivial_filter(x): + """Non-trivial filtering function, intended to remain opaque even if + some strategies introspect their filters.""" + return cond + + assert s.has_reusable_values + assert not s.filter(nontrivial_filter).has_reusable_values + + +@many_examples(base_reusable_strategies) +@given(reusable) +def test_map_breaks_reusability(s): + cond = True + + def nontrivial_map(x): + """Non-trivial mapping function, intended to remain opaque even if + some strategies introspect their mappings.""" + if cond: + return x + else: + return None + + assert s.has_reusable_values + assert not s.map(nontrivial_map).has_reusable_values + + +@many_examples(base_reusable_strategies) +@given(reusable) +def test_flatmap_breaks_reusability(s): + cond = True + + def nontrivial_flatmap(x): + """Non-trivial flat-mapping function, intended to remain opaque even + if some strategies introspect their flat-mappings.""" + if cond: + return st.just(x) + else: + return st.none() + + assert s.has_reusable_values + assert not s.flatmap(nontrivial_flatmap).has_reusable_values + + +@pytest.mark.parametrize( + "strat", + [ + st.lists(st.booleans()), + st.sets(st.booleans()), + st.dictionaries(st.booleans(), st.booleans()), + ], +) +def test_mutable_collections_do_not_have_reusable_values(strat): + assert not strat.has_reusable_values + + +def test_recursion_does_not_break_reusability(): + x = st.deferred(lambda: st.none() | st.tuples(x)) + assert x.has_reusable_values diff --git a/hypothesis-python/tests/nocover/test_sampled_from.py b/hypothesis-python/tests/nocover/test_sampled_from.py index 9eebc95e70..d850935cb5 100644 --- a/hypothesis-python/tests/nocover/test_sampled_from.py +++ b/hypothesis-python/tests/nocover/test_sampled_from.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import enum @@ -23,7 +18,7 @@ from tests.common.utils import counts_calls, fails_with -@pytest.mark.parametrize("n", [100, 10 ** 5, 10 ** 6, 2 ** 25]) +@pytest.mark.parametrize("n", [100, 10**5, 10**6, 2**25]) def test_filter_large_lists(n): filter_limit = 100 * 10000 @@ -64,7 +59,7 @@ def test_chained_filters_find_rare_value(x): @fails_with(InvalidArgument) @given(st.sets(st.sampled_from(range(10)), min_size=11)) def test_unsat_sets_of_samples(x): - assert False + raise AssertionError @given(st.sets(st.sampled_from(range(50)), min_size=50)) diff --git a/hypothesis-python/tests/nocover/test_scrutineer.py b/hypothesis-python/tests/nocover/test_scrutineer.py new file mode 100644 index 0000000000..b0050320e7 --- /dev/null +++ b/hypothesis-python/tests/nocover/test_scrutineer.py @@ -0,0 +1,103 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import sys + +import pytest + +from hypothesis.internal.compat import PYPY +from hypothesis.internal.scrutineer import make_report + +# We skip tracing for explanations under PyPy, where it has a large performance +# impact, or if there is already a trace function (e.g. coverage or a debugger) +pytestmark = pytest.mark.skipif(PYPY or sys.gettrace(), reason="See comment") + +BUG_MARKER = "# BUG" +DEADLINE_PRELUDE = """ +from datetime import timedelta +from hypothesis.errors import DeadlineExceeded +""" +PRELUDE = """ +from hypothesis import Phase, given, settings, strategies as st + +@settings(phases=tuple(Phase), derandomize=True) +""" +TRIVIAL = """ +@given(st.integers()) +def test_reports_branch_in_test(x): + if x > 10: + raise AssertionError # BUG +""" +MULTIPLE_BUGS = """ +@given(st.integers(), st.integers()) +def test_reports_branch_in_test(x, y): + if x > 10: + raise (AssertionError if x % 2 else Exception) # BUG +""" +FRAGMENTS = ( + pytest.param(TRIVIAL, id="trivial"), + pytest.param(MULTIPLE_BUGS, id="multiple-bugs"), +) + + +def get_reports(file_contents, *, testdir): + # Takes the source code string with "# BUG" comments, and returns a list of + # multi-line report strings which we expect to see in explain-mode output. + # The list length is the number of explainable bugs, usually one. + test_file = str(testdir.makepyfile(file_contents)) + pytest_stdout = str(testdir.runpytest_inprocess(test_file, "--tb=native").stdout) + + explanations = { + i: {(test_file, i)} + for i, line in enumerate(file_contents.splitlines()) + if line.endswith(BUG_MARKER) + } + expected = [ + ("\n".join(r), "\n | ".join(r)) # single, ExceptionGroup + for r in make_report(explanations).values() + ] + return pytest_stdout, expected + + +@pytest.mark.parametrize("code", FRAGMENTS) +def test_explanations(code, testdir): + pytest_stdout, expected = get_reports(PRELUDE + code, testdir=testdir) + assert len(expected) == code.count(BUG_MARKER) + for single, group in expected: + assert single in pytest_stdout or group in pytest_stdout + + +@pytest.mark.parametrize("code", FRAGMENTS) +def test_no_explanations_if_deadline_exceeded(code, testdir): + code = code.replace("AssertionError", "DeadlineExceeded(timedelta(), timedelta())") + pytest_stdout, _ = get_reports(DEADLINE_PRELUDE + PRELUDE + code, testdir=testdir) + assert "Explanation:" not in pytest_stdout + + +NO_SHOW_CONTEXTLIB = """ +from contextlib import contextmanager +from hypothesis import given, strategies as st, Phase, settings + +@contextmanager +def ctx(): + yield + +@settings(phases=list(Phase)) +@given(st.integers()) +def test(x): + with ctx(): + assert x < 100 +""" + + +@pytest.mark.skipif(PYPY, reason="Tracing is slow under PyPy") +def test_skips_uninformative_locations(testdir): + pytest_stdout, _ = get_reports(NO_SHOW_CONTEXTLIB, testdir=testdir) + assert "Explanation:" not in pytest_stdout diff --git a/hypothesis-python/tests/nocover/test_sets.py b/hypothesis-python/tests/nocover/test_sets.py index 1ff1cf9e2e..9350db9275 100644 --- a/hypothesis-python/tests/nocover/test_sets.py +++ b/hypothesis-python/tests/nocover/test_sets.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import given, settings from hypothesis.strategies import floats, integers, sets diff --git a/hypothesis-python/tests/nocover/test_sharing.py b/hypothesis-python/tests/nocover/test_sharing.py index 1a53678031..bc41945c08 100644 --- a/hypothesis-python/tests/nocover/test_sharing.py +++ b/hypothesis-python/tests/nocover/test_sharing.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import given, strategies as st diff --git a/hypothesis-python/tests/nocover/test_simple_numbers.py b/hypothesis-python/tests/nocover/test_simple_numbers.py index a299340421..c3dfda4034 100644 --- a/hypothesis-python/tests/nocover/test_simple_numbers.py +++ b/hypothesis-python/tests/nocover/test_simple_numbers.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import math import sys @@ -37,10 +32,10 @@ def test_positive_negative_int(): boundaries = pytest.mark.parametrize( "boundary", sorted( - [2 ** i for i in range(10)] - + [2 ** i - 1 for i in range(10)] - + [2 ** i + 1 for i in range(10)] - + [10 ** i for i in range(6)] + [2**i for i in range(10)] + + [2**i - 1 for i in range(10)] + + [2**i + 1 for i in range(10)] + + [10**i for i in range(6)] ), ) @@ -80,12 +75,12 @@ def test_single_integer_range_is_range(): def test_minimal_small_number_in_large_range(): - assert minimal(integers((-(2 ** 32)), 2 ** 32), lambda x: x >= 101) == 101 + assert minimal(integers((-(2**32)), 2**32), lambda x: x >= 101) == 101 def test_minimal_small_sum_float_list(): - xs = minimal(lists(floats(), min_size=10), lambda x: sum(x) >= 1.0) - assert sum(xs) <= 2.0 + xs = minimal(lists(floats(), min_size=5), lambda x: sum(x) >= 1.0) + assert xs == [0.0, 0.0, 0.0, 0.0, 1.0] def test_minimals_boundary_floats(): @@ -93,18 +88,22 @@ def f(x): print(x) return True - assert -1 <= minimal(floats(min_value=-1, max_value=1), f) <= 1 + assert minimal(floats(min_value=-1, max_value=1), f) == 0 def test_minimal_non_boundary_float(): x = minimal(floats(min_value=1, max_value=9), lambda x: x > 2) - assert 2 < x < 3 + assert x == 3 # (the smallest integer > 2) def test_minimal_float_is_zero(): assert minimal(floats(), lambda x: True) == 0.0 +def test_minimal_asymetric_bounded_float(): + assert minimal(floats(min_value=1.1, max_value=1.9), lambda x: True) == 1.5 + + def test_negative_floats_simplify_to_zero(): assert minimal(floats(), lambda x: x <= -1.0) == -1.0 @@ -127,7 +126,7 @@ def test_minimize_nan(): def test_minimize_very_large_float(): t = sys.float_info.max / 2 - assert t <= minimal(floats(), lambda x: x >= t) < math.inf + assert minimal(floats(), lambda x: x >= t) == t def is_integral(value): @@ -138,7 +137,7 @@ def is_integral(value): def test_can_minimal_float_far_from_integral(): - minimal(floats(), lambda x: math.isfinite(x) and not is_integral(x * (2 ** 32))) + minimal(floats(), lambda x: math.isfinite(x) and not is_integral(x * (2**32))) def test_list_of_fractional_float(): @@ -148,11 +147,11 @@ def test_list_of_fractional_float(): lambda x: len([t for t in x if t >= 1.5]) >= 5, timeout_after=60, ) - ).issubset([1.5, 2.0]) + ) == {2} def test_minimal_fractional_float(): - assert minimal(floats(), lambda x: x >= 1.5) in (1.5, 2.0) + assert minimal(floats(), lambda x: x >= 1.5) == 2 def test_minimizes_lists_of_negative_ints_up_to_boundary(): @@ -183,7 +182,7 @@ def test_bounds_are_respected(): @pytest.mark.parametrize("k", range(10)) def test_floats_from_zero_have_reasonable_range(k): - n = 10 ** k + n = 10**k assert minimal(floats(min_value=0.0), lambda x: x >= n) == float(n) assert minimal(floats(max_value=0.0), lambda x: x <= -n) == float(-n) @@ -212,10 +211,10 @@ class TestFloatsAreFloats: def test_unbounded(self, arg): assert isinstance(arg, float) - @given(floats(min_value=0, max_value=float(2 ** 64 - 1))) + @given(floats(min_value=0, max_value=float(2**64 - 1))) def test_int_float(self, arg): assert isinstance(arg, float) - @given(floats(min_value=float(0), max_value=float(2 ** 64 - 1))) + @given(floats(min_value=float(0), max_value=float(2**64 - 1))) def test_float_float(self, arg): assert isinstance(arg, float) diff --git a/hypothesis-python/tests/nocover/test_simple_strings.py b/hypothesis-python/tests/nocover/test_simple_strings.py index f8bcac79df..2fbd24e564 100644 --- a/hypothesis-python/tests/nocover/test_simple_strings.py +++ b/hypothesis-python/tests/nocover/test_simple_strings.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import unicodedata diff --git a/hypothesis-python/tests/nocover/test_skipping.py b/hypothesis-python/tests/nocover/test_skipping.py index f0c8958107..608b9d679a 100644 --- a/hypothesis-python/tests/nocover/test_skipping.py +++ b/hypothesis-python/tests/nocover/test_skipping.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import unittest diff --git a/hypothesis-python/tests/nocover/test_stateful.py b/hypothesis-python/tests/nocover/test_stateful.py new file mode 100644 index 0000000000..b9a2e07429 --- /dev/null +++ b/hypothesis-python/tests/nocover/test_stateful.py @@ -0,0 +1,188 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +from collections import namedtuple + +import pytest + +from hypothesis import settings as Settings +from hypothesis.stateful import Bundle, RuleBasedStateMachine, precondition, rule +from hypothesis.strategies import booleans, integers, lists + +Leaf = namedtuple("Leaf", ("label",)) +Split = namedtuple("Split", ("left", "right")) + + +class BalancedTrees(RuleBasedStateMachine): + trees = Bundle("BinaryTree") + + @rule(target=trees, x=booleans()) + def leaf(self, x): + return Leaf(x) + + @rule(target=trees, left=trees, right=trees) + def split(self, left, right): + return Split(left, right) + + @rule(tree=trees) + def test_is_balanced(self, tree): + if isinstance(tree, Leaf): + return + else: + assert abs(self.size(tree.left) - self.size(tree.right)) <= 1 + self.test_is_balanced(tree.left) + self.test_is_balanced(tree.right) + + def size(self, tree): + if isinstance(tree, Leaf): + return 1 + else: + return 1 + self.size(tree.left) + self.size(tree.right) + + +class DepthCharge: + def __init__(self, value): + if value is None: + self.depth = 0 + else: + self.depth = value.depth + 1 + + +class DepthMachine(RuleBasedStateMachine): + charges = Bundle("charges") + + @rule(targets=(charges,), child=charges) + def charge(self, child): + return DepthCharge(child) + + @rule(targets=(charges,)) + def none_charge(self): + return DepthCharge(None) + + @rule(check=charges) + def is_not_too_deep(self, check): + assert check.depth < 3 + + +class RoseTreeStateMachine(RuleBasedStateMachine): + nodes = Bundle("nodes") + + @rule(target=nodes, source=lists(nodes)) + def bunch(self, source): + return source + + @rule(source=nodes) + def shallow(self, source): + def d(ls): + if not ls: + return 0 + else: + return 1 + max(map(d, ls)) + + assert d(source) <= 5 + + +class NotTheLastMachine(RuleBasedStateMachine): + stuff = Bundle("stuff") + + def __init__(self): + super().__init__() + self.last = None + self.bye_called = False + + @rule(target=stuff) + def hi(self): + result = object() + self.last = result + return result + + @precondition(lambda self: not self.bye_called) + @rule(v=stuff) + def bye(self, v): + assert v == self.last + self.bye_called = True + + +class PopulateMultipleTargets(RuleBasedStateMachine): + b1 = Bundle("b1") + b2 = Bundle("b2") + + @rule(targets=(b1, b2)) + def populate(self): + return 1 + + @rule(x=b1, y=b2) + def fail(self, x, y): + raise AssertionError + + +class CanSwarm(RuleBasedStateMachine): + """This test will essentially never pass if you choose rules uniformly at + random, because every time the snake rule fires we return to the beginning, + so we will tend to undo progress well before we make enough progress for + the test to fail. + + This tests our swarm testing functionality in stateful testing by ensuring + that we can sometimes generate long runs of steps which exclude a + particular rule. + """ + + def __init__(self): + super().__init__() + self.seen = set() + + # The reason this rule takes a parameter is that it ensures that we do not + # achieve "swarming" by by just restricting the alphabet for single byte + # decisions, which is a thing the underlying conjecture engine will + # happily do on its own without knowledge of the rule structure. + @rule(move=integers(0, 255)) + def ladder(self, move): + self.seen.add(move) + assert len(self.seen) <= 15 + + @rule() + def snake(self): + self.seen.clear() + + +bad_machines = ( + BalancedTrees, + DepthMachine, + RoseTreeStateMachine, + NotTheLastMachine, + PopulateMultipleTargets, + CanSwarm, +) + +for m in bad_machines: + m.TestCase.settings = Settings(m.TestCase.settings, max_examples=1000) + + +cheap_bad_machines = list(bad_machines) +cheap_bad_machines.remove(BalancedTrees) + + +with_cheap_bad_machines = pytest.mark.parametrize( + "machine", cheap_bad_machines, ids=[t.__name__ for t in cheap_bad_machines] +) + + +@pytest.mark.parametrize( + "machine", bad_machines, ids=[t.__name__ for t in bad_machines] +) +def test_bad_machines_fail(machine): + test_class = machine.TestCase + try: + test_class().runTest() + raise RuntimeError("Expected an assertion error") + except AssertionError as err: + notes = err.__notes__ + steps = [l for l in notes if "Step " in l or "state." in l] + assert 1 <= len(steps) <= 50 diff --git a/hypothesis-python/tests/nocover/test_strategy_state.py b/hypothesis-python/tests/nocover/test_strategy_state.py index d950b83219..97ab43e7a8 100644 --- a/hypothesis-python/tests/nocover/test_strategy_state.py +++ b/hypothesis-python/tests/nocover/test_strategy_state.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import hashlib import math @@ -107,13 +102,10 @@ def strategy_for_tupes(self, spec): return tuples(*spec) @rule(target=strategies, source=strategies, level=integers(1, 10), mixer=text()) - def filtered_strategy(s, source, level, mixer): + def filtered_strategy(self, source, level, mixer): def is_good(x): - return bool( - Random( - hashlib.sha384((mixer + repr(x)).encode("utf-8")).digest() - ).randint(0, level) - ) + seed = hashlib.sha384((mixer + repr(x)).encode()).digest() + return bool(Random(seed).randint(0, level)) return source.filter(is_good) @@ -131,7 +123,7 @@ def float(self, source): @rule(target=varied_floats, source=varied_floats, offset=integers(-100, 100)) def adjust_float(self, source, offset): - return int_to_float(clamp(0, float_to_int(source) + offset, 2 ** 64 - 1)) + return int_to_float(clamp(0, float_to_int(source) + offset, 2**64 - 1)) @rule(target=strategies, left=varied_floats, right=varied_floats) def float_range(self, left, right): @@ -155,7 +147,7 @@ def flatmapped_strategy(self, source, result1, result2, mixer, p): def do_map(value): rep = repr(value) - random = Random(hashlib.sha384((mixer + rep).encode("utf-8")).digest()) + random = Random(hashlib.sha384((mixer + rep).encode()).digest()) if random.random() <= p: return result1 else: diff --git a/hypothesis-python/tests/nocover/test_subnormal_floats.py b/hypothesis-python/tests/nocover/test_subnormal_floats.py new file mode 100644 index 0000000000..22881eb51c --- /dev/null +++ b/hypothesis-python/tests/nocover/test_subnormal_floats.py @@ -0,0 +1,56 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import math +from sys import float_info + +import pytest + +from hypothesis.internal.floats import width_smallest_normals +from hypothesis.strategies import floats + +from tests.common.debug import assert_all_examples, find_any +from tests.common.utils import PYTHON_FTZ + + +def test_python_compiled_with_sane_math_options(): + """Python does not flush-to-zero, which violates IEEE-754 + + The other tests that rely on subnormals are skipped when Python is FTZ + (otherwise pytest will be very noisy), so this meta test ensures CI jobs + still fail as we currently don't care to support such builds of Python. + """ + assert not PYTHON_FTZ + + +skipif_ftz = pytest.mark.skipif(PYTHON_FTZ, reason="broken by unsafe compiler flags") + + +@skipif_ftz +def test_can_generate_subnormals(): + find_any(floats().filter(lambda x: x > 0), lambda x: x < float_info.min) + find_any(floats().filter(lambda x: x < 0), lambda x: x > -float_info.min) + + +@skipif_ftz +@pytest.mark.parametrize( + "min_value, max_value", [(None, None), (-1, 0), (0, 1), (-1, 1)] +) +@pytest.mark.parametrize("width", [16, 32, 64]) +def test_does_not_generate_subnormals_when_disallowed(width, min_value, max_value): + strat = floats( + min_value=min_value, + max_value=max_value, + allow_subnormal=False, + width=width, + ) + strat = strat.filter(lambda x: x != 0.0 and math.isfinite(x)) + smallest_normal = width_smallest_normals[width] + assert_all_examples(strat, lambda x: x <= -smallest_normal or x >= smallest_normal) diff --git a/hypothesis-python/tests/nocover/test_targeting.py b/hypothesis-python/tests/nocover/test_targeting.py index dac37d2195..f90dda098d 100644 --- a/hypothesis-python/tests/nocover/test_targeting.py +++ b/hypothesis-python/tests/nocover/test_targeting.py @@ -1,21 +1,16 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest -from hypothesis import Phase, given, settings, strategies as st, target +from hypothesis import Phase, given, seed, settings, strategies as st, target pytest_plugins = "pytester" @@ -34,12 +29,11 @@ def test_threshold_problem(x): @pytest.mark.parametrize("multiple", [False, True]) def test_reports_target_results(testdir, multiple): script = testdir.makepyfile(TESTSUITE.format("" if multiple else "# ")) - result = testdir.runpytest(script) + result = testdir.runpytest(script, "--tb=native", "-rN") out = "\n".join(result.stdout.lines) assert "Falsifying example" in out assert "x=101" in out assert out.count("Highest target score") == 1 - assert out.index("Highest target score") < out.index("Falsifying example") assert result.ret != 0 @@ -54,3 +48,43 @@ def test_with_targeting(ls): with pytest.raises(AssertionError): test_with_targeting() + + +@given(st.integers(), st.integers()) +def test_target_returns_value(a, b): + difference = target(abs(a - b)) + assert difference == abs(a - b) + assert isinstance(difference, int) + + +def test_targeting_can_be_disabled(): + strat = st.lists(st.integers(0, 255)) + + def score(enabled): + result = [0] + phases = [Phase.generate] + if enabled: + phases.append(Phase.target) + + @seed(0) + @settings(database=None, max_examples=200, phases=phases) + @given(strat) + def test(ls): + score = float(sum(ls)) + result[0] = max(result[0], score) + target(score) + + test() + return result[0] + + assert score(enabled=True) > score(enabled=False) + + +def test_issue_2395_regression(): + @given(d=st.floats().filter(lambda x: abs(x) < 1000)) + @settings(max_examples=1000, database=None) + @seed(93962505385993024185959759429298090872) + def test_targeting_square_loss(d): + target(-((d - 42.5) ** 2.0)) + + test_targeting_square_loss() diff --git a/hypothesis-python/tests/nocover/test_testdecorators.py b/hypothesis-python/tests/nocover/test_testdecorators.py index 3c3c83908b..64b7323711 100644 --- a/hypothesis-python/tests/nocover/test_testdecorators.py +++ b/hypothesis-python/tests/nocover/test_testdecorators.py @@ -1,53 +1,52 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER -from hypothesis import HealthCheck, given, reject, settings -from hypothesis.errors import InvalidArgument, Unsatisfiable -from hypothesis.strategies import integers +import re + +import pytest -from tests.common.utils import raises +from hypothesis import HealthCheck, given, reject, settings, strategies as st +from hypothesis.errors import InvalidArgument, Unsatisfiable def test_contains_the_test_function_name_in_the_exception_string(): look_for_one = settings(max_examples=1, suppress_health_check=HealthCheck.all()) - @given(integers()) + @given(st.integers()) @look_for_one def this_has_a_totally_unique_name(x): reject() - with raises(Unsatisfiable) as e: + with pytest.raises( + Unsatisfiable, match=re.escape(this_has_a_totally_unique_name.__name__) + ): this_has_a_totally_unique_name() - assert this_has_a_totally_unique_name.__name__ in e.value.args[0] class Foo: - @given(integers()) + @given(st.integers()) @look_for_one def this_has_a_unique_name_and_lives_on_a_class(self, x): reject() - with raises(Unsatisfiable) as e: + with pytest.raises( + Unsatisfiable, + match=re.escape(Foo.this_has_a_unique_name_and_lives_on_a_class.__name__), + ): Foo().this_has_a_unique_name_and_lives_on_a_class() - assert (Foo.this_has_a_unique_name_and_lives_on_a_class.__name__) in e.value.args[0] def test_signature_mismatch_error_message(): # Regression test for issue #1978 @settings(max_examples=2) - @given(x=integers()) + @given(x=st.integers()) def bad_test(): pass @@ -58,3 +57,9 @@ def bad_test(): str(e) == "bad_test() got an unexpected keyword argument 'x', " "from `x=integers()` in @given" ) + + +@given(data=st.data(), keys=st.lists(st.integers(), unique=True)) +def test_fixed_dict_preserves_iteration_order(data, keys): + d = data.draw(st.fixed_dictionaries({k: st.none() for k in keys})) + assert all(a == b for a, b in zip(keys, d)), f"keys={keys}, d.keys()={d.keys()}" diff --git a/hypothesis-python/tests/nocover/test_threading.py b/hypothesis-python/tests/nocover/test_threading.py index fc6bdcc059..07b2316fd9 100644 --- a/hypothesis-python/tests/nocover/test_threading.py +++ b/hypothesis-python/tests/nocover/test_threading.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import threading diff --git a/hypothesis-python/tests/nocover/test_type_lookup.py b/hypothesis-python/tests/nocover/test_type_lookup.py new file mode 100644 index 0000000000..b5c2bfbd04 --- /dev/null +++ b/hypothesis-python/tests/nocover/test_type_lookup.py @@ -0,0 +1,84 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +from typing import Callable + +import pytest + +from hypothesis import strategies as st +from hypothesis.errors import InvalidArgument +from hypothesis.internal.compat import Concatenate, ParamSpec +from hypothesis.strategies._internal.types import NON_RUNTIME_TYPES + +try: + from typing import TypeGuard # new in 3.10 +except ImportError: + TypeGuard = None + + +@pytest.mark.parametrize("non_runtime_type", NON_RUNTIME_TYPES) +def test_non_runtime_type_cannot_be_resolved(non_runtime_type): + strategy = st.from_type(non_runtime_type) + with pytest.raises( + InvalidArgument, match="there is no such thing as a runtime instance" + ): + strategy.example() + + +@pytest.mark.parametrize("non_runtime_type", NON_RUNTIME_TYPES) +def test_non_runtime_type_cannot_be_registered(non_runtime_type): + with pytest.raises( + InvalidArgument, match="there is no such thing as a runtime instance" + ): + st.register_type_strategy(non_runtime_type, st.none()) + + +@pytest.mark.skipif(Concatenate is None, reason="requires python3.10 or higher") +def test_callable_with_concatenate(): + P = ParamSpec("P") + func_type = Callable[Concatenate[int, P], None] + strategy = st.from_type(func_type) + with pytest.raises( + InvalidArgument, + match="Hypothesis can't yet construct a strategy for instances of a Callable type", + ): + strategy.example() + + with pytest.raises(InvalidArgument, match="Cannot register generic type"): + st.register_type_strategy(func_type, st.none()) + + +@pytest.mark.skipif(ParamSpec is None, reason="requires python3.10 or higher") +def test_callable_with_paramspec(): + P = ParamSpec("P") + func_type = Callable[P, None] + strategy = st.from_type(func_type) + with pytest.raises( + InvalidArgument, + match="Hypothesis can't yet construct a strategy for instances of a Callable type", + ): + strategy.example() + + with pytest.raises(InvalidArgument, match="Cannot register generic type"): + st.register_type_strategy(func_type, st.none()) + + +@pytest.mark.skipif(TypeGuard is None, reason="requires python3.10 or higher") +def test_callable_return_typegard_type(): + strategy = st.from_type(Callable[[], TypeGuard[int]]) + with pytest.raises( + InvalidArgument, + match="Hypothesis cannot yet construct a strategy for callables " + "which are PEP-647 TypeGuards", + ): + strategy.example() + + with pytest.raises(InvalidArgument, match="Cannot register generic type"): + st.register_type_strategy(Callable[[], TypeGuard[int]], st.none()) diff --git a/hypothesis-python/tests/nocover/test_type_lookup_forward_ref.py b/hypothesis-python/tests/nocover/test_type_lookup_forward_ref.py index 16532a2674..5f4f60a98d 100644 --- a/hypothesis-python/tests/nocover/test_type_lookup_forward_ref.py +++ b/hypothesis-python/tests/nocover/test_type_lookup_forward_ref.py @@ -1,33 +1,22 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER -import sys -from typing import Dict, List, Union +from typing import Dict, ForwardRef, List, Union import pytest from hypothesis import given, strategies as st from hypothesis.errors import ResolutionFailed -from hypothesis.internal.compat import ForwardRef from tests.common import utils -skip_before_python37 = pytest.mark.skipif( - sys.version_info[:2] < (3, 7), reason="typing module was broken" -) - # Mutually-recursive types # See https://github.com/HypothesisWorks/hypothesis/issues/2722 @@ -35,8 +24,8 @@ @given(st.data()) def test_mutually_recursive_types_with_typevar(data): # The previously-failing example from the issue - A = Dict[str, "B"] # noqa: F821 - an undefined name is the whole point! - B = Union[List[str], A] + A = Dict[bool, "B"] # noqa: F821 - an undefined name is the whole point! + B = Union[List[bool], A] with pytest.raises(ResolutionFailed, match=r"Could not resolve ForwardRef\('B'\)"): data.draw(st.from_type(A)) @@ -55,8 +44,8 @@ def test_mutually_recursive_types_with_typevar(data): def test_mutually_recursive_types_with_typevar_alternate(data): # It's not particularly clear why this version passed when the previous # test failed, but different behaviour means we add both to the suite. - C = Union[List[str], "D"] # noqa: F821 - an undefined name is the whole point! - D = Dict[str, C] + C = Union[List[bool], "D"] # noqa: F821 - an undefined name is the whole point! + D = Dict[bool, C] with pytest.raises(ResolutionFailed, match=r"Could not resolve ForwardRef\('D'\)"): data.draw(st.from_type(C)) diff --git a/hypothesis-python/tests/nocover/test_type_lookup_future_annotations.py b/hypothesis-python/tests/nocover/test_type_lookup_future_annotations.py new file mode 100644 index 0000000000..d33550267e --- /dev/null +++ b/hypothesis-python/tests/nocover/test_type_lookup_future_annotations.py @@ -0,0 +1,48 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +from __future__ import annotations + +from typing import TypedDict, Union + +import pytest + +from hypothesis import given, strategies as st +from hypothesis.errors import InvalidArgument + +alias = Union[int, str] + + +class A(TypedDict): + a: int + + +class B(TypedDict): + a: A + b: alias + + +@given(st.from_type(B)) +def test_complex_forward_ref_in_typed_dict(d): + assert isinstance(d["a"], dict) + assert isinstance(d["a"]["a"], int) + assert isinstance(d["b"], (int, str)) + + +def test_complex_forward_ref_in_typed_dict_local(): + local_alias = Union[int, str] + + class C(TypedDict): + a: A + b: local_alias + + c_strategy = st.from_type(C) + with pytest.raises(InvalidArgument): + c_strategy.example() diff --git a/hypothesis-python/tests/nocover/test_unusual_settings_configs.py b/hypothesis-python/tests/nocover/test_unusual_settings_configs.py index e519d07b7c..e797bef57e 100644 --- a/hypothesis-python/tests/nocover/test_unusual_settings_configs.py +++ b/hypothesis-python/tests/nocover/test_unusual_settings_configs.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import HealthCheck, Verbosity, assume, given, settings, strategies as st diff --git a/hypothesis-python/tests/nocover/test_uuids.py b/hypothesis-python/tests/nocover/test_uuids.py index df7c65d47a..2b4c2ffed6 100644 --- a/hypothesis-python/tests/nocover/test_uuids.py +++ b/hypothesis-python/tests/nocover/test_uuids.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest diff --git a/hypothesis-python/tests/numpy/__init__.py b/hypothesis-python/tests/numpy/__init__.py index e61e3c2872..fcb1ac6538 100644 --- a/hypothesis-python/tests/numpy/__init__.py +++ b/hypothesis-python/tests/numpy/__init__.py @@ -1,14 +1,9 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER diff --git a/hypothesis-python/tests/numpy/test_argument_validation.py b/hypothesis-python/tests/numpy/test_argument_validation.py index b291fbd8af..3215945152 100644 --- a/hypothesis-python/tests/numpy/test_argument_validation.py +++ b/hypothesis-python/tests/numpy/test_argument_validation.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import numpy import pytest @@ -20,6 +15,8 @@ from hypothesis.errors import InvalidArgument from hypothesis.extra import numpy as nps +from tests.common.utils import checks_deprecated_behaviour + def e(a, **kwargs): kw = ", ".join(f"{k}={v!r}" for k, v in kwargs.items()) @@ -267,7 +264,7 @@ def e(a, **kwargs): e(nps.basic_indices, shape=(0, 0), max_dims=-1), e(nps.basic_indices, shape=(0, 0), max_dims=1.0), e(nps.basic_indices, shape=(0, 0), min_dims=2, max_dims=1), - e(nps.basic_indices, shape=(0, 0), max_dims=50), + e(nps.basic_indices, shape=(3, 3, 3), max_dims="not an int"), e(nps.integer_array_indices, shape=()), e(nps.integer_array_indices, shape=(2, 0)), e(nps.integer_array_indices, shape="a"), @@ -278,3 +275,16 @@ def e(a, **kwargs): def test_raise_invalid_argument(function, kwargs): with pytest.raises(InvalidArgument): function(**kwargs).example() + + +@pytest.mark.parametrize( + ("function", "kwargs"), + [ + e(nps.basic_indices, shape=(0, 0), min_dims=50), + e(nps.basic_indices, shape=(0, 0), max_dims=50), + ], +) +@checks_deprecated_behaviour +def test_raise_invalid_argument_deprecated(function, kwargs): + with pytest.raises(InvalidArgument): + function(**kwargs).example() diff --git a/hypothesis-python/tests/numpy/test_deprecation.py b/hypothesis-python/tests/numpy/test_deprecation.py new file mode 100644 index 0000000000..f1338c9518 --- /dev/null +++ b/hypothesis-python/tests/numpy/test_deprecation.py @@ -0,0 +1,34 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +from warnings import catch_warnings + +import pytest + +from hypothesis.errors import HypothesisDeprecationWarning, InvalidArgument +from hypothesis.extra import numpy as nps + + +def test_basic_indices_bad_min_dims_warns(): + with pytest.warns(HypothesisDeprecationWarning): + with pytest.raises(InvalidArgument): + nps.basic_indices((3, 3, 3), min_dims=4).example() + + +def test_basic_indices_bad_max_dims_warns(): + with pytest.warns(HypothesisDeprecationWarning): + nps.basic_indices((3, 3, 3), max_dims=4).example() + + +def test_basic_indices_default_max_dims_does_not_warn(): + with catch_warnings(record=True) as record: + nps.basic_indices((3, 3, 3)).example() + nps.basic_indices((3, 3, 3), allow_newaxis=True).example() + assert len(record) == 0 diff --git a/hypothesis-python/tests/numpy/test_fill_values.py b/hypothesis-python/tests/numpy/test_fill_values.py index 581a149245..bb535e3da1 100644 --- a/hypothesis-python/tests/numpy/test_fill_values.py +++ b/hypothesis-python/tests/numpy/test_fill_values.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import given, strategies as st from hypothesis.extra.numpy import arrays @@ -27,7 +22,7 @@ def test_generated_lists_are_distinct(ls): @st.composite def distinct_integers(draw): used = draw(st.shared(st.builds(set), key="distinct_integers.used")) - i = draw(st.integers(0, 2 ** 64 - 1).filter(lambda x: x not in used)) + i = draw(st.integers(0, 2**64 - 1).filter(lambda x: x not in used)) used.add(i) return i @@ -52,7 +47,7 @@ def test_minimizes_to_fill(): @given( arrays( dtype=float, - elements=st.floats().filter(bool), + elements=st.floats(allow_nan=False).filter(bool), shape=(3, 3, 3), fill=st.just(1.0), ) diff --git a/hypothesis-python/tests/numpy/test_floor_ceil.py b/hypothesis-python/tests/numpy/test_floor_ceil.py new file mode 100644 index 0000000000..1d3f35f30e --- /dev/null +++ b/hypothesis-python/tests/numpy/test_floor_ceil.py @@ -0,0 +1,48 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import math + +import numpy as np +import pytest + +from hypothesis.internal.compat import ceil, floor + + +@pytest.mark.parametrize( + "value", + [ + # These are strings so that the test names are easier to read. + "2**64+1", + "2**64-1", + "2**63+1", + "2**53+1", + "-2**53-1", + "-2**63+1", + "-2**63-1", + "-2**64+1", + "-2**64-1", + ], +) +def test_our_floor_and_ceil_avoid_numpy_rounding(value): + a = np.array([eval(value)]) + + f = floor(a) + c = ceil(a) + + assert type(f) == int + assert type(c) == int + + # Using math.floor or math.ceil for these values would give an incorrect + # result. + assert (math.floor(a) > a) or (math.ceil(a) < a) + + assert f <= a <= c + assert f + 1 > a > c - 1 diff --git a/hypothesis-python/tests/numpy/test_from_dtype.py b/hypothesis-python/tests/numpy/test_from_dtype.py index a80104f506..5b4850d18c 100644 --- a/hypothesis-python/tests/numpy/test_from_dtype.py +++ b/hypothesis-python/tests/numpy/test_from_dtype.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import numpy as np import pytest @@ -47,6 +42,11 @@ bytes, ) ] +for nonstandard_typecode in ["g", "G", "S1", "q", "Q"]: + try: + STANDARD_TYPES.append(np.dtype(nonstandard_typecode)) + except Exception: + pass @given(nps.nested_dtypes()) @@ -104,11 +104,11 @@ def test_can_unicode_strings_without_decode_error(arr): pass -@pytest.mark.xfail(strict=False, reason="mitigation for issue above") +@pytest.mark.skipif(not nps.NP_FIXED_UNICODE, reason="workaround for old bug") def test_unicode_string_dtypes_need_not_be_utf8(): def cannot_encode(string): try: - string.encode("utf-8") + string.encode() return False except UnicodeEncodeError: return True diff --git a/hypothesis-python/tests/numpy/test_gen_data.py b/hypothesis-python/tests/numpy/test_gen_data.py index 34571833f5..554efc1df8 100644 --- a/hypothesis-python/tests/numpy/test_gen_data.py +++ b/hypothesis-python/tests/numpy/test_gen_data.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import sys from functools import reduce @@ -20,8 +15,16 @@ import numpy as np import pytest -from hypothesis import HealthCheck, assume, given, note, settings, strategies as st -from hypothesis.errors import InvalidArgument, Unsatisfiable +from hypothesis import ( + HealthCheck, + assume, + given, + note, + settings, + strategies as st, + target, +) +from hypothesis.errors import InvalidArgument, UnsatisfiedAssumption from hypothesis.extra import numpy as nps from tests.common.debug import find_any, minimal @@ -67,8 +70,9 @@ def test_can_minimize_large_arrays(): @flaky(max_runs=50, min_passes=1) +@np.errstate(over="ignore", invalid="ignore") def test_can_minimize_float_arrays(): - x = minimal(nps.arrays(float, 50), lambda t: t.sum() >= 1.0) + x = minimal(nps.arrays(float, 50), lambda t: np.nansum(t) >= 1.0) assert x.sum() in (1, 50) @@ -250,7 +254,7 @@ def test_array_values_are_unique(arr): def test_cannot_generate_unique_array_of_too_many_elements(): strat = nps.arrays(dtype=int, elements=st.integers(0, 5), shape=10, unique=True) - with pytest.raises(Unsatisfiable): + with pytest.raises(InvalidArgument): strat.example() @@ -273,6 +277,23 @@ def test_generates_all_values_for_unique_array(arr): assert len(set(arr)) == len(arr) +@given(nps.arrays(dtype="int8", shape=255, unique=True)) +def test_efficiently_generates_all_unique_array(arr): + # Avoids the birthday paradox with UniqueSampledListStrategy + assert len(set(arr)) == len(arr) + + +@given(st.data(), st.integers(-100, 100), st.integers(1, 100)) +def test_array_element_rewriting(data, start, size): + arr = nps.arrays( + dtype=np.dtype("int64"), + shape=size, + elements=st.integers(start, start + size - 1), + unique=True, + ) + assert set(data.draw(arr)) == set(range(start, start + size)) + + def test_may_fill_with_nan_when_unique_is_set(): find_any( nps.arrays( @@ -335,10 +356,10 @@ def test_overflowing_integers_are_deprecated(fill, data): "dtype,strat", [ ("float16", st.floats(min_value=65520, allow_infinity=False)), - ("float32", st.floats(min_value=10 ** 40, allow_infinity=False)), + ("float32", st.floats(min_value=10**40, allow_infinity=False)), ( "complex64", - st.complex_numbers(min_magnitude=10 ** 300, allow_infinity=False), + st.complex_numbers(min_magnitude=10**300, allow_infinity=False), ), ("U1", st.text(min_size=2, max_size=2)), ("S1", st.binary(min_size=2, max_size=2)), @@ -406,7 +427,7 @@ def test_unique_array_with_fill_can_use_all_elements(arr): @given(nps.arrays(dtype="uint8", shape=25, unique=True, fill=st.nothing())) def test_unique_array_without_fill(arr): - # This test covers the collision-related branchs for fully dense unique arrays. + # This test covers the collision-related branches for fully dense unique arrays. # Choosing 25 of 256 possible elements means we're almost certain to see colisions # thanks to the 'birthday paradox', but finding unique elemennts is still easy. assume(len(set(arr)) == arr.size) @@ -484,8 +505,7 @@ def test_broadcastable_shape_bounds_are_satisfied(shape, data): label="bshape", ) except InvalidArgument: - assume(False) - assert False, "unreachable" + raise UnsatisfiedAssumption from None if max_dims is None: max_dims = max(len(shape), min_dims) + 2 @@ -524,8 +544,7 @@ def test_mutually_broadcastable_shape_bounds_are_satisfied( label="shapes, result", ) except InvalidArgument: - assume(False) - assert False, "unreachable" + raise UnsatisfiedAssumption from None if max_dims is None: max_dims = max(len(base_shape), min_dims) + 2 @@ -579,7 +598,7 @@ def _broadcast_shapes(*shapes): input shapes together. Raises ValueError if the shapes are not broadcast-compatible""" - assert len(shapes) + assert shapes, "Must pass >=1 shapes to broadcast" return reduce(_broadcast_two_shapes, shapes, ()) @@ -682,30 +701,6 @@ def test_mutually_broadcastable_shape_can_broadcast( assert result == _broadcast_shapes(base_shape, *shapes) -@settings(deadline=None, max_examples=10) -@given(min_dims=st.integers(0, 32), shape=ANY_SHAPE, data=st.data()) -def test_minimize_broadcastable_shape(min_dims, shape, data): - # Ensure aligned dimensions of broadcastable shape minimizes to `(1,) * min_dims` - max_dims = data.draw(st.none() | st.integers(min_dims, 32), label="max_dims") - min_side, max_side = _draw_valid_bounds(data, shape, max_dims, permit_none=False) - smallest = minimal( - nps.broadcastable_shapes( - shape, - min_side=min_side, - max_side=max_side, - min_dims=min_dims, - max_dims=max_dims, - ) - ) - note(f"(smallest): {smallest}") - n_leading = max(len(smallest) - len(shape), 0) - n_aligned = max(len(smallest) - n_leading, 0) - expected = [min_side] * n_leading + [ - 1 if min_side <= 1 <= max_side else i for i in shape[len(shape) - n_aligned :] - ] - assert tuple(expected) == smallest - - @settings(deadline=None, max_examples=50) @given( num_shapes=st.integers(1, 3), @@ -724,7 +719,7 @@ def test_minimize_mutually_broadcastable_shape(num_shapes, min_dims, base_shape, # shrinking gets a little bit hairy when we have empty axes # and multiple num_shapes assume(min_side > 0) - note("(min_side, max_side): {}".format((min_side, max_side))) + smallest_shapes, result = minimal( nps.mutually_broadcastable_shapes( num_shapes=num_shapes, @@ -735,15 +730,18 @@ def test_minimize_mutually_broadcastable_shape(num_shapes, min_dims, base_shape, max_dims=max_dims, ) ) - note("(smallest_shapes, result): {}".format((smallest_shapes, result))) + note(f"smallest_shapes: {smallest_shapes}") + note(f"result: {result}") assert len(smallest_shapes) == num_shapes assert result == _broadcast_shapes(base_shape, *smallest_shapes) for smallest in smallest_shapes: n_leading = max(len(smallest) - len(base_shape), 0) n_aligned = max(len(smallest) - n_leading, 0) + note(f"n_leading: {n_leading}") + note(f"n_aligned: {n_aligned} {base_shape[-n_aligned:]}") expected = [min_side] * n_leading + [ - 1 if min_side <= 1 <= max_side else i - for i in base_shape[len(base_shape) - n_aligned :] + (min(1, i) if i != 1 else min_side) if min_side <= 1 <= max_side else i + for i in (base_shape[-n_aligned:] if n_aligned else ()) ] assert tuple(expected) == smallest @@ -855,7 +853,7 @@ def test_mutually_broadcastable_shapes_shrinking_with_singleton_out_of_bounds( max_dims=max_dims, ) ) - note("(smallest_shapes, result): {}".format((smallest_shapes, result))) + note(f"(smallest_shapes, result): {(smallest_shapes, result)}") assert len(smallest_shapes) == num_shapes assert result == _broadcast_shapes(base_shape, *smallest_shapes) for smallest in smallest_shapes: @@ -912,7 +910,7 @@ def test_broadcastable_shape_can_generate_arbitrary_ndims(shape, max_dims, data) find_any( nps.broadcastable_shapes(shape, min_side=0, max_dims=max_dims, **kwargs), lambda x: len(x) == desired_ndim, - settings(max_examples=10 ** 6), + settings(max_examples=10**6), ) @@ -945,7 +943,46 @@ def test_mutually_broadcastable_shapes_can_generate_arbitrary_ndims( **kwargs, ), lambda x: {len(s) for s in x.input_shapes} == set(desired_ndims), - settings(max_examples=10 ** 6), + settings(max_examples=10**6), + ) + + +@settings(deadline=None) +@given( + base_shape=nps.array_shapes(min_dims=0, max_dims=3, min_side=0, max_side=2), + max_dims=st.integers(1, 4), +) +def test_mutually_broadcastable_shapes_can_generate_interesting_singletons( + base_shape, max_dims +): + + find_any( + nps.mutually_broadcastable_shapes( + num_shapes=2, + base_shape=base_shape, + min_side=0, + max_dims=max_dims, + ), + lambda x: any(a != b for a, b in zip(*(s[::-1] for s in x.input_shapes))), # type: ignore + ) + + +@pytest.mark.parametrize("base_shape", [(), (0,), (1,), (2,), (1, 2), (2, 1), (2, 2)]) +def test_mutually_broadcastable_shapes_can_generate_mirrored_singletons(base_shape): + def f(shapes: nps.BroadcastableShapes): + x, y = shapes.input_shapes + return x.count(1) == 1 and y.count(1) == 1 and x[::-1] == y + + find_any( + nps.mutually_broadcastable_shapes( + num_shapes=2, + base_shape=base_shape, + min_side=0, + max_side=3, + min_dims=2, + max_dims=2, + ), + f, ) @@ -1021,7 +1058,7 @@ def test_advanced_integer_index_minimizes_as_documented( np.testing.assert_array_equal(s, d) -@settings(deadline=None, max_examples=10) +@settings(deadline=None, max_examples=25) @given( shape=nps.array_shapes(min_dims=1, max_dims=2, min_side=1, max_side=3), data=st.data(), @@ -1030,7 +1067,7 @@ def test_advanced_integer_index_can_generate_any_pattern(shape, data): # ensures that generated index-arrays can be used to yield any pattern of elements from an array x = np.arange(np.product(shape)).reshape(shape) - target = data.draw( + target_array = data.draw( nps.arrays( shape=nps.array_shapes(min_dims=1, max_dims=2, min_side=1, max_side=2), elements=st.sampled_from(x.flatten()), @@ -1038,12 +1075,17 @@ def test_advanced_integer_index_can_generate_any_pattern(shape, data): ), label="target", ) + + def index_selects_values_in_order(index): + selected = x[index] + target(len(set(selected.flatten())), label="unique indices") + target(float(np.sum(target_array == selected)), label="elements correct") + return np.all(target_array == selected) + find_any( - nps.integer_array_indices( - shape, result_shape=st.just(target.shape), dtype=np.dtype("int8") - ), - lambda index: np.all(target == x[index]), - settings(max_examples=10 ** 6), + nps.integer_array_indices(shape, result_shape=st.just(target_array.shape)), + index_selects_values_in_order, + settings(max_examples=10**6), ) @@ -1088,22 +1130,41 @@ def test_basic_indices_can_generate_long_ellipsis(): ) ) def test_basic_indices_replaces_whole_axis_slices_with_ellipsis(idx): - # If ... is in the slice, it replaces all ,:, entries for this shape. + # `slice(None)` (aka `:`) is the only valid index for an axis of size + # zero, so if all dimensions are 0 then a `...` will replace all the + # slices because we generate `...` for entire contiguous runs of `:` assert slice(None) not in idx +def test_basic_indices_can_generate_indices_not_covering_all_dims(): + # These "flat indices" are skippable in the underlying BasicIndexStrategy, + # so we ensure we're definitely generating them for nps.basic_indices(). + find_any( + nps.basic_indices(shape=(3, 3, 3)), + lambda ix: ( + (not isinstance(ix, tuple) and ix != Ellipsis) + or (isinstance(ix, tuple) and Ellipsis not in ix and len(ix) < 3) + ), + ) + + @given( shape=nps.array_shapes(min_dims=0, max_side=4) | nps.array_shapes(min_dims=0, min_side=0, max_side=10), - min_dims=st.integers(0, 5), - allow_ellipsis=st.booleans(), allow_newaxis=st.booleans(), + allow_ellipsis=st.booleans(), data=st.data(), ) def test_basic_indices_generate_valid_indexers( - shape, min_dims, allow_ellipsis, allow_newaxis, data + shape, allow_newaxis, allow_ellipsis, data ): - max_dims = data.draw(st.none() | st.integers(min_dims, 32), label="max_dims") + min_dims = data.draw( + st.integers(0, 5 if allow_newaxis else len(shape)), label="min_dims" + ) + max_dims = data.draw( + st.none() | st.integers(min_dims, 32 if allow_newaxis else len(shape)), + label="max_dims", + ) indexer = data.draw( nps.basic_indices( shape, @@ -1114,6 +1175,7 @@ def test_basic_indices_generate_valid_indexers( ), label="indexer", ) + # Check that disallowed things are indeed absent if not allow_newaxis: if isinstance(indexer, tuple): @@ -1128,7 +1190,7 @@ def test_basic_indices_generate_valid_indexers( # If there's a zero in the shape, the array will have no elements. array = np.zeros(shape) assert array.size == 0 - elif np.prod(shape) <= 10 ** 5: + elif np.prod(shape) <= 10**5: # If it's small enough to instantiate, do so with distinct elements. array = np.arange(np.prod(shape)).reshape(shape) else: diff --git a/hypothesis-python/tests/numpy/test_gufunc.py b/hypothesis-python/tests/numpy/test_gufunc.py index 2df021ab90..9ec70f2dc7 100644 --- a/hypothesis-python/tests/numpy/test_gufunc.py +++ b/hypothesis-python/tests/numpy/test_gufunc.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import numpy as np import pytest @@ -20,6 +15,10 @@ from hypothesis import example, given, note, settings, strategies as st from hypothesis.errors import InvalidArgument from hypothesis.extra import numpy as nps +from hypothesis.extra._array_helpers import ( + _SIGNATURE, + _hypothesis_parse_gufunc_signature, +) from tests.common.debug import find_any, minimal @@ -55,7 +54,7 @@ def test_numpy_signature_parses(sig): np_sig = np.lib.function_base._parse_gufunc_signature(sig) try: - hy_sig = nps._hypothesis_parse_gufunc_signature(sig, all_checks=False) + hy_sig = _hypothesis_parse_gufunc_signature(sig, all_checks=False) assert np_sig == hy_sig_2_np_sig(hy_sig) except InvalidArgument: shape_too_long = any(len(s) > 32 for s in np_sig[0] + np_sig[1]) @@ -67,14 +66,14 @@ def test_numpy_signature_parses(sig): sig = in_ + "->" + out.split(",(")[0] np_sig = np.lib.function_base._parse_gufunc_signature(sig) if all(len(s) <= 32 for s in np_sig[0] + np_sig[1]): - hy_sig = nps._hypothesis_parse_gufunc_signature(sig, all_checks=False) + hy_sig = _hypothesis_parse_gufunc_signature(sig, all_checks=False) assert np_sig == hy_sig_2_np_sig(hy_sig) @use_signature_examples -@given(st.from_regex(nps._SIGNATURE)) +@given(st.from_regex(_SIGNATURE)) def test_hypothesis_signature_parses(sig): - hy_sig = nps._hypothesis_parse_gufunc_signature(sig, all_checks=False) + hy_sig = _hypothesis_parse_gufunc_signature(sig, all_checks=False) try: np_sig = np.lib.function_base._parse_gufunc_signature(sig) assert np_sig == hy_sig_2_np_sig(hy_sig) @@ -82,13 +81,13 @@ def test_hypothesis_signature_parses(sig): assert "?" in sig # We can always fix this up, and it should then always validate. sig = sig.replace("?", "") - hy_sig = nps._hypothesis_parse_gufunc_signature(sig, all_checks=False) + hy_sig = _hypothesis_parse_gufunc_signature(sig, all_checks=False) np_sig = np.lib.function_base._parse_gufunc_signature(sig) assert np_sig == hy_sig_2_np_sig(hy_sig) def test_frozen_dims_signature(): - nps._hypothesis_parse_gufunc_signature("(2),(3)->(4)") + _hypothesis_parse_gufunc_signature("(2),(3)->(4)") @st.composite @@ -129,7 +128,7 @@ def test_matmul_signature_can_exercise_all_combination_of_optional_dims( signature="(m?,n),(n,p?)->(m?,p?)", max_dims=0 ), lambda shapes: shapes == target_shapes, - settings(max_examples=10 ** 6), + settings(max_examples=10**6), ) @@ -161,7 +160,7 @@ def test_matmul_sig_shrinks_as_documented(min_dims, min_side, n_fixed, data): max_dims=max_dims, ) ) - note("(smallest_shapes, result): {}".format((smallest_shapes, result))) + note(f"(smallest_shapes, result): {(smallest_shapes, result)}") # if min_dims >= 1 then core dims are never excluded # otherwise, should shrink towards excluding all optional dims @@ -183,13 +182,13 @@ def einlabels(labels): assert "x" not in labels, "we reserve x for fixed-dimensions" return "..." + "".join(i if not i.isdigit() else "x" for i in labels) - gufunc_sig = nps._hypothesis_parse_gufunc_signature(gufunc_sig) + gufunc_sig = _hypothesis_parse_gufunc_signature(gufunc_sig) input_sig = ",".join(map(einlabels, gufunc_sig.input_shapes)) return input_sig + "->" + einlabels(gufunc_sig.result_shape) @pytest.mark.parametrize( - ("gufunc_sig"), + "gufunc_sig", [ param("()->()", id="unary sum"), param("(),()->()", id="binary sum"), diff --git a/hypothesis-python/tests/numpy/test_lazy_import.py b/hypothesis-python/tests/numpy/test_import.py similarity index 63% rename from hypothesis-python/tests/numpy/test_lazy_import.py rename to hypothesis-python/tests/numpy/test_import.py index ef9fe16bb2..495c60ec03 100644 --- a/hypothesis-python/tests/numpy/test_lazy_import.py +++ b/hypothesis-python/tests/numpy/test_import.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER SHOULD_NOT_IMPORT_NUMPY = """ import sys @@ -27,3 +22,17 @@ def test_hypothesis_is_not_the_first_to_import_numpy(testdir): # We only import numpy if the user did so first. result = testdir.runpytest(testdir.makepyfile(SHOULD_NOT_IMPORT_NUMPY)) result.assert_outcomes(passed=1, failed=0) + + +# We check the wildcard import works on the module level because that's the only +# place Python actually allows us to use them. +try: + from hypothesis.extra.numpy import * # noqa: F401, F403 + + star_import_works = True +except AttributeError: + star_import_works = False + + +def test_wildcard_import(): + assert star_import_works diff --git a/hypothesis-python/tests/numpy/test_narrow_floats.py b/hypothesis-python/tests/numpy/test_narrow_floats.py index 349084cb34..29a3d123d4 100644 --- a/hypothesis-python/tests/numpy/test_narrow_floats.py +++ b/hypothesis-python/tests/numpy/test_narrow_floats.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import numpy as np import pytest diff --git a/hypothesis-python/tests/numpy/test_randomness.py b/hypothesis-python/tests/numpy/test_randomness.py index 7942beaa43..6a9d724a83 100644 --- a/hypothesis-python/tests/numpy/test_randomness.py +++ b/hypothesis-python/tests/numpy/test_randomness.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import numpy as np diff --git a/hypothesis-python/tests/numpy/test_sampled_from.py b/hypothesis-python/tests/numpy/test_sampled_from.py index edc5681db7..6904b60877 100644 --- a/hypothesis-python/tests/numpy/test_sampled_from.py +++ b/hypothesis-python/tests/numpy/test_sampled_from.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import given from hypothesis.errors import InvalidArgument diff --git a/hypothesis-python/tests/pandas/__init__.py b/hypothesis-python/tests/pandas/__init__.py index e61e3c2872..fcb1ac6538 100644 --- a/hypothesis-python/tests/pandas/__init__.py +++ b/hypothesis-python/tests/pandas/__init__.py @@ -1,14 +1,9 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER diff --git a/hypothesis-python/tests/pandas/helpers.py b/hypothesis-python/tests/pandas/helpers.py index c0ca11edd6..40a4026ae8 100644 --- a/hypothesis-python/tests/pandas/helpers.py +++ b/hypothesis-python/tests/pandas/helpers.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import numpy as np diff --git a/hypothesis-python/tests/pandas/test_argument_validation.py b/hypothesis-python/tests/pandas/test_argument_validation.py index 2143ebe102..7980cefc45 100644 --- a/hypothesis-python/tests/pandas/test_argument_validation.py +++ b/hypothesis-python/tests/pandas/test_argument_validation.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from datetime import datetime @@ -21,6 +16,7 @@ from hypothesis.extra import pandas as pdst from tests.common.arguments import argument_validation_test, e +from tests.common.utils import checks_deprecated_behaviour BAD_ARGS = [ e(pdst.data_frames), @@ -69,6 +65,7 @@ e(pdst.indexes, elements="not a strategy"), e(pdst.indexes, elements=st.text(), dtype=float), e(pdst.indexes, elements=st.none(), dtype=int), + e(pdst.indexes, elements=st.integers(0, 10), dtype=st.sampled_from([int, float])), e(pdst.indexes, dtype=int, max_size=0, min_size=1), e(pdst.indexes, dtype=int, unique="true"), e(pdst.indexes, dtype=int, min_size="0"), @@ -96,3 +93,8 @@ def test_timestamp_as_datetime_bounds(dt): assert isinstance(dt, datetime) assert lo <= dt <= hi assert not isinstance(dt, pd.Timestamp) + + +@checks_deprecated_behaviour +def test_confusing_object_dtype_aliases(): + pdst.series(elements=st.tuples(st.integers()), dtype=tuple).example() diff --git a/hypothesis-python/tests/pandas/test_data_frame.py b/hypothesis-python/tests/pandas/test_data_frame.py index 7733aa8582..1a0789c332 100644 --- a/hypothesis-python/tests/pandas/test_data_frame.py +++ b/hypothesis-python/tests/pandas/test_data_frame.py @@ -1,19 +1,15 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import numpy as np +import pytest from hypothesis import HealthCheck, given, reject, settings, strategies as st from hypothesis.extra import numpy as npst, pandas as pdst @@ -243,3 +239,21 @@ def test_will_fill_missing_columns_in_tuple_row(df): ) def test_can_generate_unique_columns(df): assert set(df[0]) == set(range(10)) + + +@pytest.mark.skip(reason="Just works on Pandas 1.4, though the changelog is silent") +@pytest.mark.parametrize("dtype", [None, object]) +def test_expected_failure_from_omitted_object_dtype(dtype): + # See https://github.com/HypothesisWorks/hypothesis/issues/3133 + col = pdst.column(elements=st.sets(st.text(), min_size=1), dtype=dtype) + + @given(pdst.data_frames(columns=[col])) + def works_with_object_dtype(df): + pass + + if dtype is object: + works_with_object_dtype() + else: + assert dtype is None + with pytest.raises(ValueError, match="Maybe passing dtype=object would help"): + works_with_object_dtype() diff --git a/hypothesis-python/tests/pandas/test_indexes.py b/hypothesis-python/tests/pandas/test_indexes.py index af33c1b937..ac5caeba5b 100644 --- a/hypothesis-python/tests/pandas/test_indexes.py +++ b/hypothesis-python/tests/pandas/test_indexes.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import numpy as np import pandas @@ -29,7 +24,7 @@ def test_gets_right_dtype_for_empty_indices(ix): assert ix.dtype == np.dtype("int64") -@given(pdst.indexes(elements=st.integers(0, 2 ** 63 - 1), max_size=0)) +@given(pdst.indexes(elements=st.integers(0, 2**63 - 1), max_size=0)) def test_gets_right_dtype_for_empty_indices_with_elements(ix): assert ix.dtype == np.dtype("int64") @@ -51,8 +46,13 @@ def test_unique_indexes_of_many_small_values(ix): assert len(set(ix)) == len(ix) +@given(pdst.indexes(dtype="int8", name=st.just("test_name"))) +def test_name_passed_on(s): + assert s.name == "test_name" + + # Sizes that fit into an int64 without overflow -range_sizes = st.integers(0, 2 ** 63 - 1) +range_sizes = st.integers(0, 2**63 - 1) @given(range_sizes, range_sizes | st.none(), st.data()) diff --git a/hypothesis-python/tests/pandas/test_series.py b/hypothesis-python/tests/pandas/test_series.py index 06ad3b99ee..c796b8175d 100644 --- a/hypothesis-python/tests/pandas/test_series.py +++ b/hypothesis-python/tests/pandas/test_series.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import numpy as np import pandas @@ -61,3 +56,8 @@ def test_will_use_a_provided_elements_strategy(s): @given(pdst.series(dtype="int8", unique=True)) def test_unique_series_are_unique(s): assert len(s) == len(set(s)) + + +@given(pdst.series(dtype="int8", name=st.just("test_name"))) +def test_name_passed_on(s): + assert s.name == "test_name" diff --git a/hypothesis-python/tests/pytest/test_capture.py b/hypothesis-python/tests/pytest/test_capture.py index 70e50ba72f..33688aab85 100644 --- a/hypothesis-python/tests/pytest/test_capture.py +++ b/hypothesis-python/tests/pytest/test_capture.py @@ -1,23 +1,18 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import sys import pytest -from hypothesis.internal.compat import WINDOWS, escape_unicode_characters +from hypothesis.internal.compat import PYPY, WINDOWS, escape_unicode_characters pytest_plugins = "pytester" @@ -53,7 +48,7 @@ def test_emits_unicode(): @settings(verbosity=Verbosity.verbose) @given(text()) def test_should_emit_unicode(t): - assert all(ord(c) <= 1000 for c in t) + assert all(ord(c) <= 1000 for c in t), ascii(t) with pytest.raises(AssertionError): test_should_emit_unicode() """ @@ -61,7 +56,8 @@ def test_should_emit_unicode(t): @pytest.mark.xfail( WINDOWS, - reason=("Encoding issues in running the subprocess, possibly pytest's fault"), + reason="Encoding issues in running the subprocess, possibly pytest's fault", + strict=False, # It's possible, if rare, for this to pass on Windows too. ) def test_output_emitting_unicode(testdir, monkeypatch): monkeypatch.setenv("LC_ALL", "C") @@ -106,15 +102,16 @@ def test_healthcheck_traceback_is_hidden(testdir): timeout_token = ": FailedHealthCheck" def_line = get_line_num(def_token, result) timeout_line = get_line_num(timeout_token, result) - expected = 6 if sys.version_info[:2] < (3, 8) else 7 + seven_min = (3, 9) if PYPY else (3, 8) + expected = 6 if sys.version_info[:2] < seven_min else 7 assert timeout_line - def_line == expected COMPOSITE_IS_NOT_A_TEST = """ -from hypothesis.strategies import composite +from hypothesis.strategies import composite, none @composite def test_data_factory(draw): - pass + return draw(none()) """ diff --git a/hypothesis-python/tests/pytest/test_checks.py b/hypothesis-python/tests/pytest/test_checks.py index cd66ea2ddf..5fedb187eb 100644 --- a/hypothesis-python/tests/pytest/test_checks.py +++ b/hypothesis-python/tests/pytest/test_checks.py @@ -1,26 +1,21 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER TEST_DECORATORS_ALONE = """ import hypothesis -from hypothesis.strategies import composite +from hypothesis.strategies import composite, none @composite def test_composite_is_not_a_test(draw): # This strategy will be instantiated, but no draws == no calls. - assert False + return draw(none()) @hypothesis.seed(0) def test_seed_without_given_fails(): @@ -38,4 +33,6 @@ def test_repro_without_given_fails(): def test_decorators_without_given_should_fail(testdir): script = testdir.makepyfile(TEST_DECORATORS_ALONE) - testdir.runpytest(script).assert_outcomes(failed=4) + result = testdir.runpytest(script) + result.assert_outcomes(failed=4) + assert "pytest_runtest_call" not in "\n".join(result.outlines) diff --git a/hypothesis-python/tests/pytest/test_compat.py b/hypothesis-python/tests/pytest/test_compat.py index edf67ba544..3687aa73fb 100644 --- a/hypothesis-python/tests/pytest/test_compat.py +++ b/hypothesis-python/tests/pytest/test_compat.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest diff --git a/hypothesis-python/tests/pytest/test_doctest.py b/hypothesis-python/tests/pytest/test_doctest.py index cb20d4b0da..ca2c34ff49 100644 --- a/hypothesis-python/tests/pytest/test_doctest.py +++ b/hypothesis-python/tests/pytest/test_doctest.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER pytest_plugins = "pytester" diff --git a/hypothesis-python/tests/pytest/test_fixtures.py b/hypothesis-python/tests/pytest/test_fixtures.py index 5700b4fc70..368911d395 100644 --- a/hypothesis-python/tests/pytest/test_fixtures.py +++ b/hypothesis-python/tests/pytest/test_fixtures.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from unittest.mock import Mock, create_autospec @@ -73,7 +68,7 @@ def test_can_inject_mock_via_fixture(mock_fixture, xs): succeeds. If this test fails, then we know we've run the test body instead of the mock. """ - assert False + raise AssertionError @given(integers()) @@ -180,3 +175,38 @@ def test_override_fixture(event_loop, x): def test_given_plus_overridden_fixture(testdir): script = testdir.makepyfile(TESTSCRIPT_OVERRIDE_FIXTURE) testdir.runpytest(script, "-Werror").assert_outcomes(passed=1, failed=0) + + +TESTSCRIPT_FIXTURE_THEN_GIVEN = """ +import pytest +from hypothesis import given, strategies as st + +@given(x=st.integers()) +@pytest.fixture() +def test(x): + pass +""" + + +def test_given_fails_if_already_decorated_with_fixture(testdir): + script = testdir.makepyfile(TESTSCRIPT_FIXTURE_THEN_GIVEN) + testdir.runpytest(script).assert_outcomes(failed=1) + + +TESTSCRIPT_GIVEN_THEN_FIXTURE = """ +import pytest +from hypothesis import given, strategies as st + +@pytest.fixture() +@given(x=st.integers()) +def test(x): + pass +""" + + +def test_fixture_errors_if_already_decorated_with_given(testdir): + script = testdir.makepyfile(TESTSCRIPT_GIVEN_THEN_FIXTURE) + if int(pytest.__version__.split(".")[0]) > 5: + testdir.runpytest(script).assert_outcomes(errors=1) + else: + testdir.runpytest(script).assert_outcomes(error=1) diff --git a/hypothesis-python/tests/pytest/test_junit.py b/hypothesis-python/tests/pytest/test_junit.py new file mode 100644 index 0000000000..709a8579b3 --- /dev/null +++ b/hypothesis-python/tests/pytest/test_junit.py @@ -0,0 +1,73 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import xml.etree.ElementTree as ET +from pathlib import Path + +pytest_plugins = "pytester" + + +TESTSUITE = """ +from hypothesis import given +from hypothesis.strategies import integers + +@given(integers()) +def test_valid(x): + assert x == x + +@given(integers()) +def test_invalid(x): + assert x != x +""" + + +def _run_and_get_junit(testdir, *args): + script = testdir.makepyfile(TESTSUITE) + testdir.runpytest(script, "--junit-xml=out.xml", *args) + return ET.parse(Path(testdir.tmpdir) / "out.xml").getroot() + + +def _findall_from_root(junit_xml, path): + if junit_xml.tag == "testsuites": + return junit_xml.findall(f"./testsuite/{path}") + # This case only exists for tests against Pytest before 5.1.0; + # see https://github.com/pytest-dev/pytest/commit/a43ba78d3bde + return junit_xml.findall(f"./{path}") + + +def suite_properties_ok(junit_xml): + # Check whether is included in . This is currently not + # the case when using pytest-xdist, which is a shame, but we can live with it. + testsuite_props = _findall_from_root(junit_xml, "properties") + return len(testsuite_props) == 1 and { + prop.get("name") for prop in testsuite_props[0].findall("property") + } == { + "hypothesis-statistics-test_outputs_valid_xunit2.py::test_valid", + "hypothesis-statistics-test_outputs_valid_xunit2.py::test_invalid", + } + + +def test_outputs_valid_xunit2(testdir): + # The thing we really care about with pytest-xdist + junitxml is that we don't + # break xunit2 compatibility by putting inside . + junit_xml = _run_and_get_junit(testdir) + testcase_props = _findall_from_root(junit_xml, "testcase/properties") + assert len(testcase_props) == 0 + # Check whether is included in + assert suite_properties_ok(junit_xml) + + +def test_outputs_valid_xunit2_with_xdist(testdir): + junit_xml = _run_and_get_junit(testdir, "-n2") + testcase_props = _findall_from_root(junit_xml, "testcase/properties") + assert len(testcase_props) == 0 + # If is included in , this assertion will fail. + # That would be a GOOD THING, and we would remove the `not` to prevent regressions. + assert not suite_properties_ok(junit_xml) diff --git a/hypothesis-python/tests/pytest/test_mark.py b/hypothesis-python/tests/pytest/test_mark.py index 2ee0fdcdf3..265b3b3c40 100644 --- a/hypothesis-python/tests/pytest/test_mark.py +++ b/hypothesis-python/tests/pytest/test_mark.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER pytest_plugins = "pytester" @@ -31,7 +26,9 @@ def test_bar(): def test_can_select_mark(testdir): script = testdir.makepyfile(TESTSUITE) - result = testdir.runpytest(script, "--verbose", "--strict", "-m", "hypothesis") + result = testdir.runpytest( + script, "--verbose", "--strict-markers", "-m", "hypothesis" + ) out = "\n".join(result.stdout.lines) assert "1 passed, 1 deselected" in out @@ -53,6 +50,8 @@ def test_bar(self): def test_can_select_mark_on_unittest(testdir): script = testdir.makepyfile(UNITTEST_TESTSUITE) - result = testdir.runpytest(script, "--verbose", "--strict", "-m", "hypothesis") + result = testdir.runpytest( + script, "--verbose", "--strict-markers", "-m", "hypothesis" + ) out = "\n".join(result.stdout.lines) assert "1 passed, 1 deselected" in out diff --git a/hypothesis-python/tests/pytest/test_parametrized_db_keys.py b/hypothesis-python/tests/pytest/test_parametrized_db_keys.py index 23692e9981..ad19c36f4f 100644 --- a/hypothesis-python/tests/pytest/test_parametrized_db_keys.py +++ b/hypothesis-python/tests/pytest/test_parametrized_db_keys.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER DB_KEY_TESTCASE = """ from hypothesis import settings, given diff --git a/hypothesis-python/tests/pytest/test_profiles.py b/hypothesis-python/tests/pytest/test_profiles.py index 4150b6a605..6486777c7a 100644 --- a/hypothesis-python/tests/pytest/test_profiles.py +++ b/hypothesis-python/tests/pytest/test_profiles.py @@ -1,21 +1,16 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest +from _hypothesis_pytestplugin import LOAD_PROFILE_OPTION -from hypothesis.extra.pytestplugin import LOAD_PROFILE_OPTION from hypothesis.version import __version__ pytest_plugins = "pytester" diff --git a/hypothesis-python/tests/pytest/test_pytest_detection.py b/hypothesis-python/tests/pytest/test_pytest_detection.py index 9319d30c82..9567b45c71 100644 --- a/hypothesis-python/tests/pytest/test_pytest_detection.py +++ b/hypothesis-python/tests/pytest/test_pytest_detection.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import subprocess import sys @@ -33,3 +28,16 @@ def test_is_not_running_under_pytest(tmpdir): pyfile = tmpdir.join("test.py") pyfile.write(FILE_TO_RUN) subprocess.check_call([sys.executable, str(pyfile)]) + + +DOES_NOT_IMPORT_HYPOTHESIS = """ +import sys + +def test_pytest_plugin_does_not_import_hypothesis(): + assert "hypothesis" not in sys.modules +""" + + +def test_plugin_does_not_import_pytest(testdir): + testdir.makepyfile(DOES_NOT_IMPORT_HYPOTHESIS) + testdir.runpytest_subprocess().assert_outcomes(passed=1) diff --git a/hypothesis-python/tests/pytest/test_reporting.py b/hypothesis-python/tests/pytest/test_reporting.py index b4b5174d86..a7cb778752 100644 --- a/hypothesis-python/tests/pytest/test_reporting.py +++ b/hypothesis-python/tests/pytest/test_reporting.py @@ -1,17 +1,14 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER + +import pytest pytest_plugins = "pytester" @@ -38,3 +35,24 @@ def test_runs_reporting_hook(testdir): assert "Captured stdout call" not in out assert "Falsifying example" in out assert result.ret != 0 + + +TEST_EXCEPTIONGROUP = """ +from hypothesis import given, strategies as st + +@given(x=st.booleans()) +def test_fuzz_sorted(x): + raise ValueError if x else TypeError +""" + + +@pytest.mark.parametrize("tb", ["auto", "long", "short", "native"]) +def test_no_missing_reports(testdir, tb): + script = testdir.makepyfile(TEST_EXCEPTIONGROUP) + result = testdir.runpytest(script, f"--tb={tb}") + out = "\n".join(result.stdout.lines) + # If the False case is missing, that means we're not printing exception info. + # See https://github.com/HypothesisWorks/hypothesis/issues/3430 With --tb=native, + # we should show the full ExceptionGroup with *both* errors. + assert "x=False" in out + assert "x=True" in out or tb != "native" diff --git a/hypothesis-python/tests/pytest/test_runs.py b/hypothesis-python/tests/pytest/test_runs.py index e45f4eb351..cd1a147f38 100644 --- a/hypothesis-python/tests/pytest/test_runs.py +++ b/hypothesis-python/tests/pytest/test_runs.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import given from hypothesis.strategies import integers diff --git a/hypothesis-python/tests/pytest/test_seeding.py b/hypothesis-python/tests/pytest/test_seeding.py index 26dab095e8..a31170cae3 100644 --- a/hypothesis-python/tests/pytest/test_seeding.py +++ b/hypothesis-python/tests/pytest/test_seeding.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import re @@ -48,7 +43,7 @@ def test_runs_repeatably_when_seed_is_set(seed, testdir): results = [ testdir.runpytest( - script, "--verbose", "--strict", "--hypothesis-seed", str(seed) + script, "--verbose", "--strict-markers", f"--hypothesis-seed={seed}", "-rN" ) for _ in range(2) ] @@ -94,7 +89,7 @@ def test_repeats_healthcheck_when_following_seed_instruction(testdir, tmpdir): script = testdir.makepyfile(health_check_test) - initial = testdir.runpytest(script, "--verbose", "--strict") + initial = testdir.runpytest(script, "--verbose", "--strict-markers") match = CONTAINS_SEED_INSTRUCTION.search("\n".join(initial.stdout.lines)) initial_output = "\n".join(initial.stdout.lines) @@ -102,12 +97,14 @@ def test_repeats_healthcheck_when_following_seed_instruction(testdir, tmpdir): match = CONTAINS_SEED_INSTRUCTION.search(initial_output) assert match is not None - rerun = testdir.runpytest(script, "--verbose", "--strict", match.group(0)) + rerun = testdir.runpytest(script, "--verbose", "--strict-markers", match.group(0)) rerun_output = "\n".join(rerun.stdout.lines) assert "FailedHealthCheck" in rerun_output assert "--hypothesis-seed" not in rerun_output - rerun2 = testdir.runpytest(script, "--verbose", "--strict", "--hypothesis-seed=10") + rerun2 = testdir.runpytest( + script, "--verbose", "--strict-markers", "--hypothesis-seed=10" + ) rerun2_output = "\n".join(rerun2.stdout.lines) assert "FailedHealthCheck" not in rerun2_output diff --git a/hypothesis-python/tests/pytest/test_skipping.py b/hypothesis-python/tests/pytest/test_skipping.py index 33b7c21488..af92d3aae4 100644 --- a/hypothesis-python/tests/pytest/test_skipping.py +++ b/hypothesis-python/tests/pytest/test_skipping.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER pytest_plugins = "pytester" @@ -28,7 +23,7 @@ def test_to_be_skipped(xs): if xs == 0: pytest.skip() # But the pytest 3.0 internals don't have such an exception, so we keep - # going and raise a MultipleFailures error. Ah well. + # going and raise a BaseExceptionGroup error. Ah well. else: assert xs == 0 """ @@ -39,6 +34,30 @@ def test_no_falsifying_example_if_pytest_skip(testdir): continue running the test and shrink process, nor should it print anything about falsifying examples.""" script = testdir.makepyfile(PYTEST_TESTSUITE) - result = testdir.runpytest(script, "--verbose", "--strict", "-m", "hypothesis") + result = testdir.runpytest( + script, "--verbose", "--strict-markers", "-m", "hypothesis" + ) out = "\n".join(result.stdout.lines) assert "Falsifying example" not in out + + +def test_issue_3453_regression(testdir): + """If ``pytest.skip() is called during a test, Hypothesis should not + continue running the test and shrink process, nor should it print anything + about falsifying examples.""" + script = testdir.makepyfile( + """ +from hypothesis import example, given, strategies as st +import pytest + +@given(value=st.none()) +@example("hello") +@example("goodbye") +def test_skip_on_first_skipping_example(value): + assert value is not None + assert value != "hello" # queue up a non-skip error which must be discarded + pytest.skip() +""" + ) + result = testdir.runpytest(script, "--tb=native") + result.assert_outcomes(skipped=1) diff --git a/hypothesis-python/tests/pytest/test_statistics.py b/hypothesis-python/tests/pytest/test_statistics.py index 73dc3ab1a1..8ec8368dfb 100644 --- a/hypothesis-python/tests/pytest/test_statistics.py +++ b/hypothesis-python/tests/pytest/test_statistics.py @@ -1,23 +1,15 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER - -from distutils.version import LooseVersion import pytest - -from hypothesis.extra.pytestplugin import PRINT_STATISTICS_OPTION +from _hypothesis_pytestplugin import PRINT_STATISTICS_OPTION pytest_plugins = "pytester" @@ -76,7 +68,9 @@ def test_prints_statistics_given_option_with_junitxml(testdir): assert "< 10% of examples satisfied assumptions" in out -@pytest.mark.skipif(LooseVersion(pytest.__version__) < "5.4.0", reason="too old") +@pytest.mark.skipif( + tuple(map(int, pytest.__version__.split(".")[:2])) < (5, 4), reason="too old" +) def test_prints_statistics_given_option_under_xdist_with_junitxml(testdir): out = get_output( testdir, TESTSUITE, PRINT_STATISTICS_OPTION, "-n", "2", "--junit-xml=out.xml" diff --git a/hypothesis-python/tests/quality/__init__.py b/hypothesis-python/tests/quality/__init__.py index e61e3c2872..fcb1ac6538 100644 --- a/hypothesis-python/tests/quality/__init__.py +++ b/hypothesis-python/tests/quality/__init__.py @@ -1,14 +1,9 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER diff --git a/hypothesis-python/tests/quality/test_deferred_strategies.py b/hypothesis-python/tests/quality/test_deferred_strategies.py index fc638b615e..e395b9749c 100644 --- a/hypothesis-python/tests/quality/test_deferred_strategies.py +++ b/hypothesis-python/tests/quality/test_deferred_strategies.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesis import strategies as st diff --git a/hypothesis-python/tests/quality/test_discovery_ability.py b/hypothesis-python/tests/quality/test_discovery_ability.py index f36a651d3e..0e6d9c0378 100644 --- a/hypothesis-python/tests/quality/test_discovery_ability.py +++ b/hypothesis-python/tests/quality/test_discovery_ability.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER # -*- coding: utf-8 -*- """Statistical tests over the forms of the distributions in the standard set of @@ -29,9 +24,10 @@ from hypothesis import HealthCheck, settings as Settings from hypothesis.errors import UnsatisfiedAssumption -from hypothesis.internal import reflection as reflection +from hypothesis.internal import reflection from hypothesis.internal.conjecture.engine import ConjectureRunner from hypothesis.strategies import ( + binary, booleans, floats, integers, @@ -87,7 +83,8 @@ def test_function(data): data.mark_interesting() successes = 0 - for _ in range(RUNS): + actual_runs = 0 + for actual_runs in range(1, RUNS + 1): # We choose the max_examples a bit larger than default so that we # run at least 100 examples outside of the small example generation # part of the generation phase. @@ -104,14 +101,22 @@ def test_function(data): successes += 1 if successes >= required_runs: return + + # If we reach a point where it's impossible to hit our target even + # if every remaining attempt were to succeed, give up early and + # report failure. + if (required_runs - successes) > (RUNS - actual_runs): + break + event = reflection.get_pretty_function_description(predicate) if condition is not None: event += "|" event += condition_string raise HypothesisFalsified( - f"P({event}) ~ {successes} / {RUNS} = {successes / RUNS:.2f} < " - f"{required_runs / RUNS:.2f} rejected" + f"P({event}) ~ {successes} / {actual_runs} = " + f"{successes / actual_runs:.2f} < {required_runs / RUNS:.2f}; " + "rejected" ) return run_test @@ -147,6 +152,8 @@ def long_list(xs): text(), lambda x: any(ord(c) > 127 for c in x), condition=lambda x: len(x) <= 3 ) +test_can_produce_large_binary_strings = define_test(binary(), lambda x: len(x) > 20) + test_can_produce_positive_infinity = define_test(floats(), lambda x: x == math.inf) test_can_produce_negative_infinity = define_test(floats(), lambda x: x == -math.inf) @@ -202,7 +209,7 @@ def distorted(x): ) test_ints_can_occasionally_be_really_large = define_test( - integers(), lambda t: t >= 2 ** 63 + integers(), lambda t: t >= 2**63 ) test_mixing_is_sometimes_distorted = define_test( diff --git a/hypothesis-python/tests/quality/test_float_shrinking.py b/hypothesis-python/tests/quality/test_float_shrinking.py index 0d4b63f645..3e545dc427 100644 --- a/hypothesis-python/tests/quality/test_float_shrinking.py +++ b/hypothesis-python/tests/quality/test_float_shrinking.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest @@ -52,12 +47,12 @@ def test_shrinks_downwards_to_integers(f): @example(1) -@given(st.integers(1, 2 ** 16 - 1)) +@given(st.integers(1, 2**16 - 1)) @settings(deadline=None, suppress_health_check=HealthCheck.all(), max_examples=10) def test_shrinks_downwards_to_integers_when_fractional(b): g = minimal( st.floats(), - lambda x: assume((0 < x < (2 ** 53)) and int(x) != x) and x >= b, - settings=settings(verbosity=Verbosity.quiet, max_examples=10 ** 6), + lambda x: assume((0 < x < (2**53)) and int(x) != x) and x >= b, + settings=settings(verbosity=Verbosity.quiet, max_examples=10**6), ) assert g == b + 0.5 diff --git a/hypothesis-python/tests/quality/test_integers.py b/hypothesis-python/tests/quality/test_integers.py index 05dc2907e7..647fc55b51 100644 --- a/hypothesis-python/tests/quality/test_integers.py +++ b/hypothesis-python/tests/quality/test_integers.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from random import Random @@ -28,7 +23,7 @@ ) from hypothesis.internal.conjecture.data import ConjectureData, Status, StopTest from hypothesis.internal.conjecture.engine import ConjectureRunner -from hypothesis.strategies._internal.numbers import WideRangeIntStrategy +from hypothesis.internal.conjecture.utils import INT_SIZES @st.composite @@ -112,7 +107,24 @@ def f(data): # have power of two sizes, so it may be up to a factor of two more than # that. bits_needed = 1 + n.bit_length() - actual_bits_needed = min(s for s in WideRangeIntStrategy.sizes if s >= bits_needed) + actual_bits_needed = min(s for s in INT_SIZES if s >= bits_needed) bytes_needed = actual_bits_needed // 8 # 3 extra bytes: two for the sampler, one for the capping value. assert len(v.buffer) == 3 + bytes_needed + + +def test_generates_boundary_values_even_when_unlikely(): + r = Random() + trillion = 10**12 + strat = st.integers(-trillion, trillion) + boundary_vals = {-trillion, -trillion + 1, trillion - 1, trillion} + for _ in range(10_000): + buffer = bytes(r.randrange(0, 255) for _ in range(1000)) + val = ConjectureData.for_buffer(buffer).draw(strat) + boundary_vals.discard(val) + if not boundary_vals: + break + else: + raise AssertionError( + f"Expected to see all boundary vals, but still have {boundary_vals}" + ) diff --git a/hypothesis-python/tests/quality/test_normalization.py b/hypothesis-python/tests/quality/test_normalization.py index 1be13903e7..8355380045 100644 --- a/hypothesis-python/tests/quality/test_normalization.py +++ b/hypothesis-python/tests/quality/test_normalization.py @@ -1,19 +1,15 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from itertools import islice +from random import Random import pytest @@ -56,10 +52,6 @@ def test_function(data): @pytest.mark.parametrize("strategy", [st.emails(), st.complex_numbers()], ids=repr) def test_harder_strategies_normalize_to_minimal(strategy, normalize_kwargs): - import random - - random.seed(0) - def test_function(data): try: v = data.draw(strategy) @@ -68,4 +60,4 @@ def test_function(data): data.output = repr(v) data.mark_interesting() - dfas.normalize(repr(strategy), test_function, **normalize_kwargs) + dfas.normalize(repr(strategy), test_function, random=Random(0), **normalize_kwargs) diff --git a/hypothesis-python/tests/quality/test_poisoned_lists.py b/hypothesis-python/tests/quality/test_poisoned_lists.py index 26eed9b88d..1d5c8d5f96 100644 --- a/hypothesis-python/tests/quality/test_poisoned_lists.py +++ b/hypothesis-python/tests/quality/test_poisoned_lists.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from random import Random @@ -28,7 +23,7 @@ class Poisoned(SearchStrategy): def __init__(self, poison_chance): - SearchStrategy.__init__(self) + super().__init__() self.__poison_chance = poison_chance self.__ints = st.integers(0, 10) @@ -41,7 +36,7 @@ def do_draw(self, data): class LinearLists(SearchStrategy): def __init__(self, elements, size): - SearchStrategy.__init__(self) + super().__init__() self.__length = st.integers(0, size) self.__elements = elements @@ -51,8 +46,8 @@ def do_draw(self, data): class Matrices(SearchStrategy): def __init__(self, elements, size): - SearchStrategy.__init__(self) - self.__length = st.integers(0, ceil(size ** 0.5)) + super().__init__() + self.__length = st.integers(0, ceil(size**0.5)) self.__elements = elements def do_draw(self, data): @@ -62,7 +57,7 @@ def do_draw(self, data): return [data.draw(self.__elements) for _ in range(n * m)] -LOTS = 10 ** 6 +LOTS = 10**6 TRIAL_SETTINGS = settings(max_examples=LOTS, database=None) diff --git a/hypothesis-python/tests/quality/test_poisoned_trees.py b/hypothesis-python/tests/quality/test_poisoned_trees.py index 1750cd7364..60b7ec5f3c 100644 --- a/hypothesis-python/tests/quality/test_poisoned_trees.py +++ b/hypothesis-python/tests/quality/test_poisoned_trees.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from random import Random @@ -24,7 +19,7 @@ POISON = "POISON" -MAX_INT = 2 ** 32 - 1 +MAX_INT = 2**32 - 1 class PoisonedTree(SearchStrategy): @@ -35,7 +30,7 @@ class PoisonedTree(SearchStrategy): """ def __init__(self, p): - SearchStrategy.__init__(self) + super().__init__() self.__p = p def do_draw(self, data): @@ -53,7 +48,7 @@ def do_draw(self, data): return (None,) -LOTS = 10 ** 6 +LOTS = 10**6 TEST_SETTINGS = settings( diff --git a/hypothesis-python/tests/quality/test_shrink_quality.py b/hypothesis-python/tests/quality/test_shrink_quality.py index ba2a0729bd..1586289dd0 100644 --- a/hypothesis-python/tests/quality/test_shrink_quality.py +++ b/hypothesis-python/tests/quality/test_shrink_quality.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from collections import OrderedDict, namedtuple from fractions import Fraction @@ -46,6 +41,19 @@ def test_integers_from_minimizes_leftwards(): assert minimal(integers(min_value=101)) == 101 +def test_minimize_bounded_integers_to_zero(): + assert minimal(integers(-10, 10)) == 0 + + +def test_minimize_bounded_integers_to_positive(): + zero = 0 + + def not_zero(x): + return x != zero + + assert minimal(integers(-10, 10).filter(not_zero)) == 1 + + def test_minimal_fractions_1(): assert minimal(fractions()) == Fraction(0) @@ -165,20 +173,20 @@ def test_dictionary(dict_class): def test_minimize_single_element_in_silly_large_int_range(): - ir = integers(-(2 ** 256), 2 ** 256) - assert minimal(ir, lambda x: x >= -(2 ** 255)) == 0 + ir = integers(-(2**256), 2**256) + assert minimal(ir, lambda x: x >= -(2**255)) == 0 def test_minimize_multiple_elements_in_silly_large_int_range(): desired_result = [0] * 20 - ir = integers(-(2 ** 256), 2 ** 256) + ir = integers(-(2**256), 2**256) x = minimal(lists(ir), lambda x: len(x) >= 20, timeout_after=20) assert x == desired_result def test_minimize_multiple_elements_in_silly_large_int_range_min_is_not_dupe(): - ir = integers(0, 2 ** 256) + ir = integers(0, 2**256) target = list(range(20)) x = minimal( diff --git a/hypothesis-python/tests/quality/test_shrinking_order.py b/hypothesis-python/tests/quality/test_shrinking_order.py index 47ed9f9d6b..b9692caf22 100644 --- a/hypothesis-python/tests/quality/test_shrinking_order.py +++ b/hypothesis-python/tests/quality/test_shrinking_order.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from itertools import islice from random import Random diff --git a/hypothesis-python/tests/quality/test_zig_zagging.py b/hypothesis-python/tests/quality/test_zig_zagging.py index f50d154952..440d9d3f58 100644 --- a/hypothesis-python/tests/quality/test_zig_zagging.py +++ b/hypothesis-python/tests/quality/test_zig_zagging.py @@ -1,20 +1,15 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER -import random from math import log +from random import Random from hypothesis import ( HealthCheck, @@ -77,8 +72,6 @@ def problem(draw): def test_avoids_zig_zag_trap(p): b, marker, lower_bound = p - random.seed(0) - n_bits = 8 * (len(b) + 1) def test_function(data): @@ -95,6 +88,7 @@ def test_function(data): test_function, database_key=None, settings=settings(base_settings, phases=(Phase.generate, Phase.shrink)), + random=Random(0), ) runner.cached_test_function(b + bytes([0]) + b + bytes([1]) + marker) diff --git a/hypothesis-python/tests/redis/__init__.py b/hypothesis-python/tests/redis/__init__.py index e61e3c2872..fcb1ac6538 100644 --- a/hypothesis-python/tests/redis/__init__.py +++ b/hypothesis-python/tests/redis/__init__.py @@ -1,14 +1,9 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER diff --git a/hypothesis-python/tests/redis/test_redis_exampledatabase.py b/hypothesis-python/tests/redis/test_redis_exampledatabase.py index b6d4ebb3ae..f51a89f517 100644 --- a/hypothesis-python/tests/redis/test_redis_exampledatabase.py +++ b/hypothesis-python/tests/redis/test_redis_exampledatabase.py @@ -1,26 +1,48 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER +import pytest from fakeredis import FakeRedis from hypothesis import strategies as st from hypothesis.database import InMemoryExampleDatabase +from hypothesis.errors import InvalidArgument from hypothesis.extra.redis import RedisExampleDatabase from hypothesis.stateful import Bundle, RuleBasedStateMachine, rule +@pytest.mark.parametrize( + "kw", + [ + {"redis": "not a redis instance"}, + {"redis": FakeRedis(), "expire_after": 10}, # not a timedelta + {"redis": FakeRedis(), "key_prefix": "not a bytestring"}, + ], +) +def test_invalid_args_raise(kw): + with pytest.raises(InvalidArgument): + RedisExampleDatabase(**kw) + + +def test_all_methods(): + db = RedisExampleDatabase(FakeRedis()) + db.save(b"key1", b"value") + assert list(db.fetch(b"key1")) == [b"value"] + db.move(b"key1", b"key2", b"value") + assert list(db.fetch(b"key1")) == [] + assert list(db.fetch(b"key2")) == [b"value"] + db.delete(b"key2", b"value") + assert list(db.fetch(b"key2")) == [] + db.delete(b"key2", b"unknown value") + + class DatabaseComparison(RuleBasedStateMachine): def __init__(self): super().__init__() diff --git a/hypothesis-python/tests/typing_extensions/__init__.py b/hypothesis-python/tests/typing_extensions/__init__.py index e61e3c2872..fcb1ac6538 100644 --- a/hypothesis-python/tests/typing_extensions/__init__.py +++ b/hypothesis-python/tests/typing_extensions/__init__.py @@ -1,14 +1,9 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER diff --git a/hypothesis-python/tests/typing_extensions/test_backported_types.py b/hypothesis-python/tests/typing_extensions/test_backported_types.py index 88f05af6eb..a6f3512de0 100644 --- a/hypothesis-python/tests/typing_extensions/test_backported_types.py +++ b/hypothesis-python/tests/typing_extensions/test_backported_types.py @@ -1,26 +1,37 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import collections -from typing import Union +import sys +from typing import Callable, DefaultDict, Dict, List, NewType, Type, Union import pytest -from typing_extensions import DefaultDict, Literal, NewType, Type, TypedDict +from typing_extensions import ( + Annotated, + Concatenate, + Literal, + NotRequired, + ParamSpec, + Required, + TypedDict, + TypeGuard, +) from hypothesis import assume, given, strategies as st +from hypothesis.errors import InvalidArgument from hypothesis.strategies import from_type +from hypothesis.strategies._internal.types import NON_RUNTIME_TYPES + +from tests.common.debug import assert_all_examples, find_any + +# See also nocover/test_type_lookup.py @pytest.mark.parametrize("value", ["dog", b"goldfish", 42, 63.4, -80.5, False]) @@ -38,6 +49,9 @@ def test_typing_extensions_Literal_nested(data): (lit[lit[1], 2], {1, 2}), (lit[1, lit[2], 3], {1, 2, 3}), (lit[lit[lit[1], lit[2]], lit[lit[3], lit[4]]], {1, 2, 3, 4}), + # See https://github.com/HypothesisWorks/hypothesis/pull/2886 + (Union[Literal["hamster"], Literal["bunny"]], {"hamster", "bunny"}), + (Union[lit[lit[1], lit[2]], lit[lit[3], lit[4]]], {1, 2, 3, 4}), ] literal_type, flattened_literals = data.draw(st.sampled_from(values)) assert data.draw(st.from_type(literal_type)) in flattened_literals @@ -78,3 +92,204 @@ def test_defaultdict(ex): assume(ex) assert all(isinstance(elem, int) for elem in ex) assert all(isinstance(elem, int) for elem in ex.values()) + + +@pytest.mark.parametrize( + "annotated_type,expected_strategy_repr", + [ + (Annotated[int, "foo"], "integers()"), + (Annotated[List[float], "foo"], "lists(floats())"), + (Annotated[Annotated[str, "foo"], "bar"], "text()"), + ( + Annotated[Annotated[List[Dict[str, bool]], "foo"], "bar"], + "lists(dictionaries(keys=text(), values=booleans()))", + ), + ], +) +def test_typing_extensions_Annotated(annotated_type, expected_strategy_repr): + assert repr(st.from_type(annotated_type)) == expected_strategy_repr + + +PositiveInt = Annotated[int, st.integers(min_value=1)] +MoreThenTenInt = Annotated[PositiveInt, st.integers(min_value=10 + 1)] +WithTwoStrategies = Annotated[int, st.integers(), st.none()] +ExtraAnnotationNoStrategy = Annotated[PositiveInt, "metadata"] + + +def arg_positive(x: PositiveInt): + assert x > 0 + + +def arg_more_than_ten(x: MoreThenTenInt): + assert x > 10 + + +@given(st.data()) +def test_annotated_positive_int(data): + data.draw(st.builds(arg_positive)) + + +@given(st.data()) +def test_annotated_more_than_ten(data): + data.draw(st.builds(arg_more_than_ten)) + + +@given(st.data()) +def test_annotated_with_two_strategies(data): + assert data.draw(st.from_type(WithTwoStrategies)) is None + + +@given(st.data()) +def test_annotated_extra_metadata(data): + assert data.draw(st.from_type(ExtraAnnotationNoStrategy)) > 0 + + +@pytest.mark.parametrize("non_runtime_type", NON_RUNTIME_TYPES) +def test_non_runtime_type_cannot_be_resolved(non_runtime_type): + strategy = st.from_type(non_runtime_type) + with pytest.raises( + InvalidArgument, match="there is no such thing as a runtime instance" + ): + strategy.example() + + +@pytest.mark.parametrize("non_runtime_type", NON_RUNTIME_TYPES) +def test_non_runtime_type_cannot_be_registered(non_runtime_type): + with pytest.raises( + InvalidArgument, match="there is no such thing as a runtime instance" + ): + st.register_type_strategy(non_runtime_type, st.none()) + + +@pytest.mark.skipif(sys.version_info <= (3, 7), reason="requires python3.8 or higher") +def test_callable_with_concatenate(): + P = ParamSpec("P") + func_type = Callable[Concatenate[int, P], None] + strategy = st.from_type(func_type) + with pytest.raises( + InvalidArgument, + match="Hypothesis can't yet construct a strategy for instances of a Callable type", + ): + strategy.example() + + with pytest.raises(InvalidArgument, match="Cannot register generic type"): + st.register_type_strategy(func_type, st.none()) + + +@pytest.mark.skipif(sys.version_info <= (3, 7), reason="requires python3.8 or higher") +def test_callable_with_paramspec(): + P = ParamSpec("P") + func_type = Callable[P, None] + strategy = st.from_type(func_type) + with pytest.raises( + InvalidArgument, + match="Hypothesis can't yet construct a strategy for instances of a Callable type", + ): + strategy.example() + + with pytest.raises(InvalidArgument, match="Cannot register generic type"): + st.register_type_strategy(func_type, st.none()) + + +@pytest.mark.skipif(sys.version_info <= (3, 7), reason="requires python3.8 or higher") +def test_callable_return_typegard_type(): + strategy = st.from_type(Callable[[], TypeGuard[int]]) + with pytest.raises( + InvalidArgument, + match="Hypothesis cannot yet construct a strategy for callables " + "which are PEP-647 TypeGuards", + ): + strategy.example() + + with pytest.raises(InvalidArgument, match="Cannot register generic type"): + st.register_type_strategy(Callable[[], TypeGuard[int]], st.none()) + + +class Movie(TypedDict): # implicitly total=True + title: str + year: NotRequired[int] + + +@given(from_type(Movie)) +def test_typeddict_not_required(value): + assert type(value) == dict + assert set(value).issubset({"title", "year"}) + assert isinstance(value["title"], str) + if "year" in value: + assert isinstance(value["year"], int) + + +def test_typeddict_not_required_can_skip(): + find_any(from_type(Movie), lambda movie: "year" not in movie) + + +class OtherMovie(TypedDict, total=False): + title: Required[str] + year: int + + +@given(from_type(OtherMovie)) +def test_typeddict_required(value): + assert type(value) == dict + assert set(value).issubset({"title", "year"}) + assert isinstance(value["title"], str) + if "year" in value: + assert isinstance(value["year"], int) + + +def test_typeddict_required_must_have(): + assert_all_examples(from_type(OtherMovie), lambda movie: "title" in movie) + + +class Story(TypedDict, total=True): + author: str + + +class Book(Story, total=False): + pages: int + + +class Novel(Book): + genre: Required[str] + rating: NotRequired[str] + + +@pytest.mark.parametrize( + "check,condition", + [ + pytest.param( + assert_all_examples, + lambda novel: "author" in novel, + id="author-is-required", + ), + pytest.param( + assert_all_examples, lambda novel: "genre" in novel, id="genre-is-required" + ), + pytest.param( + find_any, lambda novel: "pages" in novel, id="pages-may-be-present" + ), + pytest.param( + find_any, lambda novel: "pages" not in novel, id="pages-may-be-absent" + ), + pytest.param( + find_any, lambda novel: "rating" in novel, id="rating-may-be-present" + ), + pytest.param( + find_any, lambda novel: "rating" not in novel, id="rating-may-be-absent" + ), + ], +) +def test_required_and_not_required_keys(check, condition): + check(from_type(Novel), condition) + + +def test_typeddict_error_msg(): + with pytest.raises(TypeError, match="is not valid as type argument"): + + class Foo(TypedDict): + attr: Required + + with pytest.raises(TypeError, match="is not valid as type argument"): + + class Bar(TypedDict): + attr: NotRequired diff --git a/hypothesis-python/tox.ini b/hypothesis-python/tox.ini index 14bb57e075..c39605f169 100644 --- a/hypothesis-python/tox.ini +++ b/hypothesis-python/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{36,py36,37,py37,38,39}-{brief,prettyquick,full,custom} +envlist = py{37,py37,38,py38,39,py39,310,311,312}-{brief,prettyquick,full,custom} toxworkdir={env:TOX_WORK_DIR:.tox} [testenv] @@ -12,58 +12,96 @@ passenv= LC_ALL COVERAGE_FILE TOXENV + # Allow CI builds (or user builds) to force coloured terminal output. + PY_COLORS setenv= + PYTHONDEVMODE=1 brief: HYPOTHESIS_PROFILE=speedy commands = full: bash scripts/basic-test.sh - brief: python -m pytest tests/cover/test_testdecorators.py {posargs} - prettyquick: python -m pytest tests/cover/ - custom: python -m pytest {posargs} + brief: python -bb -X dev -m pytest tests/cover/test_testdecorators.py {posargs} + prettyquick: python -bb -X dev -m pytest tests/cover/ + custom: python -bb -X dev -m pytest {posargs} + +[testenv:py310-pyjion] +deps = + -r../requirements/test.txt + # We'd like to pin this, but pip-compile has to run under Python 3.10 (+) + # to do so and that's painful because other tools aren't compatible yet. + # If it's mid-2022, try again, starting by updating ci_version to 3.10 + pyjion +commands = + # TODO: restore `-n auto` https://github.com/tonybaloney/Pyjion/issues/456 + # TODO: re-enable in Actions main.yml once this actually works + pyjion -m pytest tests/cover tests/pytest tests/nocover [testenv:quality] deps= -r../requirements/test.txt commands= - python -m pytest tests/quality/ -n2 + python -bb -X dev -m pytest tests/quality/ -n auto -[testenv:pandas25] +# Note: when adding or removing tested Pandas versions, make sure to update the +# docs in numpy.rst too. To see current download rates of each minor version: +# https://pepy.tech/project/pandas?versions=1.0.*&versions=1.1.*&versions=1.2.*&versions=1.3.*&versions=1.4.*&versions=1.5.* +[testenv:pandas10] deps = -r../requirements/test.txt - pandas~=0.25.0 + pandas~=1.0.5 commands = - python -m pytest tests/pandas -n2 + python -bb -X dev -m pytest tests/pandas -n auto -[testenv:pandas100] +[testenv:pandas11] deps = -r../requirements/test.txt - pandas~=1.0.0 + pandas~=1.1.5 commands = - python -m pytest tests/pandas -n2 + python -bb -X dev -m pytest tests/pandas -n auto -[testenv:pandas111] +[testenv:pandas12] deps = -r../requirements/test.txt - pandas~=1.1.1 + pandas~=1.2.5 commands = - python -m pytest tests/pandas -n2 + python -bb -X dev -m pytest tests/pandas -n auto -[testenv:django22] +[testenv:pandas13] +deps = + -r../requirements/test.txt + pandas~=1.3.5 commands = - pip install .[pytz] - pip install django~=2.2.0 - python -m tests.django.manage test tests.django + python -bb -X dev -m pytest tests/pandas -n auto + +[testenv:pandas14] +deps = + -r../requirements/test.txt + pandas~=1.4.3 +commands = + python -bb -X dev -m pytest tests/pandas -n auto -[testenv:django30] +[testenv:pandas15] +deps = + -r../requirements/test.txt + pandas~=1.5.0 commands = - pip install .[pytz] - pip install django~=3.0.10 - python -m tests.django.manage test tests.django + python -bb -X dev -m pytest tests/pandas -n auto +# Adding a new pandas? See comment above! -[testenv:django31] +[testenv:django32] commands = pip install .[pytz] - pip install django~=3.1.1 - python -m tests.django.manage test tests.django + pip install django~=3.2.15 + python -bb -X dev -m tests.django.manage test tests.django + +[testenv:django40] +commands = + pip install django~=4.0.7 + python -bb -X dev -m tests.django.manage test tests.django + +[testenv:django41] +commands = + pip install django~=4.1.0 + python -bb -X dev -m tests.django.manage test tests.django [testenv:nose] deps = @@ -75,38 +113,66 @@ commands= deps = -r../requirements/test.txt commands= - pip install pytest==4.6 pytest-xdist==1.34 - python -m pytest tests/pytest tests/cover/test_testdecorators.py + pip install pytest==4.6.11 pytest-xdist==1.34 + python -bb -X dev -m pytest tests/pytest tests/cover/test_testdecorators.py +[testenv:pytest54] +deps = + -r../requirements/test.txt +commands= + pip install pytest==5.4.3 pytest-xdist + python -bb -X dev -m pytest tests/pytest tests/cover/test_testdecorators.py -[testenv:coverage] +[testenv:pytest62] deps = -r../requirements/test.txt +commands= + pip install pytest==6.2.5 pytest-xdist + python -bb -X dev -m pytest tests/pytest tests/cover/test_testdecorators.py + +[testenv:pytest7] +deps = + -r../requirements/test.txt +commands= + pip install pytest==7.* pytest-xdist + python -bb -X dev -m pytest tests/pytest tests/cover/test_testdecorators.py + +[testenv:coverage] +deps = -r../requirements/coverage.txt whitelist_externals= rm setenv= + PYTHONDEVMODE=1 HYPOTHESIS_INTERNAL_COVERAGE=true -commands = +commands_pre = rm -f branch-check pip install .[zoneinfo] python -m coverage --version python -m coverage debug sys - python -m coverage run --rcfile=.coveragerc -m pytest -n0 --strict tests/cover tests/conjecture tests/datetime tests/numpy tests/pandas tests/lark --ff {posargs} + # Explicitly erase any old .coverage file so the report never sees it. + python -m coverage erase +# Produce a coverage report even if the test suite fails. +# (The tox task will still count as failed.) +ignore_errors = true +commands = + python -bb -X dev -m coverage run --rcfile=.coveragerc --source=hypothesis -m pytest -n0 --strict-markers --ff {posargs} \ + tests/cover tests/conjecture tests/datetime tests/numpy tests/pandas tests/lark tests/redis tests/dpcontracts tests/codemods tests/typing_extensions python -m coverage report -m --fail-under=100 --show-missing --skip-covered python scripts/validate_branch_check.py [testenv:conjecture-coverage] deps = - -r../requirements/test.txt -r../requirements/coverage.txt -whitelist_externals= - rm setenv= + PYTHONDEVMODE=1 HYPOTHESIS_INTERNAL_COVERAGE=true +commands_pre = + python -m coverage erase +ignore_errors = true commands = - python -m coverage run --rcfile=.coveragerc --source=hypothesis.internal.conjecture -m pytest -n0 --strict tests/conjecture + python -bb -X dev -m coverage run --rcfile=.coveragerc --source=hypothesis.internal.conjecture -m pytest -n0 --strict-markers tests/conjecture python -m coverage report -m --fail-under=100 --show-missing --skip-covered @@ -115,4 +181,4 @@ deps= -r../requirements/test.txt commands= python -m pip install --editable examples/example_hypothesis_entrypoint - python -m pytest examples + python -bb -X dev -m pytest examples diff --git a/hypothesis-ruby/CHANGELOG.md b/hypothesis-ruby/CHANGELOG.md index 99a372fec3..8602f7543b 100644 --- a/hypothesis-ruby/CHANGELOG.md +++ b/hypothesis-ruby/CHANGELOG.md @@ -1,3 +1,11 @@ +# Hypothesis for Ruby 0.7.1 (2021-03-27) + +This patch fixes some internal typos. There is no user-visible change. + +# Hypothesis for Ruby 0.7.0 (2021-03-12) + +Moves rake from being a a runtime dependency to being a development dependency. Rake is used to run tests but is not required for consumers of hypothesis-ruby. + # Hypothesis for Ruby 0.6.1 (2021-02-01) This patch contains minor performance improvements for `HypothesisCoreIntegers` class instantiation. @@ -44,7 +52,7 @@ manually pass a seed. # Hypothesis for Ruby 0.1.2 (2018-09-24) -This release makes the code useable via a direct require. +This release makes the code usable via a direct require. I.e. no need for rubygems or any special LOAD_PATH. For example, if the base directory were in /opt, you'd just say: diff --git a/hypothesis-ruby/Gemfile b/hypothesis-ruby/Gemfile index 86a4398dc6..3fda5e2a37 100644 --- a/hypothesis-ruby/Gemfile +++ b/hypothesis-ruby/Gemfile @@ -7,6 +7,7 @@ gemspec gem 'rubocop', '~> 0.51.0' gem 'simplecov', '~> 0.15.1' gem 'yard', '~> 0.9.12' +gem 'rake', '>= 10.0', '< 14.0' gem 'redcarpet', '~> 3.4.0' gem 'rutie', '~> 0.0.3' gem 'minitest', '~> 5.8.4' diff --git a/hypothesis-ruby/Gemfile.lock b/hypothesis-ruby/Gemfile.lock index 8438096383..4ebefcf342 100644 --- a/hypothesis-ruby/Gemfile.lock +++ b/hypothesis-ruby/Gemfile.lock @@ -1,8 +1,7 @@ PATH remote: . specs: - hypothesis-specs (0.6.0) - rake (>= 10.0, < 13.0) + hypothesis-specs (0.7.0) rutie (~> 0.0.3) GEM @@ -57,6 +56,7 @@ PLATFORMS DEPENDENCIES hypothesis-specs! minitest (~> 5.8.4) + rake (>= 10.0, < 14.0) redcarpet (~> 3.4.0) rspec (~> 3.0) rubocop (~> 0.51.0) @@ -65,4 +65,4 @@ DEPENDENCIES yard (~> 0.9.12) BUNDLED WITH - 2.2.7 + 2.2.15 diff --git a/hypothesis-ruby/hypothesis-specs.gemspec b/hypothesis-ruby/hypothesis-specs.gemspec index 15451f54f8..c00a98788d 100644 --- a/hypothesis-ruby/hypothesis-specs.gemspec +++ b/hypothesis-ruby/hypothesis-specs.gemspec @@ -2,8 +2,8 @@ Gem::Specification.new do |s| s.name = 'hypothesis-specs' - s.version = '0.6.1' - s.date = '2021-02-01' + s.version = '0.7.1' + s.date = '2021-03-27' s.description = <<~DESCRIPTION Hypothesis is a powerful, flexible, and easy to use library for property-based testing. DESCRIPTION @@ -18,5 +18,4 @@ DESCRIPTION s.license = 'MPL-2.0' s.extensions = Dir['ext/extconf.rb'] s.add_dependency 'rutie', '~> 0.0.3' - s.add_runtime_dependency 'rake', '>= 10.0', '< 13.0' end diff --git a/hypothesis-ruby/lib/hypothesis.rb b/hypothesis-ruby/lib/hypothesis.rb index 99ff6a22cc..75d349fb62 100644 --- a/hypothesis-ruby/lib/hypothesis.rb +++ b/hypothesis-ruby/lib/hypothesis.rb @@ -69,7 +69,7 @@ def hypothesis_stable_identifier # the previous examples. # Note that essentially any answer to this method is - # "fine" in that the failure mode is that sometiems we + # "fine" in that the failure mode is that sometimes we # just won't run the same test, but it's nice to keep # this as stable as possible if the code isn't changing. diff --git a/mypy.ini b/mypy.ini index d219cc246f..ecb6e6ab37 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,22 +1,17 @@ [mypy] -python_version = 3.6 +python_version = 3.10 platform = linux disallow_untyped_decorators = True disallow_incomplete_defs = True no_implicit_optional = True +no_implicit_reexport = True follow_imports = silent ignore_missing_imports = True +strict_equality = True warn_no_return = True warn_unused_ignores = True warn_unused_configs = True warn_redundant_casts = True - - -[mypy-hypothesis.internal.*] -ignore_errors = True - -[mypy-hypothesis.strategies._internal.random]. -ignore_errors = True diff --git a/pytest.ini b/pytest.ini index 9c23e82af5..eaf26d5820 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,20 @@ [pytest] -addopts=--strict --tb=native -p pytester --runpytest=subprocess --durations=20 +# -rfEX :: Print a summary of failures, errors, and xpasses (xfails that pass). +addopts=--strict-markers --tb=native -p pytester --runpytest=subprocess --durations=20 -rfEX +xfail_strict = True filterwarnings = + error ignore::hypothesis.errors.NonInteractiveExampleWarning + # https://github.com/pandas-dev/pandas/issues/41199 + default:Creating a LegacyVersion has been deprecated and will be removed in the next major release:DeprecationWarning + default:distutils Version classes are deprecated\. Use packaging\.version instead:DeprecationWarning + # https://github.com/pandas-dev/pandas/issues/32056 (?) + default:numpy\.ufunc size changed, may indicate binary incompatibility\. Expected 216 from C header, got 232 from PyObject:RuntimeWarning + # https://github.com/lark-parser/lark/pull/1140 + default:module 'sre_constants' is deprecated:DeprecationWarning + default:module 'sre_parse' is deprecated:DeprecationWarning + # https://github.com/pandas-dev/pandas/issues/34848 + default:`np\.bool` is a deprecated alias for the builtin `bool`:DeprecationWarning + default:`np\.complex` is a deprecated alias for the builtin `complex`:DeprecationWarning + default:`np\.object` is a deprecated alias for the builtin `object`:DeprecationWarning diff --git a/requirements/coverage.in b/requirements/coverage.in index 86bd8ff487..c196eba125 100644 --- a/requirements/coverage.in +++ b/requirements/coverage.in @@ -1,5 +1,14 @@ +backports.zoneinfo +black +click coverage +dpcontracts +fakeredis lark-parser +libcst numpy pandas +python-dateutil pytz +typing-extensions +-r test.in diff --git a/requirements/coverage.txt b/requirements/coverage.txt index 80ecc3d4d4..5189687a12 100644 --- a/requirements/coverage.txt +++ b/requirements/coverage.txt @@ -1,28 +1,104 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile --output-file=requirements/coverage.txt hypothesis-python/setup.py requirements/coverage.in # -attrs==20.3.0 - # via hypothesis (hypothesis-python/setup.py) -coverage==5.4 +async-timeout==4.0.2 + # via redis +attrs==22.1.0 + # via + # hypothesis (hypothesis-python/setup.py) + # pytest +backports-zoneinfo==0.2.1 + # via -r requirements/coverage.in +black==22.10.0 + # via -r requirements/coverage.in +click==8.1.3 + # via + # -r requirements/coverage.in + # black +coverage==6.5.0 # via -r requirements/coverage.in -lark-parser==0.11.1 +deprecated==1.2.13 + # via redis +dpcontracts==0.6.0 # via -r requirements/coverage.in -numpy==1.20.1 +exceptiongroup==1.0.1 ; python_version < "3.11" + # via + # hypothesis (hypothesis-python/setup.py) + # pytest +execnet==1.9.0 + # via pytest-xdist +fakeredis==1.10.1 + # via -r requirements/coverage.in +iniconfig==1.1.1 + # via pytest +lark-parser==0.12.0 + # via -r requirements/coverage.in +libcst==0.4.9 + # via -r requirements/coverage.in +mypy-extensions==0.4.3 + # via + # black + # typing-inspect +numpy==1.23.4 # via # -r requirements/coverage.in # pandas -pandas==1.2.2 +packaging==21.3 + # via + # pytest + # redis +pandas==1.5.1 # via -r requirements/coverage.in -python-dateutil==2.8.1 - # via pandas -pytz==2021.1 +pathspec==0.10.1 + # via black +pexpect==4.8.0 + # via -r requirements/test.in +platformdirs==2.5.3 + # via black +pluggy==1.0.0 + # via pytest +ptyprocess==0.7.0 + # via pexpect +pyparsing==3.0.9 + # via packaging +pytest==7.2.0 + # via + # -r requirements/test.in + # pytest-xdist +pytest-xdist==3.0.2 + # via -r requirements/test.in +python-dateutil==2.8.2 + # via + # -r requirements/coverage.in + # pandas +pytz==2022.6 # via # -r requirements/coverage.in # pandas -six==1.15.0 +pyyaml==6.0 + # via libcst +redis==4.3.4 + # via fakeredis +six==1.16.0 # via python-dateutil -sortedcontainers==2.3.0 - # via hypothesis (hypothesis-python/setup.py) +sortedcontainers==2.4.0 + # via + # fakeredis + # hypothesis (hypothesis-python/setup.py) +tomli==2.0.1 + # via + # black + # pytest +typing-extensions==4.4.0 + # via + # -r requirements/coverage.in + # black + # libcst + # typing-inspect +typing-inspect==0.8.0 + # via libcst +wrapt==1.14.1 + # via deprecated diff --git a/requirements/test.txt b/requirements/test.txt index 515c27fa9d..2714a25808 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,43 +1,38 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile --output-file=requirements/test.txt hypothesis-python/setup.py requirements/test.in # -apipkg==1.5 - # via execnet -attrs==20.3.0 +attrs==22.1.0 # via # hypothesis (hypothesis-python/setup.py) # pytest -execnet==1.8.0 +exceptiongroup==1.0.1 ; python_version < "3.11" + # via + # hypothesis (hypothesis-python/setup.py) + # pytest +execnet==1.9.0 # via pytest-xdist iniconfig==1.1.1 # via pytest -packaging==20.9 +packaging==21.3 # via pytest pexpect==4.8.0 # via -r requirements/test.in -pluggy==0.13.1 +pluggy==1.0.0 # via pytest ptyprocess==0.7.0 # via pexpect -py==1.10.0 - # via - # pytest - # pytest-forked -pyparsing==2.4.7 +pyparsing==3.0.9 # via packaging -pytest-forked==1.3.0 - # via pytest-xdist -pytest-xdist==2.2.1 - # via -r requirements/test.in -pytest==6.2.2 +pytest==7.2.0 # via # -r requirements/test.in - # pytest-forked # pytest-xdist -sortedcontainers==2.3.0 +pytest-xdist==3.0.2 + # via -r requirements/test.in +sortedcontainers==2.4.0 # via hypothesis (hypothesis-python/setup.py) -toml==0.10.2 +tomli==2.0.1 # via pytest diff --git a/requirements/tools.in b/requirements/tools.in index 7ade546f27..3003c4cc85 100644 --- a/requirements/tools.in +++ b/requirements/tools.in @@ -1,29 +1,42 @@ -blacken-docs +codespell coverage django dpcontracts flake8 flake8-2020 flake8-bandit +flake8-builtins flake8-bugbear flake8-comprehensions +flake8-datetimez flake8-docstrings flake8-mutable -ipython < 7.17 # drops support for Python 3.6 +flake8-noqa +flake8-pie +flake8-pytest-style +flake8-return +flake8-simplify +# flake8-strftime # See https://github.com/python-formate/flake8_strftime/issues/47 +ipython lark-parser libcst mypy pip-tools +pyright pytest python-dateutil requests restructuredtext-lint shed sphinx +sphinx-codeautolink sphinx-hoverxref sphinx-rtd-theme sphinx-selective-exclude -toml tox -traitlets < 5.0 # drops support for Python 3.6 twine +types-click +types-pkg_resources +types-pytz +types-redis +typing-extensions diff --git a/requirements/tools.txt b/requirements/tools.txt index fa97e44021..15ddf27177 100644 --- a/requirements/tools.txt +++ b/requirements/tools.txt @@ -1,316 +1,374 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile --output-file=requirements/tools.txt hypothesis-python/setup.py requirements/tools.in # alabaster==0.7.12 # via sphinx -appdirs==1.4.4 - # via - # black - # virtualenv -asgiref==3.3.1 +asgiref==3.5.2 # via django -attrs==20.3.0 +astor==0.8.1 + # via flake8-simplify +asttokens==2.1.0 + # via stack-data +attrs==22.1.0 # via # flake8-bugbear # hypothesis (hypothesis-python/setup.py) # pytest -autoflake==1.4 +autoflake==1.7.7 # via shed -babel==2.9.0 +babel==2.11.0 # via sphinx backcall==0.2.0 # via ipython -bandit==1.7.0 +backports-zoneinfo==0.2.1 + # via django +bandit==1.7.4 # via flake8-bandit -black==20.8b1 - # via - # blacken-docs - # shed -blacken-docs==1.9.2 - # via -r requirements/tools.in -bleach==3.3.0 +beautifulsoup4==4.11.1 + # via sphinx-codeautolink +black==22.10.0 + # via shed +bleach==5.0.1 # via readme-renderer -certifi==2020.12.5 +build==0.9.0 + # via pip-tools +certifi==2022.9.24 # via requests -cffi==1.14.5 +cffi==1.15.1 # via cryptography -chardet==4.0.0 +charset-normalizer==2.1.1 # via requests -click==7.1.2 +click==8.1.3 # via # black # pip-tools - # pybetter - # pyemojify -colorama==0.4.4 - # via twine -com2ann==0.1.1 +codespell==2.2.2 + # via -r requirements/tools.in +com2ann==0.3.0 # via shed -coverage==5.4 +commonmark==0.9.1 + # via rich +coverage==6.5.0 # via -r requirements/tools.in -cryptography==3.4.5 +cryptography==38.0.3 # via secretstorage -decorator==4.4.2 - # via - # ipython - # traitlets -distlib==0.3.1 +decorator==5.1.1 + # via ipython +distlib==0.3.6 # via virtualenv -django==3.1.6 +django==4.1.3 # via -r requirements/tools.in -docutils==0.16 +docutils==0.17.1 # via # readme-renderer # restructuredtext-lint # sphinx + # sphinx-rtd-theme dpcontracts==0.6.0 # via -r requirements/tools.in -filelock==3.0.12 +exceptiongroup==1.0.1 ; python_version < "3.11" + # via + # hypothesis (hypothesis-python/setup.py) + # pytest +executing==1.2.0 + # via stack-data +filelock==3.8.0 # via # tox # virtualenv -flake8-2020==1.6.0 - # via -r requirements/tools.in -flake8-bandit==2.1.2 - # via -r requirements/tools.in -flake8-bugbear==20.11.1 - # via -r requirements/tools.in -flake8-comprehensions==3.3.1 - # via -r requirements/tools.in -flake8-docstrings==1.5.0 - # via -r requirements/tools.in -flake8-mutable==1.2.0 - # via -r requirements/tools.in -flake8-polyfill==1.0.2 - # via flake8-bandit -flake8==3.8.4 +flake8==5.0.4 # via # -r requirements/tools.in # flake8-2020 # flake8-bandit # flake8-bugbear + # flake8-builtins # flake8-comprehensions + # flake8-datetimez # flake8-docstrings # flake8-mutable - # flake8-polyfill -gitdb==4.0.5 + # flake8-noqa + # flake8-simplify +flake8-2020==1.7.0 + # via -r requirements/tools.in +flake8-bandit==4.1.1 + # via -r requirements/tools.in +flake8-bugbear==22.10.27 + # via -r requirements/tools.in +flake8-builtins==2.0.1 + # via -r requirements/tools.in +flake8-comprehensions==3.10.1 + # via -r requirements/tools.in +flake8-datetimez==20.10.0 + # via -r requirements/tools.in +flake8-docstrings==1.6.0 + # via -r requirements/tools.in +flake8-mutable==1.2.0 + # via -r requirements/tools.in +flake8-noqa==1.2.9 + # via -r requirements/tools.in +flake8-pie==0.16.0 + # via -r requirements/tools.in +flake8-plugin-utils==1.3.2 + # via + # flake8-pytest-style + # flake8-return +flake8-pytest-style==1.6.0 + # via -r requirements/tools.in +flake8-return==1.2.0 + # via -r requirements/tools.in +flake8-simplify==0.19.3 + # via -r requirements/tools.in +gitdb==4.0.9 # via gitpython -gitpython==3.1.13 +gitpython==3.1.29 # via bandit -idna==2.10 +idna==3.4 # via requests -imagesize==1.2.0 +imagesize==1.4.1 # via sphinx +importlib-metadata==5.0.0 + # via + # keyring + # sphinx + # twine iniconfig==1.1.1 # via pytest -ipython-genutils==0.2.0 - # via traitlets -ipython==7.16.1 +ipython==8.6.0 # via -r requirements/tools.in -isort==5.7.0 +isort==5.10.1 # via shed -jedi==0.18.0 +jaraco-classes==3.2.3 + # via keyring +jedi==0.18.1 # via ipython -jeepney==0.6.0 +jeepney==0.8.0 # via # keyring # secretstorage -jinja2==2.11.3 +jinja2==3.1.2 # via sphinx -keyring==22.0.1 +keyring==23.11.0 # via twine -lark-parser==0.11.1 +lark-parser==0.12.0 # via -r requirements/tools.in -libcst==0.3.17 +libcst==0.4.9 # via # -r requirements/tools.in - # pybetter # shed -markupsafe==1.1.1 +markupsafe==2.1.1 # via jinja2 -mccabe==0.6.1 +matplotlib-inline==0.1.6 + # via ipython +mccabe==0.7.0 # via flake8 +more-itertools==9.0.0 + # via jaraco-classes +mypy==0.990 + # via -r requirements/tools.in mypy-extensions==0.4.3 # via # black # mypy # typing-inspect -mypy==0.800 - # via -r requirements/tools.in -packaging==20.9 +nodeenv==1.7.0 + # via pyright +packaging==21.3 # via - # bleach + # build # pytest # sphinx # tox -parso==0.8.1 +parso==0.8.3 # via jedi -pathspec==0.8.1 +pathspec==0.10.1 # via black -pbr==5.5.1 +pbr==5.11.0 # via stevedore +pep517==0.13.0 + # via build pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -pip-tools==5.5.0 +pip-tools==6.9.0 # via -r requirements/tools.in -pkginfo==1.7.0 +pkginfo==1.8.3 # via twine -pluggy==0.13.1 +platformdirs==2.5.3 + # via + # black + # virtualenv +pluggy==1.0.0 # via # pytest # tox -prompt-toolkit==3.0.16 +prompt-toolkit==3.0.32 # via ipython ptyprocess==0.7.0 # via pexpect -py==1.10.0 - # via - # pytest - # tox -pybetter==0.3.6.1 - # via shed -pycodestyle==2.6.0 - # via - # flake8 - # flake8-bandit -pycparser==2.20 +pure-eval==0.2.2 + # via stack-data +py==1.11.0 + # via tox +pycodestyle==2.9.1 + # via flake8 +pycparser==2.21 # via cffi -pydocstyle==5.1.1 +pydocstyle==6.1.1 # via flake8-docstrings -pyemojify==0.2.0 - # via pybetter -pyflakes==2.2.0 +pyflakes==2.5.0 # via # autoflake # flake8 -pygments==2.7.4 +pygments==2.13.0 # via # ipython - # pybetter # readme-renderer + # rich # sphinx -pyparsing==2.4.7 +pyparsing==3.0.9 # via packaging -pytest==6.2.2 +pyright==1.1.279 # via -r requirements/tools.in -python-dateutil==2.8.1 +pytest==7.2.0 # via -r requirements/tools.in -pytz==2021.1 - # via - # babel - # django -pyupgrade==2.10.0 +python-dateutil==2.8.2 + # via -r requirements/tools.in +pytz==2022.6 + # via babel +pyupgrade==3.2.2 # via shed -pyyaml==5.4.1 +pyyaml==6.0 # via # bandit # libcst -readme-renderer==28.0 - # via twine -regex==2020.11.13 - # via black -requests-toolbelt==0.9.1 +readme-renderer==37.3 # via twine -requests==2.25.1 +requests==2.28.1 # via # -r requirements/tools.in # requests-toolbelt # sphinx # twine -restructuredtext-lint==1.3.2 +requests-toolbelt==0.10.1 + # via twine +restructuredtext-lint==1.4.0 # via -r requirements/tools.in -rfc3986==1.4.0 +rfc3986==2.0.0 + # via twine +rich==12.6.0 # via twine -secretstorage==3.3.1 +secretstorage==3.3.3 # via keyring -shed==0.3.2 +shed==0.10.7 # via -r requirements/tools.in -six==1.15.0 +six==1.16.0 # via - # bandit + # asttokens # bleach # python-dateutil - # readme-renderer # tox - # traitlets - # virtualenv -smmap==3.0.5 +smmap==5.0.0 # via gitdb -snowballstemmer==2.1.0 +snowballstemmer==2.2.0 # via # pydocstyle # sphinx -sortedcontainers==2.3.0 +sortedcontainers==2.4.0 # via hypothesis (hypothesis-python/setup.py) -sphinx-hoverxref==0.5b1 +soupsieve==2.3.2.post1 + # via beautifulsoup4 +sphinx==5.3.0 + # via + # -r requirements/tools.in + # sphinx-codeautolink + # sphinx-hoverxref + # sphinx-rtd-theme +sphinx-codeautolink==0.12.1 + # via -r requirements/tools.in +sphinx-hoverxref==1.3.0 # via -r requirements/tools.in -sphinx-rtd-theme==0.5.1 +sphinx-rtd-theme==1.1.1 # via -r requirements/tools.in sphinx-selective-exclude==1.0.3 # via -r requirements/tools.in -sphinx==3.4.3 - # via - # -r requirements/tools.in - # sphinx-rtd-theme sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx -sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-htmlhelp==2.0.0 # via sphinx +sphinxcontrib-jquery==3.0.0 + # via sphinx-hoverxref sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-qthelp==1.0.3 # via sphinx -sphinxcontrib-serializinghtml==1.1.4 +sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlparse==0.4.1 +sqlparse==0.4.3 # via django -stevedore==3.3.0 +stack-data==0.6.1 + # via ipython +stevedore==4.1.1 # via bandit -tokenize-rt==4.1.0 +tokenize-rt==5.0.0 # via pyupgrade -toml==0.10.2 +tomli==2.0.1 # via - # -r requirements/tools.in + # autoflake # black + # build + # mypy + # pep517 # pytest # tox -tox==3.21.4 +tox==3.27.0 # via -r requirements/tools.in -tqdm==4.56.2 - # via twine -traitlets==4.3.3 +traitlets==5.5.0 # via - # -r requirements/tools.in # ipython -twine==3.3.0 + # matplotlib-inline +twine==4.0.1 # via -r requirements/tools.in -typed-ast==1.4.2 - # via - # black - # mypy -typing-extensions==3.7.4.3 +types-click==7.1.8 + # via -r requirements/tools.in +types-pkg-resources==0.1.3 + # via -r requirements/tools.in +types-pytz==2022.6.0.1 + # via -r requirements/tools.in +types-redis==4.3.21.4 + # via -r requirements/tools.in +typing-extensions==4.4.0 # via + # -r requirements/tools.in # black + # flake8-noqa + # flake8-pie # libcst # mypy + # rich # typing-inspect -typing-inspect==0.6.0 +typing-inspect==0.8.0 # via libcst -urllib3==1.26.3 - # via requests -virtualenv==20.4.2 +urllib3==1.26.12 + # via + # requests + # twine +virtualenv==20.16.7 # via tox wcwidth==0.2.5 # via prompt-toolkit webencodings==0.5.1 # via bleach +wheel==0.38.4 + # via pip-tools +zipp==3.10.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/tooling/ignore-list.txt b/tooling/ignore-list.txt new file mode 100644 index 0000000000..203487fd6e --- /dev/null +++ b/tooling/ignore-list.txt @@ -0,0 +1,9 @@ +crate +damon +nd +ned +nin +strat +tread +rouge +tey diff --git a/tooling/scripts/tool-hash.py b/tooling/scripts/tool-hash.py index 9e14ec8dad..dcc72a171e 100755 --- a/tooling/scripts/tool-hash.py +++ b/tooling/scripts/tool-hash.py @@ -3,20 +3,15 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import hashlib import sys if __name__ == "__main__": - print(hashlib.sha384(sys.stdin.read().encode("utf-8")).hexdigest()[:10]) + print(hashlib.sha384(sys.stdin.read().encode()).hexdigest()[:10]) diff --git a/tooling/setup.py b/tooling/setup.py index d93928add1..16c431e495 100644 --- a/tooling/setup.py +++ b/tooling/setup.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import os @@ -33,9 +28,9 @@ def local_file(name): author_email="david@drmaciver.com", packages=setuptools.find_packages(SOURCE), package_dir={"": SOURCE}, - url=("https://github.com/HypothesisWorks/hypothesis-python/tree/master/tooling"), + url="https://github.com/HypothesisWorks/hypothesis-python/tree/master/tooling", license="MPL v2", description="A library for property-based testing", - python_requires=">=3.6", + python_requires=">=3.7", long_description=open(README).read(), ) diff --git a/tooling/src/hypothesistooling/__init__.py b/tooling/src/hypothesistooling/__init__.py index 715f2c6aaf..afe062fc71 100644 --- a/tooling/src/hypothesistooling/__init__.py +++ b/tooling/src/hypothesistooling/__init__.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import os import shlex diff --git a/tooling/src/hypothesistooling/__main__.py b/tooling/src/hypothesistooling/__main__.py index fc9f8c6a05..b73f31bc6d 100644 --- a/tooling/src/hypothesistooling/__main__.py +++ b/tooling/src/hypothesistooling/__main__.py @@ -1,30 +1,27 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import os +import pathlib import re import subprocess import sys -from datetime import datetime from glob import glob +import requests +from coverage.config import CoverageConfig + import hypothesistooling as tools import hypothesistooling.projects.conjecturerust as cr import hypothesistooling.projects.hypothesispython as hp import hypothesistooling.projects.hypothesisruby as hr -from coverage.config import CoverageConfig from hypothesistooling import installers as install, releasemanagement as rm from hypothesistooling.scripts import pip_tool @@ -63,18 +60,26 @@ def check_installed(): don't fail to run if a previous install failed midway).""" +def codespell(*files): + pip_tool( + "codespell", + "--check-hidden", + "--check-filenames", + "--ignore-words=./tooling/ignore-list.txt", + "--skip=__pycache__,.mypy_cache,.venv,.git,tlds-alpha-by-domain.txt", + *files, + ) + + @task() def lint(): pip_tool( "flake8", - *[f for f in tools.all_files() if f.endswith(".py")], + *(f for f in tools.all_files() if f.endswith(".py")), "--config", os.path.join(tools.ROOT, ".flake8"), ) - - -HEAD = tools.hash_for_name("HEAD") -MASTER = tools.hash_for_name("origin/master") + codespell(*(f for f in tools.all_files() if not f.endswith("by-domain.txt"))) def do_release(package): @@ -108,6 +113,9 @@ def do_release(package): @task() def deploy(): + HEAD = tools.hash_for_name("HEAD") + MASTER = tools.hash_for_name("origin/master") + print("Current head: ", HEAD) print("Current master:", MASTER) @@ -127,24 +135,18 @@ def deploy(): sys.exit(0) -CURRENT_YEAR = datetime.utcnow().year - - -HEADER = f""" +# See https://www.linuxfoundation.org/blog/copyright-notices-in-open-source-software-projects/ +HEADER = """ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-{CURRENT_YEAR} David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER""".strip() +""".strip() @task() @@ -166,39 +168,39 @@ def should_format_doc_file(path): files = tools.all_files() if format_all else changed doc_files_to_format = [f for f in sorted(files) if should_format_doc_file(f)] - pip_tool("blacken-docs", *doc_files_to_format) - files_to_format = [f for f in sorted(files) if should_format_file(f)] - if not files_to_format: + if not (files_to_format or doc_files_to_format): return # .coveragerc lists several regex patterns to treat as nocover pragmas, and # we want to find (and delete) cases where # pragma: no cover is redundant. + def warn(msg): + raise Exception(msg) + config = CoverageConfig() - config.from_file(os.path.join(hp.BASE_DIR, ".coveragerc"), our_file=True) + config.from_file(os.path.join(hp.BASE_DIR, ".coveragerc"), warn=warn, our_file=True) pattern = "|".join(l for l in config.exclude_list if "pragma" not in l) - unused_pragma_pattern = re.compile(f"({pattern}).*# pragma: no cover") + unused_pragma_pattern = re.compile(f"(({pattern}).*) # pragma: no (branch|cover)") + last_header_line = HEADER.splitlines()[-1].rstrip() for f in files_to_format: lines = [] with open(f, encoding="utf-8") as o: shebang = None first = True - header_done = False + in_header = True for l in o.readlines(): if first: first = False if l[:2] == "#!": shebang = l continue - if "END HEADER" in l and not header_done: + elif in_header and l.rstrip() == last_header_line: + in_header = False lines = [] - header_done = True - elif unused_pragma_pattern.search(l) is not None: - lines.append(l.replace("# pragma: no cover", "")) else: - lines.append(l) + lines.append(unused_pragma_pattern.sub(r"\1", l)) source = "".join(lines).strip() with open(f, "w", encoding="utf-8") as o: if shebang is not None: @@ -210,6 +212,7 @@ def should_format_doc_file(path): o.write(source) o.write("\n") + codespell("--write-changes", *files_to_format, *doc_files_to_format) pip_tool("shed", *files_to_format, *doc_files_to_format) @@ -257,16 +260,82 @@ def compile_requirements(upgrade=False): ) +def update_python_versions(): + install.ensure_python(PYTHONS[ci_version]) + cmd = "~/.cache/hypothesis-build-runtimes/pyenv/bin/pyenv install --list" + result = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE).stdout.decode() + # pyenv reports available versions in chronological order, so we keep the newest + # *unless* our current ends with a digit (is stable) and the candidate does not. + stable = re.compile(r".*3\.\d+.\d+$") + best = {} + for line in map(str.strip, result.splitlines()): + if m := re.match(r"(?:pypy)?3\.(?:[789]|\d\d)", line): + key = m.group() + if stable.match(line) or not stable.match(best.get(key, line)): + best[key] = line + + if best == PYTHONS: + return + + # Write the new mapping back to this file + thisfile = pathlib.Path(__file__) + before = thisfile.read_text() + after = re.sub(r"\nPYTHONS = \{[^{}]+\}", f"\nPYTHONS = {best}", before) + thisfile.write_text(after) + pip_tool("shed", str(thisfile)) + + # Automatically sync ci_version with the version in build.sh + build_sh = pathlib.Path(tools.ROOT) / "build.sh" + sh_before = build_sh.read_text() + sh_after = re.sub(r"3\.\d\d?\.\d\d?", best[ci_version], sh_before) + if sh_before != sh_after: + build_sh.unlink() # so bash doesn't reload a modified file + build_sh.write_text(sh_after) + build_sh.chmod(0o755) + + +def update_vendored_files(): + vendor = pathlib.Path(hp.PYTHON_SRC) / "hypothesis" / "vendor" + + # Turns out that as well as adding new gTLDs, IANA can *terminate* old ones + url = "http://data.iana.org/TLD/tlds-alpha-by-domain.txt" + fname = vendor / url.split("/")[-1] + new = requests.get(url).content + # If only the timestamp in the header comment has changed, skip the update. + if fname.read_bytes().splitlines()[1:] != new.splitlines()[1:]: + fname.write_bytes(new) + + # Always require the latest version of the tzdata package + tz_url = "https://pypi.org/pypi/tzdata/json" + tzdata_version = requests.get(tz_url).json()["info"]["version"] + setup = pathlib.Path(hp.BASE_DIR, "setup.py") + new = re.sub(r"tzdata>=(.+?) ", f"tzdata>={tzdata_version} ", setup.read_text()) + setup.write_text(new) + + +def has_diff(file_or_directory): + diff = ["git", "diff", "--no-patch", "--exit-code", "--", file_or_directory] + return subprocess.call(diff) != 0 + + @task() def upgrade_requirements(): + update_vendored_files() compile_requirements(upgrade=True) subprocess.call(["./build.sh", "format"], cwd=tools.ROOT) # exits 1 if changed - if hp.has_source_changes() and not os.path.isfile(hp.RELEASE_FILE): - with open(hp.RELEASE_FILE, mode="w") as f: - f.write( - "RELEASE_TYPE: patch\n\nThis patch updates our autoformatting " - "tools, improving our code style without any API changes.\n" + if has_diff(hp.PYTHON_SRC) and not os.path.isfile(hp.RELEASE_FILE): + if has_diff(f"{hp.PYTHON_SRC}/hypothesis/vendor/tlds-alpha-by-domain.txt"): + msg = ( + "our vendored `list of top-level domains " + "`__,\nwhich is used by the " + "provisional :func:`~hypothesis.provisional.domains` strategy." ) + else: + msg = "our autoformatting tools, improving our code style without any API changes." + with open(hp.RELEASE_FILE, mode="w") as f: + f.write(f"RELEASE_TYPE: patch\n\nThis patch updates {msg}\n") + update_python_versions() + subprocess.call(["git", "add", "."], cwd=tools.ROOT) @task() @@ -287,7 +356,7 @@ def documentation(): ) -def run_tox(task, version): +def run_tox(task, version, *args): python = install.python_executable(version) # Create a version of the name that tox will pick up for the correct @@ -304,75 +373,74 @@ def run_tox(task, version): env["PATH"] = os.path.dirname(python) + ":" + env["PATH"] print(env["PATH"]) - pip_tool("tox", "-e", task, env=env, cwd=hp.HYPOTHESIS_PYTHON) - - -# Via https://github.com/pyenv/pyenv/tree/master/plugins/python-build/share/python-build -PY36 = "3.6.12" -PY37 = "3.7.9" -PY38 = PYMAIN = "3.8.7" # Note: keep this in sync with build.sh -PY39 = "3.9.1" -PYPY36 = "pypy3.6-7.3.1" -PYPY37 = "pypy3.7-7.3.2" - - -# ALIASES are the executable names for each Python version -ALIASES = {PYPY36: "pypy3", PYPY37: "pypy3"} - -for n in [PY36, PY37, PY38, PY39]: - major, minor, patch = n.replace("-dev", ".").split(".") - ALIASES[n] = f"python{major}.{minor}" - + pip_tool("tox", "-e", task, *args, env=env, cwd=hp.HYPOTHESIS_PYTHON) + + +# update_python_versions(), above, keeps the contents of this dict up to date. +# When a version is added or removed, manually update the env lists in tox.ini and +# workflows/main.yml, and the `Programming Language ::` specifiers in setup.py +PYTHONS = { + "3.7": "3.7.15", + "3.8": "3.8.15", + "3.9": "3.9.15", + "3.10": "3.10.8", + "3.11": "3.11.0", + "3.12": "3.12-dev", + "pypy3.7": "pypy3.7-7.3.9", + "pypy3.8": "pypy3.8-7.3.9", + "pypy3.9": "pypy3.9-7.3.9", +} +ci_version = "3.8" # Keep this in sync with GH Actions main.yml python_tests = task( if_changed=( hp.PYTHON_SRC, hp.PYTHON_TESTS, + os.path.join(tools.ROOT, "pytest.ini"), os.path.join(hp.HYPOTHESIS_PYTHON, "scripts"), ) ) -@python_tests -def check_py36(): - run_tox("py36-full", PY36) - - -@python_tests -def check_py37(): - run_tox("py37-full", PY37) - - -@python_tests -def check_py38(): - run_tox("py38-full", PY38) - - -@python_tests -def check_py39(): - run_tox("py39-full", PY39) +# ALIASES are the executable names for each Python version +ALIASES = {} +for key, version in PYTHONS.items(): + if key.startswith("pypy"): + ALIASES[version] = "pypy3" + name = key.replace(".", "") + else: + ALIASES[version] = f"python{key}" + name = f"py3{key[2:]}" + TASKS[f"check-{name}"] = python_tests( + lambda n=f"{name}-full", v=version: run_tox(n, v) + ) @python_tests -def check_pypy36(): - run_tox("pypy3-full", PYPY36) +def check_py310_pyjion(): + run_tox("py310-pyjion", PYTHONS["3.10"]) -@python_tests -def check_pypy37(): - run_tox("pypy3-full", PYPY37) +@task() +def tox(*args): + if len(args) < 2: + print("Usage: ./build.sh tox TOX_ENV PY_VERSION [tox args]") + sys.exit(1) + run_tox(args[0], args[1], *args[2:]) def standard_tox_task(name): - TASKS["check-" + name] = python_tests(lambda: run_tox(name, PYMAIN)) + TASKS["check-" + name] = python_tests(lambda: run_tox(name, PYTHONS[ci_version])) standard_tox_task("nose") standard_tox_task("pytest46") +standard_tox_task("pytest54") +standard_tox_task("pytest62") -for n in [22, 30, 31]: +for n in [32, 40, 41]: standard_tox_task(f"django{n}") -for n in [25, 100, 111]: +for n in [10, 11, 12, 13, 14, 15]: standard_tox_task(f"pandas{n}") standard_tox_task("coverage") @@ -381,21 +449,24 @@ def standard_tox_task(name): @task() def check_quality(): - run_tox("quality", PYMAIN) + run_tox("quality", PYTHONS[ci_version]) @task(if_changed=(hp.PYTHON_SRC, os.path.join(hp.HYPOTHESIS_PYTHON, "examples"))) def check_examples3(): - run_tox("examples3", PYMAIN) + run_tox("examples3", PYTHONS[ci_version]) @task() -def check_whole_repo_tests(): +def check_whole_repo_tests(*args): install.ensure_shellcheck() subprocess.check_call( [sys.executable, "-m", "pip", "install", "--upgrade", hp.HYPOTHESIS_PYTHON] ) - subprocess.check_call([sys.executable, "-m", "pytest", tools.REPO_TESTS]) + + if not args: + args = [tools.REPO_TESTS] + subprocess.check_call([sys.executable, "-m", "pytest", *args]) @task() @@ -484,7 +555,7 @@ def audit_conjecture_rust(): def tasks(): """Print a list of all task names supported by the build system.""" for task_name in sorted(TASKS.keys()): - print(task_name) + print(" " + task_name) if __name__ == "__main__": @@ -510,6 +581,11 @@ def tasks(): ) sys.exit(1) + if task_to_run not in TASKS: + print(f"\nUnknown task {task_to_run!r}. Available tasks are:") + tasks() + sys.exit(1) + try: TASKS[task_to_run](*args) except subprocess.CalledProcessError as e: diff --git a/tooling/src/hypothesistooling/installers.py b/tooling/src/hypothesistooling/installers.py index 9385f92252..99959eeaf2 100644 --- a/tooling/src/hypothesistooling/installers.py +++ b/tooling/src/hypothesistooling/installers.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """Module for obtaining various versions of Python. diff --git a/tooling/src/hypothesistooling/junkdrawer.py b/tooling/src/hypothesistooling/junkdrawer.py index f7064599fa..0292bb7b2d 100644 --- a/tooling/src/hypothesistooling/junkdrawer.py +++ b/tooling/src/hypothesistooling/junkdrawer.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """Dumping ground module for things that don't have anywhere better to go. diff --git a/tooling/src/hypothesistooling/projects/__init__.py b/tooling/src/hypothesistooling/projects/__init__.py index e61e3c2872..fcb1ac6538 100644 --- a/tooling/src/hypothesistooling/projects/__init__.py +++ b/tooling/src/hypothesistooling/projects/__init__.py @@ -1,14 +1,9 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER diff --git a/tooling/src/hypothesistooling/projects/conjecturerust.py b/tooling/src/hypothesistooling/projects/conjecturerust.py index 5e5e7fea79..7b43fe82b7 100644 --- a/tooling/src/hypothesistooling/projects/conjecturerust.py +++ b/tooling/src/hypothesistooling/projects/conjecturerust.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import os import subprocess diff --git a/tooling/src/hypothesistooling/projects/hypothesispython.py b/tooling/src/hypothesistooling/projects/hypothesispython.py index 3b454d2bd3..8a14520215 100644 --- a/tooling/src/hypothesistooling/projects/hypothesispython.py +++ b/tooling/src/hypothesistooling/projects/hypothesispython.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import os import re @@ -19,8 +14,9 @@ import subprocess import sys -import hypothesistooling as tools import requests + +import hypothesistooling as tools from hypothesistooling import releasemanagement as rm PACKAGE_NAME = "hypothesis-python" @@ -81,9 +77,9 @@ def build_docs(builder="html"): ) -CHANGELOG_ANCHOR = re.compile(r"^\.\. _v\d+\.\d+\.\d+:$") -CHANGELOG_BORDER = re.compile(r"^-+$") -CHANGELOG_HEADER = re.compile(r"^\d+\.\d+\.\d+ - \d\d\d\d-\d\d-\d\d$") +CHANGELOG_ANCHOR = re.compile(r"^\.\. _v\d+\.\d+\.\d+:$", flags=re.MULTILINE) +CHANGELOG_BORDER = re.compile(r"^-+$", flags=re.MULTILINE) +CHANGELOG_HEADER = re.compile(r"^\d+\.\d+\.\d+ - \d\d\d\d-\d\d-\d\d$", flags=re.M) def update_changelog_and_version(): @@ -193,8 +189,9 @@ def upload_distribution(): lines = f.readlines() entries = [i for i, l in enumerate(lines) if CHANGELOG_HEADER.match(l)] anchor = current_version().replace(".", "-") - changelog_body = "".join(lines[entries[0] + 2 : entries[1]]).strip() + ( - "\n\n*[The canonical version of these notes (with links) is on readthedocs.]" + changelog_body = ( + "".join(lines[entries[0] + 2 : entries[1]]).strip() + + "\n\n*[The canonical version of these notes (with links) is on readthedocs.]" f"(https://hypothesis.readthedocs.io/en/latest/changes.html#v{anchor})*" ) diff --git a/tooling/src/hypothesistooling/projects/hypothesisruby.py b/tooling/src/hypothesistooling/projects/hypothesisruby.py index 89fef77c96..06b85de7b9 100644 --- a/tooling/src/hypothesistooling/projects/hypothesisruby.py +++ b/tooling/src/hypothesistooling/projects/hypothesisruby.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import os import subprocess @@ -103,7 +98,7 @@ def tag_name(): def has_source_changes(): """Returns True if any source files have changed.""" - return tools.has_changes([RUST_SRC, RUBY_SRC]) or cr.has_release() + return tools.has_changes([RUST_SRC, RUBY_SRC, GEMSPEC_FILE]) or cr.has_release() def current_version(): diff --git a/tooling/src/hypothesistooling/releasemanagement.py b/tooling/src/hypothesistooling/releasemanagement.py index c732f48c94..5b2ddc03c3 100644 --- a/tooling/src/hypothesistooling/releasemanagement.py +++ b/tooling/src/hypothesistooling/releasemanagement.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER """Helpful common code for release management tasks that is shared across multiple projects. @@ -48,7 +43,7 @@ def assignment_matcher(name): i.e. group 1 is the assignment, group 2 is the value. In the above example group 1 would be " foo = " and group 2 would be "1" """ - return re.compile(r"\A(\s*{}\s*=\s*)(.+)\Z".format(re.escape(name))) + return re.compile(rf"\A(\s*{re.escape(name)}\s*=\s*)(.+)\Z") def extract_assignment_from_string(contents, name): diff --git a/tooling/src/hypothesistooling/scripts.py b/tooling/src/hypothesistooling/scripts.py index 607e9120bd..7a3b74c3b5 100644 --- a/tooling/src/hypothesistooling/scripts.py +++ b/tooling/src/hypothesistooling/scripts.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import os import re diff --git a/whole-repo-tests/test_ci_config.py b/whole-repo-tests/test_ci_config.py new file mode 100644 index 0000000000..3a6897206b --- /dev/null +++ b/whole-repo-tests/test_ci_config.py @@ -0,0 +1,27 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +from pathlib import Path + +import pytest + +from hypothesistooling.__main__ import PYTHONS + +ci_checks = " ".join( + line.strip() + for line in Path(".github/workflows/main.yml").read_text().splitlines() + if "- check-py" in line +) + + +@pytest.mark.parametrize("version", sorted(PYTHONS)) +def test_python_versions_are_tested_in_ci(version): + slug = version.replace("pypy", "py").replace(".", "") + assert f"- check-py{slug}" in ci_checks, f"Add {version} to main.yml and tox.ini" diff --git a/whole-repo-tests/test_deploy.py b/whole-repo-tests/test_deploy.py index fe30a89353..50d417f338 100644 --- a/whole-repo-tests/test_deploy.py +++ b/whole-repo-tests/test_deploy.py @@ -1,22 +1,18 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import os -import hypothesistooling as tools import pytest + +import hypothesistooling as tools from hypothesistooling import __main__ as main, releasemanagement as rm @@ -24,7 +20,8 @@ "project", [p for p in tools.all_projects() if p.has_release()] ) def test_release_file_exists_and_is_valid(project, monkeypatch): - assert not tools.has_uncommitted_changes(project.BASE_DIR) + if not tools.has_uncommitted_changes(project.BASE_DIR): + pytest.xfail("Cannot run release process with uncommitted changes") monkeypatch.setattr(tools, "create_tag", lambda *args, **kwargs: None) monkeypatch.setattr(tools, "push_tag", lambda name: None) diff --git a/whole-repo-tests/test_documentation.py b/whole-repo-tests/test_documentation.py index 6bda2a1051..4b80523e5d 100644 --- a/whole-repo-tests/test_documentation.py +++ b/whole-repo-tests/test_documentation.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesistooling.__main__ import documentation diff --git a/whole-repo-tests/test_mypy.py b/whole-repo-tests/test_mypy.py new file mode 100644 index 0000000000..7cc2d2eec7 --- /dev/null +++ b/whole-repo-tests/test_mypy.py @@ -0,0 +1,565 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import subprocess +import textwrap + +import pytest + +from hypothesistooling.projects.hypothesispython import PYTHON_SRC +from hypothesistooling.scripts import pip_tool, tool_path + +PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] + + +def test_mypy_passes_on_hypothesis(): + pip_tool("mypy", PYTHON_SRC) + + +@pytest.mark.skip( + reason="Hypothesis type-annotates the public API as a convenience for users, " + "but strict checks for our internals would be a net drag on productivity." +) +def test_mypy_passes_on_hypothesis_strict(): + pip_tool("mypy", "--strict", PYTHON_SRC) + + +def get_mypy_output(fname, *extra_args): + return subprocess.run( + [tool_path("mypy"), *extra_args, fname], + encoding="utf-8", + capture_output=True, + text=True, + ).stdout + + +def get_mypy_analysed_type(fname, val): + out = get_mypy_output(fname).rstrip() + msg = "Success: no issues found in 1 source file" + if out.endswith(msg): + out = out[: -len(msg)] + assert len(out.splitlines()) == 1 + # See https://mypy.readthedocs.io/en/latest/common_issues.html#reveal-type + # The shell output for `reveal_type([1, 2, 3])` looks like a literal: + # file.py:2: error: Revealed type is 'builtins.list[builtins.int*]' + return ( + out.split("Revealed type is ")[1] + .strip() + .strip('"' + "'") + .replace("builtins.", "") + .replace("*", "") + .replace( + "hypothesis.strategies._internal.strategies.SearchStrategy", + "SearchStrategy", + ) + ) + + +def assert_mypy_errors(fname, expected, python_version=None): + _args = ["--no-error-summary", "--show-error-codes"] + + if python_version: + _args.append(f"--python-version={python_version}") + + out = get_mypy_output(fname, *_args) + del _args + # Shell output looks like: + # file.py:2: error: Incompatible types in assignment ... [assignment] + + def convert_lines(): + for error_line in out.splitlines(): + col, category = error_line.split(":")[-3:-1] + if category.strip() != "error": + # mypy outputs "note" messages for overload problems, even with + # --hide-error-context. Don't include these + continue + + # Intentional print so we can check mypy's output if a test fails + print(error_line) + error_code = error_line.split("[")[-1].rstrip("]") + if error_code == "empty-body": + continue + yield (int(col), error_code) + + assert sorted(convert_lines()) == sorted(expected) + + +@pytest.mark.parametrize( + "val,expect", + [ + ("integers()", "int"), + ("text()", "str"), + ("integers().map(str)", "str"), + ("booleans().filter(bool)", "bool"), + ("lists(none())", "list[None]"), + ("dictionaries(integers(), datetimes())", "dict[int, datetime.datetime]"), + ("data()", "hypothesis.strategies._internal.core.DataObject"), + ("none() | integers()", "Union[None, int]"), + # Ex`-1 stands for recursion in the whole type, i.e. Ex`0 == Union[...] + ("recursive(integers(), lists)", "Union[list[Ex`-1], int]"), + # We have overloads for up to five types, then fall back to Any. + # (why five? JSON atoms are None|bool|int|float|str and we do that a lot) + ("one_of(integers(), text())", "Union[int, str]"), + ( + "one_of(integers(), text(), none(), binary(), builds(list))", + "Union[int, str, None, bytes, list[_T`1]]", + ), + ( + "one_of(integers(), text(), none(), binary(), builds(list), builds(dict))", + "Any", + ), + ("tuples()", "Tuple[]"), # Should be `Tuple[()]`, but this is what mypy prints + ("tuples(integers())", "Tuple[int]"), + ("tuples(integers(), text())", "Tuple[int, str]"), + ( + "tuples(integers(), text(), integers(), text(), integers())", + "Tuple[int, str, int, str, int]", + ), + ( + "tuples(text(), text(), text(), text(), text(), text())", + "tuple[Any, ...]", + ), + ], +) +def test_revealed_types(tmpdir, val, expect): + """Check that Mypy picks up the expected `X` in SearchStrategy[`X`].""" + f = tmpdir.join(expect + ".py") + f.write( + "from hypothesis.strategies import *\n" + f"s = {val}\n" + "reveal_type(s)\n" # fmt: skip + ) + typ = get_mypy_analysed_type(str(f.realpath()), val) + assert typ == f"SearchStrategy[{expect}]" + + +def test_data_object_type_tracing(tmpdir): + f = tmpdir.join("check_mypy_on_st_data.py") + f.write( + "from hypothesis.strategies import data, integers\n" + "d = data().example()\n" + "s = d.draw(integers())\n" + "reveal_type(s)\n" + ) + got = get_mypy_analysed_type(str(f.realpath()), "data().draw(integers())") + assert got == "int" + + +def test_drawfn_type_tracing(tmpdir): + f = tmpdir.join("check_mypy_on_st_drawfn.py") + f.write( + "from hypothesis.strategies import DrawFn, text\n" + "def comp(draw: DrawFn) -> str:\n" + " s = draw(text(), 123)\n" + " reveal_type(s)\n" + " return s\n" + ) + got = get_mypy_analysed_type(str(f.realpath()), ...) + assert got == "str" + + +def test_composite_type_tracing(tmpdir): + f = tmpdir.join("check_mypy_on_st_composite.py") + f.write( + "from hypothesis.strategies import composite, DrawFn\n" + "@composite\n" + "def comp(draw: DrawFn, x: int) -> int:\n" + " return x\n" + "reveal_type(comp)\n" + ) + got = get_mypy_analysed_type(str(f.realpath()), ...) + assert got == "def (x: int) -> SearchStrategy[int]" + + +@pytest.mark.parametrize( + "source, expected", + [ + ("", "def ()"), + ("like=f", "def (x: int) -> int"), + ("returns=booleans()", "def () -> bool"), + ("like=f, returns=booleans()", "def (x: int) -> bool"), + ], +) +def test_functions_type_tracing(tmpdir, source, expected): + f = tmpdir.join("check_mypy_on_st_composite.py") + f.write( + "from hypothesis.strategies import booleans, functions\n" + "def f(x: int) -> int: return x\n" + f"g = functions({source}).example()\n" + "reveal_type(g)\n" + ) + got = get_mypy_analysed_type(str(f.realpath()), ...) + assert got == expected, (got, expected) + + +def test_settings_preserves_type(tmpdir): + f = tmpdir.join("check_mypy_on_settings.py") + f.write( + "from hypothesis import settings\n" + "@settings(max_examples=10)\n" + "def f(x: int) -> int:\n" + " return x\n" + "reveal_type(f)\n" + ) + got = get_mypy_analysed_type(str(f.realpath()), ...) + assert got == "def (x: int) -> int" + + +def test_stateful_bundle_generic_type(tmpdir): + f = tmpdir.join("check_mypy_on_stateful_bundle.py") + f.write( + "from hypothesis.stateful import Bundle\n" + "b: Bundle[int] = Bundle('test')\n" + "reveal_type(b.example())\n" + ) + got = get_mypy_analysed_type(str(f.realpath()), ...) + assert got == "int" + + +@pytest.mark.parametrize("decorator", ["rule", "initialize"]) +@pytest.mark.parametrize( + "target_args", + [ + "target=b1", + "targets=(b1,)", + "targets=(b1, b2)", + ], +) +@pytest.mark.parametrize("returns", ["int", "MultipleResults[int]"]) +def test_stateful_rule_targets(tmpdir, decorator, target_args, returns): + f = tmpdir.join("check_mypy_on_stateful_rule.py") + f.write( + "from hypothesis.stateful import *\n" + "b1: Bundle[int] = Bundle('b1')\n" + "b2: Bundle[int] = Bundle('b2')\n" + f"@{decorator}({target_args})\n" + f"def my_rule() -> {returns}:\n" + " ...\n" + ) + assert_mypy_errors(str(f.realpath()), []) + + +@pytest.mark.parametrize("decorator", ["rule", "initialize"]) +def test_stateful_rule_no_targets(tmpdir, decorator): + f = tmpdir.join("check_mypy_on_stateful_rule.py") + f.write( + "from hypothesis.stateful import *\n" + f"@{decorator}()\n" + "def my_rule() -> None:\n" + " ...\n" + ) + assert_mypy_errors(str(f.realpath()), []) + + +@pytest.mark.parametrize("decorator", ["rule", "initialize"]) +def test_stateful_target_params_mutually_exclusive(tmpdir, decorator): + f = tmpdir.join("check_mypy_on_stateful_rule.py") + f.write( + "from hypothesis.stateful import *\n" + "b1: Bundle[int] = Bundle('b1')\n" + f"@{decorator}(target=b1, targets=(b1,))\n" + "def my_rule() -> int:\n" + " ...\n" + ) + # Also outputs "misc" error "Untyped decorator makes function "my_rule" + # untyped, due to the inability to resolve to an appropriate overloaded + # variant + assert_mypy_errors(str(f.realpath()), [(3, "call-overload"), (3, "misc")]) + + +@pytest.mark.parametrize("decorator", ["rule", "initialize"]) +@pytest.mark.parametrize( + "target_args", + [ + "target=b1", + "targets=(b1,)", + "targets=(b1, b2)", + "", + ], +) +@pytest.mark.parametrize("returns", ["int", "MultipleResults[int]"]) +def test_stateful_target_params_return_type(tmpdir, decorator, target_args, returns): + f = tmpdir.join("check_mypy_on_stateful_rule.py") + f.write( + "from hypothesis.stateful import *\n" + "b1: Bundle[str] = Bundle('b1')\n" + "b2: Bundle[str] = Bundle('b2')\n" + f"@{decorator}({target_args})\n" + f"def my_rule() -> {returns}:\n" + " ...\n" + ) + assert_mypy_errors(str(f.realpath()), [(4, "arg-type")]) + + +@pytest.mark.parametrize("decorator", ["rule", "initialize"]) +def test_stateful_no_target_params_return_type(tmpdir, decorator): + f = tmpdir.join("check_mypy_on_stateful_rule.py") + f.write( + "from hypothesis.stateful import *\n" + f"@{decorator}()\n" + "def my_rule() -> int:\n" + " ...\n" + ) + assert_mypy_errors(str(f.realpath()), [(2, "arg-type")]) + + +@pytest.mark.parametrize("decorator", ["rule", "initialize"]) +@pytest.mark.parametrize("use_multi", [True, False]) +def test_stateful_bundle_variance(tmpdir, decorator, use_multi): + f = tmpdir.join("check_mypy_on_stateful_bundle.py") + if use_multi: + return_type = "MultipleResults[Dog]" + return_expr = "multiple(dog, dog)" + else: + return_type = "Dog" + return_expr = "dog" + + f.write( + "from hypothesis.stateful import *\n" + "class Animal: pass\n" + "class Dog(Animal): pass\n" + "a: Bundle[Animal] = Bundle('animal')\n" + "d: Bundle[Dog] = Bundle('dog')\n" + f"@{decorator}(target=a, dog=d)\n" + f"def my_rule(dog: Dog) -> {return_type}:\n" + f" return {return_expr}\n" + ) + assert_mypy_errors(str(f.realpath()), []) + + +@pytest.mark.parametrize("decorator", ["rule", "initialize"]) +def test_stateful_multiple_return(tmpdir, decorator): + f = tmpdir.join("check_mypy_on_stateful_multiple.py") + f.write( + "from hypothesis.stateful import *\n" + "b: Bundle[int] = Bundle('b')\n" + f"@{decorator}(target=b)\n" + "def my_rule() -> MultipleResults[int]:\n" + " return multiple(1, 2, 3)\n" + ) + assert_mypy_errors(str(f.realpath()), []) + + +@pytest.mark.parametrize("decorator", ["rule", "initialize"]) +def test_stateful_multiple_return_invalid(tmpdir, decorator): + f = tmpdir.join("check_mypy_on_stateful_multiple.py") + f.write( + "from hypothesis.stateful import *\n" + "b: Bundle[str] = Bundle('b')\n" + f"@{decorator}(target=b)\n" + "def my_rule() -> MultipleResults[int]:\n" + " return multiple(1, 2, 3)\n" + ) + assert_mypy_errors(str(f.realpath()), [(3, "arg-type")]) + + +@pytest.mark.parametrize( + "wrapper,expected", + [ + ("{}", "int"), + ("st.lists({})", "list[int]"), + ], +) +def test_stateful_consumes_type_tracing(tmpdir, wrapper, expected): + f = tmpdir.join("check_mypy_on_stateful_rule.py") + wrapped = wrapper.format("consumes(b)") + f.write( + "from hypothesis.stateful import *\n" + "from hypothesis import strategies as st\n" + "b: Bundle[int] = Bundle('b')\n" + f"s = {wrapped}\n" + "reveal_type(s.example())\n" + ) + got = get_mypy_analysed_type(str(f.realpath()), ...) + assert got == expected + + +def test_stateful_consumed_bundle_cannot_be_target(tmpdir): + f = tmpdir.join("check_mypy_on_stateful_rule.py") + f.write( + "from hypothesis.stateful import *\n" + "b: Bundle[int] = Bundle('b')\n" + "rule(target=consumes(b))\n" + ) + assert_mypy_errors(str(f.realpath()), [(3, "call-overload")]) + + +@pytest.mark.parametrize( + "return_val,errors", + [ + ("True", []), + ("0", [(2, "arg-type"), (2, "return-value")]), + ], +) +def test_stateful_precondition_requires_predicate(tmpdir, return_val, errors): + f = tmpdir.join("check_mypy_on_stateful_precondition.py") + f.write( + "from hypothesis.stateful import *\n" + f"@precondition(lambda self: {return_val})\n" + "def my_rule() -> None: ...\n" + ) + assert_mypy_errors(str(f.realpath()), errors) + + +def test_stateful_precondition_lambda(tmpdir): + f = tmpdir.join("check_mypy_on_stateful_precondition.py") + f.write( + "from hypothesis.stateful import *\n" + "class MyMachine(RuleBasedStateMachine):\n" + " valid: bool\n" + " @precondition(lambda self: self.valid)\n" + " @rule()\n" + " def my_rule(self) -> None: ...\n" + ) + # Note that this doesn't fully check the code because of the `Any` parameter + # type. `lambda self: self.invalid` would unfortunately pass too + assert_mypy_errors(str(f.realpath()), []) + + +def test_stateful_precondition_instance_method(tmpdir): + f = tmpdir.join("check_mypy_on_stateful_precondition.py") + f.write( + "from hypothesis.stateful import *\n" + "class MyMachine(RuleBasedStateMachine):\n" + " valid: bool\n" + " def check(self) -> bool:\n" + " return self.valid\n" + " @precondition(check)\n" + " @rule()\n" + " def my_rule(self) -> None: ...\n" + ) + assert_mypy_errors(str(f.realpath()), []) + + +def test_stateful_precondition_precond_requires_one_arg(tmpdir): + f = tmpdir.join("check_mypy_on_stateful_precondition.py") + f.write( + "from hypothesis.stateful import *\n" + "precondition(lambda: True)\n" + "precondition(lambda a, b: True)\n" + ) + # Additional "Cannot infer type of lambda" errors + assert_mypy_errors( + str(f.realpath()), + [(2, "arg-type"), (2, "misc"), (3, "arg-type"), (3, "misc")], + ) + + +def test_pos_only_args(tmpdir): + f = tmpdir.join("check_mypy_on_pos_arg_only_strats.py") + f.write( + textwrap.dedent( + """ + import hypothesis.strategies as st + + st.tuples(a1=st.integers()) + st.tuples(a1=st.integers(), a2=st.integers()) + + st.one_of(a1=st.integers()) + st.one_of(a1=st.integers(), a2=st.integers()) + """ + ) + ) + assert_mypy_errors( + str(f.realpath()), + [ + (4, "call-overload"), + (5, "call-overload"), + (7, "call-overload"), + (8, "call-overload"), + ], + ) + + +@pytest.mark.parametrize("python_version", PYTHON_VERSIONS) +def test_mypy_passes_on_basic_test(tmpdir, python_version): + f = tmpdir.join("check_mypy_on_basic_tests.py") + f.write( + textwrap.dedent( + """ + import hypothesis + import hypothesis.strategies as st + + @hypothesis.given(x=st.text()) + def test_foo(x: str) -> None: + assert x == x + + from hypothesis import given + from hypothesis.strategies import text + + @given(x=text()) + def test_bar(x: str) -> None: + assert x == x + """ + ) + ) + assert_mypy_errors(str(f.realpath()), [], python_version=python_version) + + +@pytest.mark.parametrize("python_version", PYTHON_VERSIONS) +def test_given_only_allows_strategies(tmpdir, python_version): + f = tmpdir.join("check_mypy_given_expects_strategies.py") + f.write( + textwrap.dedent( + """ + from hypothesis import given + + @given(1) + def f(): + pass + """ + ) + ) + assert_mypy_errors( + str(f.realpath()), [(4, "call-overload")], python_version=python_version + ) + + +@pytest.mark.parametrize("python_version", PYTHON_VERSIONS) +def test_raises_for_mixed_pos_kwargs_in_given(tmpdir, python_version): + f = tmpdir.join("raises_for_mixed_pos_kwargs_in_given.py") + f.write( + textwrap.dedent( + """ + from hypothesis import given + from hypothesis.strategies import text + + @given(text(), x=text()) + def test_bar(x): + ... + """ + ) + ) + assert_mypy_errors( + str(f.realpath()), [(5, "call-overload")], python_version=python_version + ) + + +def test_register_random_interface(tmpdir): + f = tmpdir.join("check_mypy_on_pos_arg_only_strats.py") + f.write( + textwrap.dedent( + """ + from random import Random + from hypothesis import register_random + + class MyRandom: + def __init__(self) -> None: + r = Random() + self.seed = r.seed + self.setstate = r.setstate + self.getstate = r.getstate + + register_random(MyRandom()) + register_random(None) # type: ignore[arg-type] + """ + ) + ) + assert_mypy_errors(str(f.realpath()), []) diff --git a/whole-repo-tests/test_pyright.py b/whole-repo-tests/test_pyright.py new file mode 100644 index 0000000000..89f79f70d1 --- /dev/null +++ b/whole-repo-tests/test_pyright.py @@ -0,0 +1,246 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +from __future__ import annotations + +import json +import subprocess +import textwrap +from pathlib import Path +from typing import Any + +import pytest + +from hypothesistooling.projects.hypothesispython import HYPOTHESIS_PYTHON, PYTHON_SRC +from hypothesistooling.scripts import pip_tool, tool_path + +PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] + + +@pytest.mark.skip( + reason="Hypothesis type-annotates the public API as a convenience for users, " + "but strict checks for our internals would be a net drag on productivity." +) +def test_pyright_passes_on_hypothesis(): + pip_tool("pyright", "--project", HYPOTHESIS_PYTHON) + + +@pytest.mark.parametrize("python_version", PYTHON_VERSIONS) +def test_pyright_passes_on_basic_test(tmp_path: Path, python_version: str): + file = tmp_path / "test.py" + file.write_text( + textwrap.dedent( + """ + import hypothesis + import hypothesis.strategies as st + + @hypothesis.given(x=st.text()) + def test_foo(x: str): + assert x == x + + from hypothesis import given + from hypothesis.strategies import text + + @given(x=text()) + def test_bar(x: str): + assert x == x + """ + ) + ) + _write_config( + tmp_path, {"typeCheckingMode": "strict", "pythonVersion": python_version} + ) + assert _get_pyright_errors(file) == [] + + +@pytest.mark.parametrize("python_version", PYTHON_VERSIONS) +def test_given_only_allows_strategies(tmp_path: Path, python_version: str): + file = tmp_path / "test.py" + file.write_text( + textwrap.dedent( + """ + from hypothesis import given + + @given(1) + def f(): + pass + """ + ) + ) + _write_config( + tmp_path, {"typeCheckingMode": "strict", "pythonVersion": python_version} + ) + assert ( + sum( + e["message"].startswith( + 'Argument of type "Literal[1]" cannot be assigned to parameter "_given_arguments"' + ) + for e in _get_pyright_errors(file) + ) + == 1 + ) + + +def test_pyright_issue_3296(tmp_path: Path): + file = tmp_path / "test.py" + file.write_text( + textwrap.dedent( + """ + from hypothesis.strategies import lists, integers + + lists(integers()).map(sorted) + """ + ) + ) + _write_config(tmp_path, {"typeCheckingMode": "strict"}) + assert _get_pyright_errors(file) == [] + + +def test_pyright_raises_for_mixed_pos_kwargs_in_given(tmp_path: Path): + file = tmp_path / "test.py" + file.write_text( + textwrap.dedent( + """ + from hypothesis import given + from hypothesis.strategies import text + + @given(text(), x=text()) + def test_bar(x: str): + pass + """ + ) + ) + _write_config(tmp_path, {"typeCheckingMode": "strict"}) + assert ( + sum( + e["message"].startswith( + 'No overloads for "given" match the provided arguments' + ) + for e in _get_pyright_errors(file) + ) + == 1 + ) + + +def test_pyright_issue_3348(tmp_path: Path): + file = tmp_path / "test.py" + file.write_text( + textwrap.dedent( + """ + import hypothesis.strategies as st + + st.tuples(st.integers(), st.integers()) + st.one_of(st.integers(), st.integers()) + st.one_of([st.integers(), st.floats()]) # sequence of strats should be OK + st.sampled_from([1, 2]) + """ + ) + ) + _write_config(tmp_path, {"typeCheckingMode": "strict"}) + assert _get_pyright_errors(file) == [] + + +def test_pyright_tuples_pos_args_only(tmp_path: Path): + file = tmp_path / "test.py" + file.write_text( + textwrap.dedent( + """ + import hypothesis.strategies as st + + st.tuples(a1=st.integers()) + st.tuples(a1=st.integers(), a2=st.integers()) + """ + ) + ) + _write_config(tmp_path, {"typeCheckingMode": "strict"}) + assert ( + sum( + e["message"].startswith( + 'No overloads for "tuples" match the provided arguments' + ) + for e in _get_pyright_errors(file) + ) + == 2 + ) + + +def test_pyright_one_of_pos_args_only(tmp_path: Path): + file = tmp_path / "test.py" + file.write_text( + textwrap.dedent( + """ + import hypothesis.strategies as st + + st.one_of(a1=st.integers()) + st.one_of(a1=st.integers(), a2=st.integers()) + """ + ) + ) + _write_config(tmp_path, {"typeCheckingMode": "strict"}) + assert ( + sum( + e["message"].startswith( + 'No overloads for "one_of" match the provided arguments' + ) + for e in _get_pyright_errors(file) + ) + == 2 + ) + + +def test_register_random_protocol(tmp_path: Path): + file = tmp_path / "test.py" + file.write_text( + textwrap.dedent( + """ + from random import Random + from hypothesis import register_random + + class MyRandom: + def __init__(self) -> None: + r = Random() + self.seed = r.seed + self.setstate = r.setstate + self.getstate = r.getstate + + register_random(MyRandom()) + register_random(None) # type: ignore + """ + ) + ) + _write_config(tmp_path, {"reportUnnecessaryTypeIgnoreComment": True}) + assert _get_pyright_errors(file) == [] + + +# ---------- Helpers for running pyright ---------- # + + +def _get_pyright_output(file: Path) -> dict[str, Any]: + proc = subprocess.run( + [tool_path("pyright"), "--outputjson"], + cwd=file.parent, + encoding="utf-8", + text=True, + capture_output=True, + ) + try: + return json.loads(proc.stdout) + except Exception: + print(proc.stdout) + raise + + +def _get_pyright_errors(file: Path) -> list[dict[str, Any]]: + return _get_pyright_output(file)["generalDiagnostics"] + + +def _write_config(config_dir: Path, data: dict[str, Any] | None = None): + config = {"extraPaths": [PYTHON_SRC], **(data or {})} + (config_dir / "pyrightconfig.json").write_text(json.dumps(config)) diff --git a/whole-repo-tests/test_release_files.py b/whole-repo-tests/test_release_files.py index d583a9eec1..95bbb3e83f 100644 --- a/whole-repo-tests/test_release_files.py +++ b/whole-repo-tests/test_release_files.py @@ -1,21 +1,18 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER -import hypothesistooling as tools import pytest + +import hypothesistooling as tools from hypothesistooling import releasemanagement as rm +from hypothesistooling.projects import hypothesispython as hp @pytest.mark.parametrize("project", tools.all_projects()) @@ -26,3 +23,14 @@ def test_release_file_exists_and_is_valid(project): "one to describe your changes." ) rm.parse_release_file(project.RELEASE_FILE) + + +@pytest.mark.skipif(not hp.has_release(), reason="Checking that release") +def test_release_file_has_no_merge_conflicts(): + _, message = rm.parse_release_file(hp.RELEASE_FILE) + assert "<<<" not in message, "Merge conflict in RELEASE.rst" + _, *recent_changes, _ = hp.CHANGELOG_ANCHOR.split(hp.changelog(), maxsplit=12) + for entry in recent_changes: + _, version, old_msg = (x.strip() for x in hp.CHANGELOG_BORDER.split(entry)) + assert message not in old_msg, f"Release notes already published for {version}" + assert old_msg not in message, f"Copied {version} release notes - merge error?" diff --git a/whole-repo-tests/test_release_management.py b/whole-repo-tests/test_release_management.py index 6610b4666e..d02563fc76 100644 --- a/whole-repo-tests/test_release_management.py +++ b/whole-repo-tests/test_release_management.py @@ -1,19 +1,15 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import pytest + from hypothesistooling.releasemanagement import ( bump_version_info, parse_release_file_contents, diff --git a/whole-repo-tests/test_requirements.py b/whole-repo-tests/test_requirements.py index 193b16224e..de0cb64e9e 100644 --- a/whole-repo-tests/test_requirements.py +++ b/whole-repo-tests/test_requirements.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER from hypothesistooling.__main__ import check_requirements diff --git a/whole-repo-tests/test_rst_is_valid.py b/whole-repo-tests/test_rst_is_valid.py index 50a76529b5..51c673cbb6 100644 --- a/whole-repo-tests/test_rst_is_valid.py +++ b/whole-repo-tests/test_rst_is_valid.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import os @@ -33,7 +28,7 @@ def is_sphinx(f): def test_passes_rst_lint(): - pip_tool("rst-lint", *[f for f in ALL_RST if not is_sphinx(f)]) + pip_tool("rst-lint", *(f for f in ALL_RST if not is_sphinx(f))) def test_passes_flake8(): diff --git a/whole-repo-tests/test_shellcheck.py b/whole-repo-tests/test_shellcheck.py index 821256b33d..996b5c74b7 100644 --- a/whole-repo-tests/test_shellcheck.py +++ b/whole-repo-tests/test_shellcheck.py @@ -1,17 +1,12 @@ # This file is part of Hypothesis, which may be found at # https://github.com/HypothesisWorks/hypothesis/ # -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. # # This Source Code Form is subject to the terms of the Mozilla Public License, # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER import subprocess diff --git a/whole-repo-tests/test_type_hints.py b/whole-repo-tests/test_type_hints.py deleted file mode 100644 index 94882e9237..0000000000 --- a/whole-repo-tests/test_type_hints.py +++ /dev/null @@ -1,98 +0,0 @@ -# This file is part of Hypothesis, which may be found at -# https://github.com/HypothesisWorks/hypothesis/ -# -# Most of this work is copyright (C) 2013-2021 David R. MacIver -# (david@drmaciver.com), but it contains contributions by others. See -# CONTRIBUTING.rst for a full list of people who may hold copyright, and -# consult the git log if you need to determine who owns an individual -# contribution. -# -# This Source Code Form is subject to the terms of the Mozilla Public License, -# v. 2.0. If a copy of the MPL was not distributed with this file, You can -# obtain one at https://mozilla.org/MPL/2.0/. -# -# END HEADER - -import os -import subprocess - -import pytest -from hypothesistooling.projects.hypothesispython import PYTHON_SRC -from hypothesistooling.scripts import pip_tool, tool_path - - -def test_mypy_passes_on_hypothesis(): - pip_tool("mypy", PYTHON_SRC) - - -def get_mypy_analysed_type(fname, val): - out = subprocess.Popen( - [tool_path("mypy"), fname], - stdout=subprocess.PIPE, - encoding="utf-8", - universal_newlines=True, - # We set the MYPYPATH explicitly, because PEP561 discovery wasn't - # working in CI as of mypy==0.730 - hopefully a temporary workaround. - env=dict(os.environ, MYPYPATH=PYTHON_SRC), - ).stdout.read() - assert len(out.splitlines()) == 1 - # See https://mypy.readthedocs.io/en/latest/common_issues.html#reveal-type - # The shell output for `reveal_type([1, 2, 3])` looks like a literal: - # file.py:2: error: Revealed type is 'builtins.list[builtins.int*]' - return ( - out.split("Revealed type is ")[1] - .strip() - .strip("'") - .replace("builtins.", "") - .replace("*", "") - ) - - -@pytest.mark.parametrize( - "val,expect", - [ - ("integers()", "int"), - ("text()", "str"), - ("integers().map(str)", "str"), - ("booleans().filter(bool)", "bool"), - ("lists(none())", "list[None]"), - ("dictionaries(integers(), datetimes())", "dict[int, datetime.datetime]"), - ("data()", "hypothesis.strategies._internal.core.DataObject"), - ("none() | integers()", "Union[None, int]"), - # Ex`-1 stands for recursion in the whole type, i.e. Ex`0 == Union[...] - ("recursive(integers(), lists)", "Union[list[Ex`-1], int]"), - # We have overloads for up to five types, then fall back to Any. - # (why five? JSON atoms are None|bool|int|float|str and we do that a lot) - ("one_of(integers(), text())", "Union[int, str]"), - ( - "one_of(integers(), text(), none(), binary(), builds(list))", - "Union[int, str, None, bytes, list[_T`1]]", - ), - ( - "one_of(integers(), text(), none(), binary(), builds(list), builds(dict))", - "Any", - ), - ], -) -def test_revealed_types(tmpdir, val, expect): - """Check that Mypy picks up the expected `X` in SearchStrategy[`X`].""" - f = tmpdir.join(expect + ".py") - f.write( - "from hypothesis.strategies import *\n" - "s = {}\n" - "reveal_type(s)\n".format(val) - ) - typ = get_mypy_analysed_type(str(f.realpath()), val) - assert typ == f"hypothesis.strategies._internal.strategies.SearchStrategy[{expect}]" - - -def test_data_object_type_tracing(tmpdir): - f = tmpdir.join("check_mypy_on_st_data.py") - f.write( - "from hypothesis.strategies import data, integers\n" - "d = data().example()\n" - "s = d.draw(integers())\n" - "reveal_type(s)\n" - ) - got = get_mypy_analysed_type(str(f.realpath()), "data().draw(integers())") - assert got == "int" diff --git a/whole-repo-tests/test_validate_branch_check.py b/whole-repo-tests/test_validate_branch_check.py new file mode 100644 index 0000000000..d236762982 --- /dev/null +++ b/whole-repo-tests/test_validate_branch_check.py @@ -0,0 +1,93 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import json +import os +import subprocess +import sys + +from hypothesistooling.projects.hypothesispython import BASE_DIR + +BRANCH_CHECK = "branch-check" +VALIDATE_BRANCH_CHECK = os.path.join(BASE_DIR, "scripts", "validate_branch_check.py") + + +def write_entries(tmp_path, entries): + with open(tmp_path / BRANCH_CHECK, "w") as f: + f.writelines([json.dumps(entry) + "\n" for entry in entries]) + + +def run_validate_branch_check(tmp_path, *, check, **kwargs): + return subprocess.run( + [sys.executable, VALIDATE_BRANCH_CHECK], + cwd=tmp_path, + text=True, + capture_output=True, + check=check, + **kwargs, + ) + + +def test_validates_branches(tmp_path): + write_entries( + tmp_path, + [ + {"name": name, "value": value} + for name in ("first", "second", "third") + for value in (False, True) + ], + ) + + output = run_validate_branch_check(tmp_path, check=True) + assert output.stdout == "Successfully validated 3 branches.\n" + + +def test_validates_one_branch(tmp_path): + write_entries( + tmp_path, [{"name": "sole", "value": value} for value in (False, True)] + ) + + output = run_validate_branch_check(tmp_path, check=True) + assert output.stdout == "Successfully validated 1 branch.\n" + + +def test_fails_on_zero_branches(tmp_path): + write_entries(tmp_path, []) + + output = run_validate_branch_check(tmp_path, check=False) + assert output.returncode == 1 + assert output.stdout == "No branches found in the branch-check file?\n" + + +def test_reports_uncovered_branches(tmp_path): + write_entries( + tmp_path, + [ + {"name": "branch that is always taken", "value": True}, + {"name": "some other branch that is never taken", "value": False}, + {"name": "covered branch", "value": True}, + {"name": "covered branch", "value": False}, + ], + ) + + output = run_validate_branch_check(tmp_path, check=False) + assert output.returncode == 1 + assert output.stdout == "\n".join( + [ + "Some branches were not properly covered.", + "", + "The following were always True:", + " * branch that is always taken", + "", + "The following were always False:", + " * some other branch that is never taken", + "", + ] + )