diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 33fc67d2f8..2be9c0e137 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.9.0 +current_version = 1.9.1 commit = True message = Prepare for {new_version} release diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 52d1b27f5d..aec76ee0f6 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -96,6 +96,7 @@ jobs: path: ./wheelhouse/*.whl build_wheels_macos: + needs: [build_sdist] name: Wheel for MacOS-${{ matrix.cibw_python }}-${{ matrix.cibw_arch }} runs-on: ${{ matrix.os }} strategy: @@ -106,7 +107,17 @@ jobs: cibw_arch: ["x86_64", "arm64"] steps: - - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 + with: + name: dist + path: dist + - uses: actions/download-artifact@v3 + with: + name: tests + path: tests + - name: Extract sdist + run: | + tar zxvf dist/*.tar.gz --strip-components=1 - name: Sets env vars for compilation if: matrix.cibw_arch == 'arm64' run: | diff --git a/NEWS.rst b/NEWS.rst index 119459db13..448a875fd3 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -8,6 +8,17 @@ Changelog .. towncrier release notes start +memray 1.9.1 (2023-08-01) +------------------------- + +Bug Fixes +~~~~~~~~~ + +- Fix an issue that stopped Memray's experimental support for ``greenlet`` from working with versions of the ``greenlet`` module older than 1.0. (#432) +- Fix a bug leading to a deadlock when Memray is used to profile an application that uses the jemalloc implementation of ``malloc``. (#433) +- Fix a bug causing the ``summary`` reporter to generate empty reports. (#435) + + memray 1.9.0 (2023-07-28) ------------------------- diff --git a/requirements-test.txt b/requirements-test.txt index edc64d568a..690576cc3e 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,6 +1,6 @@ Cython coverage[toml] -greenlet; python_version < '3.11' +greenlet; python_version < '3.12' pytest pytest-cov ipython diff --git a/setup.py b/setup.py index 6bcb9680d7..188edb4eb8 100644 --- a/setup.py +++ b/setup.py @@ -111,7 +111,7 @@ def build_js_files(self): test_requires = [ "Cython", - "greenlet; python_version < '3.11'", + "greenlet; python_version < '3.12'", "pytest", "pytest-cov", "ipython", diff --git a/src/memray/_memray.pyx b/src/memray/_memray.pyx index 86a66f84e8..4dafb2ae4a 100644 --- a/src/memray/_memray.pyx +++ b/src/memray/_memray.pyx @@ -691,7 +691,7 @@ cdef class Tracker: self._previous_thread_profile_func = threading._profile_hook threading.setprofile(start_thread_trace) - if "greenlet._greenlet" in sys.modules: + if "greenlet" in sys.modules: NativeTracker.beginTrackingGreenlets() NativeTracker.createTracker( diff --git a/src/memray/_memray/hooks.cpp b/src/memray/_memray/hooks.cpp index e51173f2ac..cbf3bfdcbf 100644 --- a/src/memray/_memray/hooks.cpp +++ b/src/memray/_memray/hooks.cpp @@ -166,8 +166,14 @@ malloc(size_t size) noexcept { assert(hooks::malloc); - void* ptr = hooks::malloc(size); - tracking_api::Tracker::trackAllocation(ptr, size, hooks::Allocator::MALLOC); + void* ptr; + { + tracking_api::RecursionGuard guard; + ptr = hooks::malloc(size); + } + if (ptr) { + tracking_api::Tracker::trackAllocation(ptr, size, hooks::Allocator::MALLOC); + } return ptr; } @@ -182,7 +188,10 @@ free(void* ptr) noexcept tracking_api::Tracker::trackDeallocation(ptr, 0, hooks::Allocator::FREE); } - hooks::free(ptr); + { + tracking_api::RecursionGuard guard; + hooks::free(ptr); + } } void* @@ -224,8 +233,14 @@ void* mmap(void* addr, size_t length, int prot, int flags, int fd, off_t offset) noexcept { assert(hooks::mmap); - void* ptr = hooks::mmap(addr, length, prot, flags, fd, offset); - tracking_api::Tracker::trackAllocation(ptr, length, hooks::Allocator::MMAP); + void* ptr; + { + tracking_api::RecursionGuard guard; + ptr = hooks::mmap(addr, length, prot, flags, fd, offset); + } + if (ptr != MAP_FAILED) { + tracking_api::Tracker::trackAllocation(ptr, length, hooks::Allocator::MMAP); + } return ptr; } @@ -234,8 +249,14 @@ void* mmap64(void* addr, size_t length, int prot, int flags, int fd, off64_t offset) noexcept { assert(hooks::mmap64); - void* ptr = hooks::mmap64(addr, length, prot, flags, fd, offset); - tracking_api::Tracker::trackAllocation(ptr, length, hooks::Allocator::MMAP); + void* ptr; + { + tracking_api::RecursionGuard guard; + ptr = hooks::mmap64(addr, length, prot, flags, fd, offset); + } + if (ptr != MAP_FAILED) { + tracking_api::Tracker::trackAllocation(ptr, length, hooks::Allocator::MMAP); + } return ptr; } #endif @@ -245,7 +266,10 @@ munmap(void* addr, size_t length) noexcept { assert(hooks::munmap); tracking_api::Tracker::trackDeallocation(addr, length, hooks::Allocator::MUNMAP); - return hooks::munmap(addr, length); + { + tracking_api::RecursionGuard guard; + return hooks::munmap(addr, length); + } } void* @@ -291,7 +315,9 @@ dlopen(const char* filename, int flag) noexcept } if (ret) { tracking_api::Tracker::invalidate_module_cache(); - if (filename && nullptr != strstr(filename, "/_greenlet.")) { + if (filename + && (nullptr != strstr(filename, "/_greenlet.") || nullptr != strstr(filename, "/greenlet."))) + { tracking_api::Tracker::beginTrackingGreenlets(); } } @@ -303,7 +329,11 @@ dlclose(void* handle) noexcept { assert(hooks::dlclose); - int ret = hooks::dlclose(handle); + int ret; + { + tracking_api::RecursionGuard guard; + ret = hooks::dlclose(handle); + } tracking_api::NativeTrace::flushCache(); if (!ret) tracking_api::Tracker::invalidate_module_cache(); return ret; @@ -349,7 +379,11 @@ pvalloc(size_t size) noexcept { assert(hooks::pvalloc); - void* ret = hooks::pvalloc(size); + void* ret; + { + tracking_api::RecursionGuard guard; + ret = hooks::pvalloc(size); + } if (ret) { tracking_api::Tracker::trackAllocation(ret, size, hooks::Allocator::PVALLOC); } diff --git a/src/memray/_memray/tracking_api.cpp b/src/memray/_memray/tracking_api.cpp index 6961ed4fb5..61121ea2a5 100644 --- a/src/memray/_memray/tracking_api.cpp +++ b/src/memray/_memray/tracking_api.cpp @@ -340,7 +340,11 @@ PythonStackTracker::installGreenletTraceFunctionIfNeeded() // `greenlet._greenlet` has already been imported. PyObject* _greenlet = PyDict_GetItemString(modules, "greenlet._greenlet"); if (!_greenlet) { - return; + // Before greenlet 1.0, the extension module was just named "greenlet" + _greenlet = PyDict_GetItemString(modules, "greenlet"); + if (!_greenlet) { + return; + } } // Borrowed reference diff --git a/src/memray/_version.py b/src/memray/_version.py index 0a0a43a57e..38cf6dbeb5 100644 --- a/src/memray/_version.py +++ b/src/memray/_version.py @@ -1 +1 @@ -__version__ = "1.9.0" +__version__ = "1.9.1" diff --git a/src/memray/reporters/summary.py b/src/memray/reporters/summary.py index 3970bdbdd0..ce79ea48a9 100644 --- a/src/memray/reporters/summary.py +++ b/src/memray/reporters/summary.py @@ -15,7 +15,7 @@ def __init__(self, data: Iterable[AllocationRecord], native: bool): super().__init__() self.data = data self._tui = TUI(pid=None, cmd_line=None, native=native) - self._tui.update_snapshot(data) + self._tui.update_snapshot(tuple(data)) @classmethod def from_snapshot( diff --git a/src/memray/reporters/tui.py b/src/memray/reporters/tui.py index daae62cdf6..9833ff6fa3 100644 --- a/src/memray/reporters/tui.py +++ b/src/memray/reporters/tui.py @@ -11,6 +11,7 @@ from typing import Iterable from typing import List from typing import Optional +from typing import Sequence from typing import Set from typing import Tuple @@ -426,7 +427,7 @@ def generate_layout(self) -> Layout: self.layout["footer"].update(self.footer()) return self.layout - def update_snapshot(self, snapshot: Iterable[AllocationRecord]) -> None: + def update_snapshot(self, snapshot: Sequence[AllocationRecord]) -> None: for record in snapshot: if record.tid in self._seen_threads: continue diff --git a/tests/integration/test_greenlet.py b/tests/integration/test_greenlet.py index 3ec74c382a..4090aae279 100644 --- a/tests/integration/test_greenlet.py +++ b/tests/integration/test_greenlet.py @@ -10,7 +10,7 @@ from tests.utils import filter_relevant_allocations pytestmark = pytest.mark.skipif( - sys.version_info >= (3, 11), reason="Greenlet does not yet support Python 3.11" + sys.version_info >= (3, 12), reason="Greenlet does not yet support Python 3.12" ) @@ -20,10 +20,6 @@ def test_integration_with_greenlet(tmpdir): output = Path(tmpdir) / "test.bin" subprocess_code = textwrap.dedent( f""" - import mmap - import sys - import gc - import greenlet from memray import Tracker @@ -112,10 +108,6 @@ def test_importing_greenlet_after_tracking_starts(tmpdir): output = Path(tmpdir) / "test.bin" subprocess_code = textwrap.dedent( f""" - import mmap - import sys - import gc - from memray import Tracker from memray._test import MemoryAllocator @@ -155,6 +147,7 @@ def test(): with Tracker(output): import greenlet + fruit = greenlet.greenlet(apple) animal = greenlet.greenlet(ant) test() diff --git a/tests/unit/test_summary_reporter.py b/tests/unit/test_summary_reporter.py index d2d7ec5b31..83e36d50d0 100644 --- a/tests/unit/test_summary_reporter.py +++ b/tests/unit/test_summary_reporter.py @@ -171,3 +171,43 @@ def test_max_rows(): ] actual = [line.rstrip() for line in output.getvalue().splitlines()] assert actual == expected + + +def test_non_sequence_iterable(): + # GIVEN + snapshot = ( + MockAllocationRecord( + tid=1, + address=0x1000000, + size=1024 - (4 - i), + allocator=AllocatorType.MALLOC, + stack_id=1, + n_allocations=i + 1, + _stack=[ + (f"function{i}", f"/src/lel_{i}.py", i), + (f"function{i+1}", f"/src/lel_{i+1}.py", i), + ], + ) + for i in range(5) + ) + + reporter = SummaryReporter.from_snapshot(snapshot) + output = StringIO() + + # WHEN + reporter.render(sort_column=1, max_rows=3, file=output) + + # THEN + expected = [ + "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━┳━━━━━━━┳━━━━━━━┳━━━━━━━┓", + "┃ ┃ ┃ Total ┃ ┃ Own ┃ ┃", + "┃ ┃