diff --git a/docs/html/reference/index.rst b/docs/html/reference/index.rst index f3312948193..d21b7a9801a 100644 --- a/docs/html/reference/index.rst +++ b/docs/html/reference/index.rst @@ -13,6 +13,7 @@ Reference Guide pip_list pip_show pip_search + pip_cache pip_check pip_config pip_wheel diff --git a/docs/html/reference/pip_cache.rst b/docs/html/reference/pip_cache.rst new file mode 100644 index 00000000000..8ad99f65cba --- /dev/null +++ b/docs/html/reference/pip_cache.rst @@ -0,0 +1,22 @@ + +.. _`pip cache`: + +pip cache +--------- + +.. contents:: + +Usage +***** + +.. pip-command-usage:: cache + +Description +*********** + +.. pip-command-description:: cache + +Options +******* + +.. pip-command-options:: cache diff --git a/docs/man/commands/cache.rst b/docs/man/commands/cache.rst new file mode 100644 index 00000000000..8f8e197f922 --- /dev/null +++ b/docs/man/commands/cache.rst @@ -0,0 +1,20 @@ +:orphan: + +========= +pip-cache +========= + +Description +*********** + +.. pip-command-description:: cache + +Usage +***** + +.. pip-command-usage:: cache + +Options +******* + +.. pip-command-options:: cache diff --git a/news/6391.feature b/news/6391.feature new file mode 100644 index 00000000000..e13df852713 --- /dev/null +++ b/news/6391.feature @@ -0,0 +1 @@ +Add ``pip cache`` command for inspecting/managing pip's wheel cache. diff --git a/src/pip/_internal/commands/__init__.py b/src/pip/_internal/commands/__init__.py index 2a311f8fc89..b43a96c13f3 100644 --- a/src/pip/_internal/commands/__init__.py +++ b/src/pip/_internal/commands/__init__.py @@ -64,6 +64,10 @@ 'pip._internal.commands.search', 'SearchCommand', 'Search PyPI for packages.', )), + ('cache', CommandInfo( + 'pip._internal.commands.cache', 'CacheCommand', + "Inspect and manage pip's wheel cache.", + )), ('wheel', CommandInfo( 'pip._internal.commands.wheel', 'WheelCommand', 'Build wheels from your requirements.', diff --git a/src/pip/_internal/commands/cache.py b/src/pip/_internal/commands/cache.py new file mode 100644 index 00000000000..7e3f72e080b --- /dev/null +++ b/src/pip/_internal/commands/cache.py @@ -0,0 +1,165 @@ +from __future__ import absolute_import + +import logging +import os +import textwrap + +import pip._internal.utils.filesystem as filesystem +from pip._internal.cli.base_command import Command +from pip._internal.cli.status_codes import ERROR, SUCCESS +from pip._internal.exceptions import CommandError, PipError +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from optparse import Values + from typing import Any, List + + +logger = logging.getLogger(__name__) + + +class CacheCommand(Command): + """ + Inspect and manage pip's wheel cache. + + Subcommands: + + info: Show information about the cache. + list: List filenames of packages stored in the cache. + remove: Remove one or more package from the cache. + purge: Remove all items from the cache. + + can be a glob expression or a package name. + """ + + usage = """ + %prog info + %prog list [] + %prog remove + %prog purge + """ + + def run(self, options, args): + # type: (Values, List[Any]) -> int + handlers = { + "info": self.get_cache_info, + "list": self.list_cache_items, + "remove": self.remove_cache_items, + "purge": self.purge_cache, + } + + # Determine action + if not args or args[0] not in handlers: + logger.error("Need an action ({}) to perform.".format( + ", ".join(sorted(handlers))) + ) + return ERROR + + action = args[0] + + # Error handling happens here, not in the action-handlers. + try: + handlers[action](options, args[1:]) + except PipError as e: + logger.error(e.args[0]) + return ERROR + + return SUCCESS + + def get_cache_info(self, options, args): + # type: (Values, List[Any]) -> None + if args: + raise CommandError('Too many arguments') + + num_packages = len(self._find_wheels(options, '*')) + + cache_location = self._wheels_cache_dir(options) + cache_size = filesystem.format_directory_size(cache_location) + + message = textwrap.dedent(""" + Location: {location} + Size: {size} + Number of wheels: {package_count} + """).format( + location=cache_location, + package_count=num_packages, + size=cache_size, + ).strip() + + logger.info(message) + + def list_cache_items(self, options, args): + # type: (Values, List[Any]) -> None + if len(args) > 1: + raise CommandError('Too many arguments') + + if args: + pattern = args[0] + else: + pattern = '*' + + files = self._find_wheels(options, pattern) + + if not files: + logger.info('Nothing cached.') + return + + results = [] + for filename in files: + wheel = os.path.basename(filename) + size = filesystem.format_file_size(filename) + results.append(' - {} ({})'.format(wheel, size)) + logger.info('Cache contents:\n') + logger.info('\n'.join(sorted(results))) + + def remove_cache_items(self, options, args): + # type: (Values, List[Any]) -> None + if len(args) > 1: + raise CommandError('Too many arguments') + + if not args: + raise CommandError('Please provide a pattern') + + files = self._find_wheels(options, args[0]) + if not files: + raise CommandError('No matching packages') + + for filename in files: + os.unlink(filename) + logger.debug('Removed %s', filename) + logger.info('Files removed: %s', len(files)) + + def purge_cache(self, options, args): + # type: (Values, List[Any]) -> None + if args: + raise CommandError('Too many arguments') + + return self.remove_cache_items(options, ['*']) + + def _wheels_cache_dir(self, options): + # type: (Values) -> str + return os.path.join(options.cache_dir, 'wheels') + + def _find_wheels(self, options, pattern): + # type: (Values, str) -> List[str] + wheel_dir = self._wheels_cache_dir(options) + + # The wheel filename format, as specified in PEP 427, is: + # {distribution}-{version}(-{build})?-{python}-{abi}-{platform}.whl + # + # Additionally, non-alphanumeric values in the distribution are + # normalized to underscores (_), meaning hyphens can never occur + # before `-{version}`. + # + # Given that information: + # - If the pattern we're given contains a hyphen (-), the user is + # providing at least the version. Thus, we can just append `*.whl` + # to match the rest of it. + # - If the pattern we're given doesn't contain a hyphen (-), the + # user is only providing the name. Thus, we append `-*.whl` to + # match the hyphen before the version, followed by anything else. + # + # PEP 427: https://www.python.org/dev/peps/pep-0427/ + pattern = pattern + ("*.whl" if "-" in pattern else "-*.whl") + + return filesystem.find_files(wheel_dir, pattern) diff --git a/src/pip/_internal/utils/filesystem.py b/src/pip/_internal/utils/filesystem.py index 36578fb6244..d97992acb48 100644 --- a/src/pip/_internal/utils/filesystem.py +++ b/src/pip/_internal/utils/filesystem.py @@ -1,4 +1,5 @@ import errno +import fnmatch import os import os.path import random @@ -14,10 +15,11 @@ from pip._vendor.six import PY2 from pip._internal.utils.compat import get_path_uid +from pip._internal.utils.misc import format_size from pip._internal.utils.typing import MYPY_CHECK_RUNNING, cast if MYPY_CHECK_RUNNING: - from typing import Any, BinaryIO, Iterator + from typing import Any, BinaryIO, Iterator, List, Union class NamedTemporaryFileResult(BinaryIO): @property @@ -176,3 +178,42 @@ def _test_writable_dir_win(path): raise EnvironmentError( 'Unexpected condition testing for writable directory' ) + + +def find_files(path, pattern): + # type: (str, str) -> List[str] + """Returns a list of absolute paths of files beneath path, recursively, + with filenames which match the UNIX-style shell glob pattern.""" + result = [] # type: List[str] + for root, dirs, files in os.walk(path): + matches = fnmatch.filter(files, pattern) + result.extend(os.path.join(root, f) for f in matches) + return result + + +def file_size(path): + # type: (str) -> Union[int, float] + # If it's a symlink, return 0. + if os.path.islink(path): + return 0 + return os.path.getsize(path) + + +def format_file_size(path): + # type: (str) -> str + return format_size(file_size(path)) + + +def directory_size(path): + # type: (str) -> Union[int, float] + size = 0.0 + for root, _dirs, files in os.walk(path): + for filename in files: + file_path = os.path.join(root, filename) + size += file_size(file_path) + return size + + +def format_directory_size(path): + # type: (str) -> str + return format_size(directory_size(path)) diff --git a/tests/functional/test_cache.py b/tests/functional/test_cache.py new file mode 100644 index 00000000000..a464ece7945 --- /dev/null +++ b/tests/functional/test_cache.py @@ -0,0 +1,218 @@ +import os +import shutil +from glob import glob + +import pytest + + +@pytest.fixture +def cache_dir(script): + result = script.run( + 'python', '-c', + 'from pip._internal.locations import USER_CACHE_DIR;' + 'print(USER_CACHE_DIR)' + ) + return result.stdout.strip() + + +@pytest.fixture +def wheel_cache_dir(cache_dir): + return os.path.normcase(os.path.join(cache_dir, 'wheels')) + + +@pytest.fixture +def wheel_cache_files(wheel_cache_dir): + destination = os.path.join(wheel_cache_dir, 'arbitrary', 'pathname') + + if not os.path.exists(destination): + return [] + + filenames = glob(os.path.join(destination, '*.whl')) + files = [] + for filename in filenames: + files.append(os.path.join(destination, filename)) + return files + + +@pytest.fixture +def populate_wheel_cache(wheel_cache_dir): + destination = os.path.join(wheel_cache_dir, 'arbitrary', 'pathname') + os.makedirs(destination) + + files = [ + ('yyy-1.2.3', os.path.join(destination, 'yyy-1.2.3-py3-none-any.whl')), + ('zzz-4.5.6', os.path.join(destination, 'zzz-4.5.6-py3-none-any.whl')), + ('zzz-4.5.7', os.path.join(destination, 'zzz-4.5.7-py3-none-any.whl')), + ('zzz-7.8.9', os.path.join(destination, 'zzz-7.8.9-py3-none-any.whl')), + ] + + for _name, filename in files: + with open(filename, 'w'): + pass + + return files + + +@pytest.fixture +def empty_wheel_cache(wheel_cache_dir): + if os.path.exists(wheel_cache_dir): + shutil.rmtree(wheel_cache_dir) + + +def list_matches_wheel(wheel_name, result): + """Returns True if any line in `result`, which should be the output of + a `pip cache list` call, matches `wheel_name`. + + E.g., If wheel_name is `foo-1.2.3` it searches for a line starting with + `- foo-1.2.3-py3-none-any.whl `.""" + lines = result.stdout.splitlines() + expected = ' - {}-py3-none-any.whl '.format(wheel_name) + return any(map(lambda l: l.startswith(expected), lines)) + + +@pytest.fixture +def remove_matches_wheel(wheel_cache_dir): + """Returns True if any line in `result`, which should be the output of + a `pip cache remove`/`pip cache purge` call, matches `wheel_name`. + + E.g., If wheel_name is `foo-1.2.3`, it searches for a line equal to + `Removed /arbitrary/pathname/foo-1.2.3-py3-none-any.whl`. + """ + + def _remove_matches_wheel(wheel_name, result): + lines = result.stdout.splitlines() + + wheel_filename = '{}-py3-none-any.whl'.format(wheel_name) + + # The "/arbitrary/pathname/" bit is an implementation detail of how + # the `populate_wheel_cache` fixture is implemented. + path = os.path.join( + wheel_cache_dir, 'arbitrary', 'pathname', wheel_filename, + ) + expected = 'Removed {}'.format(path) + return expected in lines + + return _remove_matches_wheel + + +@pytest.mark.usefixtures("populate_wheel_cache") +def test_cache_info(script, wheel_cache_dir, wheel_cache_files): + result = script.pip('cache', 'info') + + assert 'Location: {}'.format(wheel_cache_dir) in result.stdout + num_wheels = len(wheel_cache_files) + assert 'Number of wheels: {}'.format(num_wheels) in result.stdout + + +@pytest.mark.usefixtures("populate_wheel_cache") +def test_cache_list(script): + """Running `pip cache list` should return exactly what the + populate_wheel_cache fixture adds.""" + result = script.pip('cache', 'list') + + assert list_matches_wheel('yyy-1.2.3', result) + assert list_matches_wheel('zzz-4.5.6', result) + assert list_matches_wheel('zzz-4.5.7', result) + assert list_matches_wheel('zzz-7.8.9', result) + + +@pytest.mark.usefixtures("empty_wheel_cache") +def test_cache_list_with_empty_cache(script): + """Running `pip cache list` with an empty cache should print + "Nothing cached." and exit.""" + result = script.pip('cache', 'list') + assert result.stdout == "Nothing cached.\n" + + +def test_cache_list_too_many_args(script): + """Passing `pip cache list` too many arguments should cause an error.""" + script.pip('cache', 'list', 'aaa', 'bbb', + expect_error=True) + + +@pytest.mark.usefixtures("populate_wheel_cache") +def test_cache_list_name_match(script): + """Running `pip cache list zzz` should list zzz-4.5.6, zzz-4.5.7, + zzz-7.8.9, but nothing else.""" + result = script.pip('cache', 'list', 'zzz', '--verbose') + + assert not list_matches_wheel('yyy-1.2.3', result) + assert list_matches_wheel('zzz-4.5.6', result) + assert list_matches_wheel('zzz-4.5.7', result) + assert list_matches_wheel('zzz-7.8.9', result) + + +@pytest.mark.usefixtures("populate_wheel_cache") +def test_cache_list_name_and_version_match(script): + """Running `pip cache list zzz-4.5.6` should list zzz-4.5.6, but + nothing else.""" + result = script.pip('cache', 'list', 'zzz-4.5.6', '--verbose') + + assert not list_matches_wheel('yyy-1.2.3', result) + assert list_matches_wheel('zzz-4.5.6', result) + assert not list_matches_wheel('zzz-4.5.7', result) + assert not list_matches_wheel('zzz-7.8.9', result) + + +@pytest.mark.usefixture("populate_wheel_cache") +def test_cache_remove_no_arguments(script): + """Running `pip cache remove` with no arguments should cause an error.""" + script.pip('cache', 'remove', expect_error=True) + + +def test_cache_remove_too_many_args(script): + """Passing `pip cache remove` too many arguments should cause an error.""" + script.pip('cache', 'remove', 'aaa', 'bbb', + expect_error=True) + + +@pytest.mark.usefixtures("populate_wheel_cache") +def test_cache_remove_name_match(script, remove_matches_wheel): + """Running `pip cache remove zzz` should remove zzz-4.5.6 and zzz-7.8.9, + but nothing else.""" + result = script.pip('cache', 'remove', 'zzz', '--verbose') + + assert not remove_matches_wheel('yyy-1.2.3', result) + assert remove_matches_wheel('zzz-4.5.6', result) + assert remove_matches_wheel('zzz-4.5.7', result) + assert remove_matches_wheel('zzz-7.8.9', result) + + +@pytest.mark.usefixtures("populate_wheel_cache") +def test_cache_remove_name_and_version_match(script, remove_matches_wheel): + """Running `pip cache remove zzz-4.5.6` should remove zzz-4.5.6, but + nothing else.""" + result = script.pip('cache', 'remove', 'zzz-4.5.6', '--verbose') + + assert not remove_matches_wheel('yyy-1.2.3', result) + assert remove_matches_wheel('zzz-4.5.6', result) + assert not remove_matches_wheel('zzz-4.5.7', result) + assert not remove_matches_wheel('zzz-7.8.9', result) + + +@pytest.mark.usefixtures("populate_wheel_cache") +def test_cache_purge(script, remove_matches_wheel): + """Running `pip cache purge` should remove all cached wheels.""" + result = script.pip('cache', 'purge', '--verbose') + + assert remove_matches_wheel('yyy-1.2.3', result) + assert remove_matches_wheel('zzz-4.5.6', result) + assert remove_matches_wheel('zzz-4.5.7', result) + assert remove_matches_wheel('zzz-7.8.9', result) + + +@pytest.mark.usefixtures("populate_wheel_cache") +def test_cache_purge_too_many_args(script, wheel_cache_files): + """Running `pip cache purge aaa` should raise an error and remove no + cached wheels.""" + result = script.pip('cache', 'purge', 'aaa', '--verbose', + expect_error=True) + assert result.stdout == '' + + # This would be `result.stderr == ...`, but Pip prints deprecation + # warnings on Python 2.7, so we check if the _line_ is in stderr. + assert 'ERROR: Too many arguments' in result.stderr.splitlines() + + # Make sure nothing was deleted. + for filename in wheel_cache_files: + assert os.path.exists(filename)