diff --git a/docs/changelog/2265.bugfix.rst b/docs/changelog/2265.bugfix.rst new file mode 100644 index 000000000..bc90ccc7e --- /dev/null +++ b/docs/changelog/2265.bugfix.rst @@ -0,0 +1,3 @@ +Try using previous updates of ``pip``, ``setuptools`` & ``wheel`` +when inside an update grace period rather than always falling back +to embedded wheels - by :user:`mayeut`. diff --git a/docs/changelog/2266.bugfix.rst b/docs/changelog/2266.bugfix.rst new file mode 100644 index 000000000..26ff3adfd --- /dev/null +++ b/docs/changelog/2266.bugfix.rst @@ -0,0 +1,2 @@ +New patch versions of ``pip``, ``setuptools`` & ``wheel`` are now +returned in the expected timeframe. - by :user:`mayeut`. diff --git a/docs/changelog/2267.bugfix.rst b/docs/changelog/2267.bugfix.rst new file mode 100644 index 000000000..05af3cda4 --- /dev/null +++ b/docs/changelog/2267.bugfix.rst @@ -0,0 +1,2 @@ +Manual upgrades of ``pip``, ``setuptools`` & ``wheel`` are +not discarded by a periodic update - by :user:`mayeut`. diff --git a/src/virtualenv/app_data/via_disk_folder.py b/src/virtualenv/app_data/via_disk_folder.py index 257a85fd5..265db1bad 100644 --- a/src/virtualenv/app_data/via_disk_folder.py +++ b/src/virtualenv/app_data/via_disk_folder.py @@ -14,7 +14,7 @@ │ │ └── -> CopyPipInstall / SymlinkPipInstall │ │ └── -> pip-20.1.1-py2.py3-none-any │ └── embed -│ └── 1 +│ └── 2 -> json format versioning │ └── *.json -> for every distribution contains data about newer embed versions and releases └─── unzip └── @@ -101,7 +101,7 @@ def py_info_clear(self): filename.unlink() def embed_update_log(self, distribution, for_py_version): - return EmbedDistributionUpdateStoreDisk(self.lock / "wheel" / for_py_version / "embed" / "1", distribution) + return EmbedDistributionUpdateStoreDisk(self.lock / "wheel" / for_py_version / "embed" / "2", distribution) @property def house(self): diff --git a/src/virtualenv/seed/wheels/periodic_update.py b/src/virtualenv/seed/wheels/periodic_update.py index 3ea764e02..1a6cbe89e 100644 --- a/src/virtualenv/seed/wheels/periodic_update.py +++ b/src/virtualenv/seed/wheels/periodic_update.py @@ -36,6 +36,12 @@ pass # pragma: no cov +GRACE_PERIOD_CI = timedelta(hours=1) # prevent version switch in the middle of a CI run +GRACE_PERIOD_MINOR = timedelta(days=28) +UPDATE_PERIOD = timedelta(days=14) +UPDATE_ABORTED_DELAY = timedelta(hours=1) + + def periodic_update(distribution, of_version, for_py_version, wheel, search_dirs, app_data, do_periodic_update, env): if do_periodic_update: handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_data, env) @@ -48,20 +54,20 @@ def _update_wheel(ver): return updated_wheel u_log = UpdateLog.from_app_data(app_data, distribution, for_py_version) - u_log_older_than_hour = now - u_log.completed > timedelta(hours=1) if u_log.completed is not None else False if of_version is None: for _, group in groupby(u_log.versions, key=lambda v: v.wheel.version_tuple[0:2]): - version = next(group) # use only latest patch version per minor, earlier assumed to be buggy - if wheel is not None and Path(version.filename).name == wheel.name: - break - if u_log.periodic is False or (u_log_older_than_hour and version.use(now)): - wheel = _update_wheel(version) - break - elif u_log.periodic is False or u_log_older_than_hour: + # use only latest patch version per minor, earlier assumed to be buggy + all_patches = list(group) + ignore_grace_period_minor = any(version for version in all_patches if version.use(now)) + for version in all_patches: + if wheel is not None and Path(version.filename).name == wheel.name: + return wheel + if version.use(now, ignore_grace_period_minor): + return _update_wheel(version) + else: for version in u_log.versions: if version.wheel.version == of_version: - wheel = _update_wheel(version) - break + return _update_wheel(version) return wheel @@ -88,10 +94,11 @@ def load_datetime(value): class NewVersion(object): - def __init__(self, filename, found_date, release_date): + def __init__(self, filename, found_date, release_date, source): self.filename = filename self.found_date = found_date self.release_date = release_date + self.source = source @classmethod def from_dict(cls, dictionary): @@ -99,6 +106,7 @@ def from_dict(cls, dictionary): filename=dictionary["filename"], found_date=load_datetime(dictionary["found_date"]), release_date=load_datetime(dictionary["release_date"]), + source=dictionary["source"], ) def to_dict(self): @@ -106,23 +114,32 @@ def to_dict(self): "filename": self.filename, "release_date": dump_datetime(self.release_date), "found_date": dump_datetime(self.found_date), + "source": self.source, } - def use(self, now): - compare_from = self.release_date or self.found_date - return now - compare_from >= timedelta(days=28) + def use(self, now, ignore_grace_period_minor=False, ignore_grace_period_ci=False): + if self.source == "manual": + return True + elif self.source == "periodic": + if self.found_date < now - GRACE_PERIOD_CI or ignore_grace_period_ci: + if not ignore_grace_period_minor: + compare_from = self.release_date or self.found_date + return now - compare_from >= GRACE_PERIOD_MINOR + return True + return False def __repr__(self): - return "{}(filename={}), found_date={}, release_date={})".format( + return "{}(filename={}), found_date={}, release_date={}, source={})".format( self.__class__.__name__, self.filename, self.found_date, self.release_date, + self.source, ) def __eq__(self, other): return type(self) == type(other) and all( - getattr(self, k) == getattr(other, k) for k in ["filename", "release_date", "found_date"] + getattr(self, k) == getattr(other, k) for k in ["filename", "release_date", "found_date", "source"] ) def __ne__(self, other): @@ -170,12 +187,12 @@ def needs_update(self): if self.completed is None: # never completed return self._check_start(now) else: - if now - self.completed <= timedelta(days=14): + if now - self.completed <= UPDATE_PERIOD: return False return self._check_start(now) def _check_start(self, now): - return self.started is None or now - self.started > timedelta(hours=1) + return self.started is None or now - self.started > UPDATE_ABORTED_DELAY def trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, env, periodic): @@ -231,12 +248,24 @@ def _run_do_update(app_data, distribution, embed_filename, for_py_version, perio embed_update_log = app_data.embed_update_log(distribution, for_py_version) u_log = UpdateLog.from_dict(embed_update_log.read()) now = datetime.now() + if periodic: + source = "periodic" + # mark everything not updated manually as source "periodic" + for version in u_log.versions: + if version.source != "manual": + version.source = source + else: + source = "manual" + # mark everything as source "manual" + for version in u_log.versions: + version.source = source + if wheel_filename is not None: dest = wheelhouse / wheel_filename.name if not dest.exists(): copy2(str(wheel_filename), str(wheelhouse)) last, last_version, versions = None, None, [] - while last is None or not last.use(now): + while last is None or not last.use(now, ignore_grace_period_ci=True): download_time = datetime.now() dest = acquire.download_wheel( distribution=distribution, @@ -250,7 +279,7 @@ def _run_do_update(app_data, distribution, embed_filename, for_py_version, perio if dest is None or (u_log.versions and u_log.versions[0].filename == dest.name): break release_date = release_date_for_wheel_path(dest.path) - last = NewVersion(filename=dest.path.name, release_date=release_date, found_date=download_time) + last = NewVersion(filename=dest.path.name, release_date=release_date, found_date=download_time, source=source) logging.info("detected %s in %s", last, datetime.now() - download_time) versions.append(last) last_wheel = Wheel(Path(last.filename)) diff --git a/tests/unit/seed/wheels/test_bundle.py b/tests/unit/seed/wheels/test_bundle.py index 16ea2fdac..767a2b49f 100644 --- a/tests/unit/seed/wheels/test_bundle.py +++ b/tests/unit/seed/wheels/test_bundle.py @@ -1,12 +1,14 @@ from __future__ import absolute_import, unicode_literals import os +from datetime import datetime import pytest from virtualenv.app_data import AppDataDiskFolder from virtualenv.seed.wheels.bundle import from_bundle from virtualenv.seed.wheels.embed import get_embed_wheel +from virtualenv.seed.wheels.periodic_update import dump_datetime from virtualenv.seed.wheels.util import Version, Wheel from virtualenv.util.path import Path @@ -23,17 +25,19 @@ def next_pip_wheel(for_py_version): @pytest.fixture(scope="module") def app_data(tmp_path_factory, for_py_version, next_pip_wheel): temp_folder = tmp_path_factory.mktemp("module-app-data") + now = dump_datetime(datetime.now()) app_data_ = AppDataDiskFolder(str(temp_folder)) app_data_.embed_update_log("pip", for_py_version).write( { - "completed": "2000-01-01T00:00:00.000000Z", + "completed": now, "periodic": True, - "started": "2000-01-01T00:00:00.000000Z", + "started": now, "versions": [ { "filename": next_pip_wheel.name, "found_date": "2000-01-01T00:00:00.000000Z", "release_date": "2000-01-01T00:00:00.000000Z", + "source": "periodic", } ], } diff --git a/tests/unit/seed/wheels/test_periodic_update.py b/tests/unit/seed/wheels/test_periodic_update.py index 9b32770f5..e35dfedf3 100644 --- a/tests/unit/seed/wheels/test_periodic_update.py +++ b/tests/unit/seed/wheels/test_periodic_update.py @@ -42,7 +42,7 @@ def clear_pypi_info_cache(): def test_manual_upgrade(session_app_data, caplog, mocker, for_py_version): wheel = get_embed_wheel("pip", for_py_version) - new_version = NewVersion(wheel.path, datetime.now(), datetime.now() - timedelta(days=20)) + new_version = NewVersion(wheel.path, datetime.now(), datetime.now() - timedelta(days=20), "manual") def _do_update(distribution, for_py_version, embed_filename, app_data, search_dirs, periodic): # noqa if distribution == "pip": @@ -73,7 +73,7 @@ def test_pick_periodic_update(tmp_path, session_app_data, mocker, for_py_version u_log = UpdateLog( started=datetime.now() - timedelta(days=30), completed=completed, - versions=[NewVersion(filename=current.path, found_date=completed, release_date=completed)], + versions=[NewVersion(filename=current.path, found_date=completed, release_date=completed, source="periodic")], periodic=True, ) read_dict = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) @@ -93,9 +93,9 @@ def test_periodic_update_stops_at_current(mocker, session_app_data, for_py_versi started=completed, completed=completed, versions=[ - NewVersion(wheel_path(current, (1,)), completed, now - timedelta(days=1)), - NewVersion(filename=current.path, found_date=completed, release_date=now - timedelta(days=2)), - NewVersion(wheel_path(current, (-1,)), completed, now - timedelta(days=30)), + NewVersion(wheel_path(current, (1,)), completed, now - timedelta(days=1), "periodic"), + NewVersion(current.path, completed, now - timedelta(days=2), "periodic"), + NewVersion(wheel_path(current, (-1,)), completed, now - timedelta(days=30), "periodic"), ], periodic=True, ) @@ -107,21 +107,67 @@ def test_periodic_update_stops_at_current(mocker, session_app_data, for_py_versi def test_periodic_update_latest_per_patch(mocker, session_app_data, for_py_version): current = get_embed_wheel("setuptools", for_py_version) - now, completed = datetime.now(), datetime.now() - timedelta(days=29) + expected_path = wheel_path(current, (0, 1, 2)) + now = datetime.now() + completed = now - timedelta(hours=2) u_log = UpdateLog( started=completed, completed=completed, + periodic=True, versions=[ - NewVersion(wheel_path(current, (0, 1, 2)), completed, now - timedelta(days=1)), - NewVersion(wheel_path(current, (0, 1, 1)), completed, now - timedelta(days=30)), - NewVersion(filename=str(current.path), found_date=completed, release_date=now - timedelta(days=2)), + NewVersion(expected_path, completed, now - timedelta(days=1), "periodic"), + NewVersion(wheel_path(current, (0, 1, 1)), completed, now - timedelta(days=30), "periodic"), + NewVersion(str(current.path), completed, now - timedelta(days=31), "periodic"), ], + ) + mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) + + result = periodic_update("setuptools", None, for_py_version, current, [], session_app_data, False, os.environ) + assert str(result.path) == expected_path + + +def test_periodic_update_latest_per_patch_prev_is_manual(mocker, session_app_data, for_py_version): + current = get_embed_wheel("setuptools", for_py_version) + expected_path = wheel_path(current, (0, 1, 2)) + now = datetime.now() + completed = now - timedelta(hours=2) + u_log = UpdateLog( + started=completed, + completed=completed, periodic=True, + versions=[ + NewVersion(expected_path, completed, completed, "periodic"), + NewVersion(wheel_path(current, (0, 1, 1)), completed, now - timedelta(days=10), "manual"), + NewVersion(wheel_path(current, (0, 1, 0)), completed, now - timedelta(days=11), "periodic"), + NewVersion(str(current.path), completed, now - timedelta(days=12), "manual"), + ], ) mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) result = periodic_update("setuptools", None, for_py_version, current, [], session_app_data, False, os.environ) - assert result.path == current.path + assert str(result.path) == expected_path + + +def test_manual_update_honored(mocker, session_app_data, for_py_version): + current = get_embed_wheel("setuptools", for_py_version) + expected_path = wheel_path(current, (0, 1, 1)) + now = datetime.now() + completed = now + u_log = UpdateLog( + started=completed, + completed=completed, + periodic=True, + versions=[ + NewVersion(wheel_path(current, (0, 1, 2)), completed, completed, "periodic"), + NewVersion(expected_path, completed, now - timedelta(days=10), "manual"), + NewVersion(wheel_path(current, (0, 1, 0)), completed, now - timedelta(days=11), "periodic"), + NewVersion(str(current.path), completed, now - timedelta(days=12), "manual"), + ], + ) + mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) + + result = periodic_update("setuptools", None, for_py_version, current, [], session_app_data, False, os.environ) + assert str(result.path) == expected_path def wheel_path(wheel, of): @@ -338,7 +384,7 @@ def _release(of, context): assert copy.call_count == 1 expected = [ - NewVersion(Path(wheel).name, _UP_NOW, None if release is None else release.replace(microsecond=0)) + NewVersion(Path(wheel).name, _UP_NOW, None if release is None else release.replace(microsecond=0), "periodic") for wheel, release in pip_version_remote ] assert versions == expected @@ -371,7 +417,7 @@ def _download_wheel(distribution, version_spec, for_py_version, search_dirs, app u_log = UpdateLog( started=_UP_NOW - timedelta(days=31), completed=released, - versions=[NewVersion(filename=wheel.path.name, found_date=released, release_date=released)], + versions=[NewVersion(filename=wheel.path.name, found_date=released, release_date=released, source="periodic")], periodic=True, ) read_dict = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) @@ -395,21 +441,20 @@ def _download_wheel(distribution, version_spec, for_py_version, search_dirs, app "filename": wheel.path.name, "release_date": dump_datetime(released), "found_date": dump_datetime(released), + "source": "manual", # changed from "periodic" to "manual" }, ], } def test_new_version_eq(): - value = NewVersion("a", datetime.now(), datetime.now()) + value = NewVersion("a", datetime.now(), datetime.now(), "periodic") assert value == value def test_new_version_ne(): - assert NewVersion("a", datetime.now(), datetime.now()) != NewVersion( - "a", - datetime.now(), - datetime.now() + timedelta(hours=1), + assert NewVersion("a", datetime.now(), datetime.now(), "periodic") != NewVersion( + "a", datetime.now(), datetime.now() + timedelta(hours=1), "manual" ) @@ -472,3 +517,93 @@ def download(): assert read_dict.call_count == 1 assert write.call_count == 1 + + +def test_download_periodic_stop_at_first_usable(tmp_path, mocker, freezer): + freezer.move_to(_UP_NOW) + wheel = get_embed_wheel("pip", "3.9") + app_data_outer = AppDataDiskFolder(str(tmp_path / "app")) + pip_version_remote = [wheel_path(wheel, (0, 1, 1)), wheel_path(wheel, (0, 1, 0))] + rel_date_remote = [_UP_NOW - timedelta(days=1), _UP_NOW - timedelta(days=30)] + at = {"download": 0, "release_date": 0} + + def download(): + while True: + path = pip_version_remote[at["download"]] + at["download"] += 1 + yield Wheel(Path(path)) + + download_gen = download() + download_wheel = mocker.patch( + "virtualenv.seed.wheels.acquire.download_wheel", side_effect=lambda *a, **k: next(download_gen) + ) + + def rel_date(): + while True: + value = rel_date_remote[at["release_date"]] + at["release_date"] += 1 + yield value + + rel_date_gen = rel_date() + release_date = mocker.patch( + "virtualenv.seed.wheels.periodic_update.release_date_for_wheel_path", + side_effect=lambda *a, **k: next(rel_date_gen), + ) + + last_update = _UP_NOW - timedelta(days=14) + u_log = UpdateLog(started=last_update, completed=last_update, versions=[], periodic=True) + read_dict = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) + write = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.write") + + do_update("pip", "3.9", str(wheel.path), str(app_data_outer), [], True) + + assert download_wheel.call_count == 2 + assert release_date.call_count == 2 + + assert read_dict.call_count == 1 + assert write.call_count == 1 + + +def test_download_manual_stop_at_first_usable(tmp_path, mocker, freezer): + freezer.move_to(_UP_NOW) + wheel = get_embed_wheel("pip", "3.9") + app_data_outer = AppDataDiskFolder(str(tmp_path / "app")) + pip_version_remote = [wheel_path(wheel, (0, 1, 1))] + rel_date_remote = [_UP_NOW + timedelta(hours=1)] + at = {"download": 0, "release_date": 0} + + def download(): + while True: + path = pip_version_remote[at["download"]] + at["download"] += 1 + yield Wheel(Path(path)) + + download_gen = download() + download_wheel = mocker.patch( + "virtualenv.seed.wheels.acquire.download_wheel", side_effect=lambda *a, **k: next(download_gen) + ) + + def rel_date(): + while True: + value = rel_date_remote[at["release_date"]] + at["release_date"] += 1 + yield value + + rel_date_gen = rel_date() + release_date = mocker.patch( + "virtualenv.seed.wheels.periodic_update.release_date_for_wheel_path", + side_effect=lambda *a, **k: next(rel_date_gen), + ) + + last_update = _UP_NOW - timedelta(days=14) + u_log = UpdateLog(started=last_update, completed=last_update, versions=[], periodic=True) + read_dict = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.read", return_value=u_log.to_dict()) + write = mocker.patch("virtualenv.app_data.via_disk_folder.JSONStoreDisk.write") + + do_update("pip", "3.9", str(wheel.path), str(app_data_outer), [], False) + + assert download_wheel.call_count == 1 + assert release_date.call_count == 1 + + assert read_dict.call_count == 1 + assert write.call_count == 1