Skip to content

Commit

Permalink
Add --target support to sync and install (astral-sh#3257)
Browse files Browse the repository at this point in the history
## Summary

The approach taken here is to model `--target` as an install scheme in
which all the directories are just subdirectories of the `--target`.
From there, everything else... just works? Like, upgrade, uninstalls,
editables, etc. all "just work".

Closes astral-sh#1517.
  • Loading branch information
charliermarsh committed Apr 25, 2024
1 parent 71ffb2e commit ed8f6e4
Show file tree
Hide file tree
Showing 14 changed files with 284 additions and 36 deletions.
1 change: 1 addition & 0 deletions crates/uv-interpreter/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ uv-warnings = { workspace = true }

configparser = { workspace = true }
fs-err = { workspace = true, features = ["tokio"] }
itertools = { workspace = true }
once_cell = { workspace = true }
regex = { workspace = true }
rmp-serde = { workspace = true }
Expand Down
68 changes: 51 additions & 17 deletions crates/uv-interpreter/src/interpreter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ use uv_cache::{Cache, CacheBucket, CachedByTimestamp, Freshness, Timestamp};
use uv_fs::{write_atomic_sync, PythonExt, Simplified};
use uv_toolchain::PythonVersion;

use crate::Error;
use crate::Virtualenv;
use crate::{Error, Target};

/// A Python executable and its associated platform markers.
#[derive(Debug, Clone)]
Expand All @@ -35,6 +35,7 @@ pub struct Interpreter {
sys_executable: PathBuf,
stdlib: PathBuf,
tags: OnceCell<Tags>,
target: Option<Target>,
gil_disabled: bool,
}

Expand Down Expand Up @@ -62,6 +63,7 @@ impl Interpreter {
sys_executable: info.sys_executable,
stdlib: info.stdlib,
tags: OnceCell::new(),
target: None,
})
}

Expand Down Expand Up @@ -91,6 +93,7 @@ impl Interpreter {
sys_executable: PathBuf::from("/dev/null"),
stdlib: PathBuf::from("/dev/null"),
tags: OnceCell::new(),
target: None,
gil_disabled: false,
}
}
Expand All @@ -106,6 +109,17 @@ impl Interpreter {
}
}

/// Return a new [`Interpreter`] to install into the given `--target` directory.
///
/// Initializes the `--target` directory with the expected layout.
#[must_use]
pub fn with_target(self, target: Target) -> Self {
Self {
target: Some(target),
..self
}
}

/// Returns the path to the Python virtual environment.
#[inline]
pub fn platform(&self) -> &Platform {
Expand Down Expand Up @@ -135,9 +149,15 @@ impl Interpreter {
///
/// See: <https://github.com/pypa/pip/blob/0ad4c94be74cc24874c6feb5bb3c2152c398a18e/src/pip/_internal/utils/virtualenv.py#L14>
pub fn is_virtualenv(&self) -> bool {
// Maybe this should return `false` if it's a target?
self.prefix != self.base_prefix
}

/// Returns `true` if the environment is a `--target` environment.
pub fn is_target(&self) -> bool {
self.target.is_some()
}

/// Returns `Some` if the environment is externally managed, optionally including an error
/// message from the `EXTERNALLY-MANAGED` file.
///
Expand All @@ -148,6 +168,11 @@ impl Interpreter {
return None;
}

// If we're installing into a target directory, it's never externally managed.
if self.is_target() {
return None;
}

let Ok(contents) = fs::read_to_string(self.stdlib.join("EXTERNALLY-MANAGED")) else {
return None;
};
Expand Down Expand Up @@ -303,28 +328,37 @@ impl Interpreter {
self.gil_disabled
}

/// Return the `--target` directory for this interpreter, if any.
pub fn target(&self) -> Option<&Target> {
self.target.as_ref()
}

/// Return the [`Layout`] environment used to install wheels into this interpreter.
pub fn layout(&self) -> Layout {
Layout {
python_version: self.python_tuple(),
sys_executable: self.sys_executable().to_path_buf(),
os_name: self.markers.os_name.clone(),
scheme: Scheme {
purelib: self.purelib().to_path_buf(),
platlib: self.platlib().to_path_buf(),
scripts: self.scripts().to_path_buf(),
data: self.data().to_path_buf(),
include: if self.is_virtualenv() {
// If the interpreter is a venv, then the `include` directory has a different structure.
// See: https://github.com/pypa/pip/blob/0ad4c94be74cc24874c6feb5bb3c2152c398a18e/src/pip/_internal/locations/_sysconfig.py#L172
self.prefix.join("include").join("site").join(format!(
"python{}.{}",
self.python_major(),
self.python_minor()
))
} else {
self.include().to_path_buf()
},
scheme: if let Some(target) = self.target.as_ref() {
target.scheme()
} else {
Scheme {
purelib: self.purelib().to_path_buf(),
platlib: self.platlib().to_path_buf(),
scripts: self.scripts().to_path_buf(),
data: self.data().to_path_buf(),
include: if self.is_virtualenv() {
// If the interpreter is a venv, then the `include` directory has a different structure.
// See: https://github.com/pypa/pip/blob/0ad4c94be74cc24874c6feb5bb3c2152c398a18e/src/pip/_internal/locations/_sysconfig.py#L172
self.prefix.join("include").join("site").join(format!(
"python{}.{}",
self.python_major(),
self.python_minor()
))
} else {
self.include().to_path_buf()
},
}
},
}
}
Expand Down
2 changes: 2 additions & 0 deletions crates/uv-interpreter/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,14 @@ pub use crate::find_python::{find_best_python, find_default_python, find_request
pub use crate::interpreter::Interpreter;
use crate::interpreter::InterpreterInfoError;
pub use crate::python_environment::PythonEnvironment;
pub use crate::target::Target;
pub use crate::virtualenv::Virtualenv;

mod cfg;
mod find_python;
mod interpreter;
mod python_environment;
mod target;
mod virtualenv;

#[derive(Debug, Error)]
Expand Down
44 changes: 32 additions & 12 deletions crates/uv-interpreter/src/python_environment.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use itertools::Either;
use std::env;
use std::path::{Path, PathBuf};

Expand All @@ -8,7 +9,7 @@ use uv_cache::Cache;
use uv_fs::{LockedFile, Simplified};

use crate::cfg::PyVenvConfiguration;
use crate::{find_default_python, find_requested_python, Error, Interpreter};
use crate::{find_default_python, find_requested_python, Error, Interpreter, Target};

/// A Python environment, consisting of a Python [`Interpreter`] and its associated paths.
#[derive(Debug, Clone)]
Expand Down Expand Up @@ -68,7 +69,16 @@ impl PythonEnvironment {
}
}

/// Returns the location of the Python interpreter.
/// Create a [`PythonEnvironment`] from an existing [`Interpreter`] and `--target` directory.
#[must_use]
pub fn with_target(self, target: Target) -> Self {
Self {
interpreter: self.interpreter.with_target(target),
..self
}
}

/// Returns the root (i.e., `prefix`) of the Python interpreter.
pub fn root(&self) -> &Path {
&self.root
}
Expand Down Expand Up @@ -97,15 +107,19 @@ impl PythonEnvironment {
/// Some distributions also create symbolic links from `purelib` to `platlib`; in such cases, we
/// still deduplicate the entries, returning a single path.
pub fn site_packages(&self) -> impl Iterator<Item = &Path> {
let purelib = self.interpreter.purelib();
let platlib = self.interpreter.platlib();
std::iter::once(purelib).chain(
if purelib == platlib || is_same_file(purelib, platlib).unwrap_or(false) {
None
} else {
Some(platlib)
},
)
if let Some(target) = self.interpreter.target() {
Either::Left(std::iter::once(target.root()))
} else {
let purelib = self.interpreter.purelib();
let platlib = self.interpreter.platlib();
Either::Right(std::iter::once(purelib).chain(
if purelib == platlib || is_same_file(purelib, platlib).unwrap_or(false) {
None
} else {
Some(platlib)
},
))
}
}

/// Returns the path to the `bin` directory inside a virtual environment.
Expand All @@ -115,7 +129,13 @@ impl PythonEnvironment {

/// Grab a file lock for the virtual environment to prevent concurrent writes across processes.
pub fn lock(&self) -> Result<LockedFile, std::io::Error> {
if self.interpreter.is_virtualenv() {
if let Some(target) = self.interpreter.target() {
// If we're installing into a `--target`, use a target-specific lock file.
LockedFile::acquire(
target.root().join(".lock"),
target.root().simplified_display(),
)
} else if self.interpreter.is_virtualenv() {
// If the environment a virtualenv, use a virtualenv-specific lock file.
LockedFile::acquire(self.root.join(".lock"), self.root.simplified_display())
} else {
Expand Down
40 changes: 40 additions & 0 deletions crates/uv-interpreter/src/target.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
use std::path::{Path, PathBuf};

use pypi_types::Scheme;

/// A `--target` directory into which packages can be installed, separate from a virtual environment
/// or system Python interpreter.
#[derive(Debug, Clone)]
pub struct Target(PathBuf);

impl Target {
/// Return the [`Scheme`] for the `--target` directory.
pub fn scheme(&self) -> Scheme {
Scheme {
purelib: self.0.clone(),
platlib: self.0.clone(),
scripts: self.0.join("bin"),
data: self.0.clone(),
include: self.0.join("include"),
}
}

/// Initialize the `--target` directory.
pub fn init(&self) -> std::io::Result<()> {
fs_err::create_dir_all(&self.0)?;
fs_err::create_dir_all(self.0.join("bin"))?;
fs_err::create_dir_all(self.0.join("include"))?;
Ok(())
}

/// Return the path to the `--target` directory.
pub fn root(&self) -> &Path {
&self.0
}
}

impl From<PathBuf> for Target {
fn from(path: PathBuf) -> Self {
Self(path)
}
}
1 change: 1 addition & 0 deletions crates/uv-workspace/src/settings.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ pub struct PipOptions {
pub python: Option<String>,
pub system: Option<bool>,
pub break_system_packages: Option<bool>,
pub target: Option<PathBuf>,
pub offline: Option<bool>,
pub index_url: Option<IndexUrl>,
pub extra_index_url: Option<Vec<IndexUrl>>,
Expand Down
15 changes: 15 additions & 0 deletions crates/uv/src/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -768,6 +768,11 @@ pub(crate) struct PipSyncArgs {
#[arg(long, overrides_with("break_system_packages"))]
pub(crate) no_break_system_packages: bool,

/// Install packages into the specified directory, rather than into the virtual environment
/// or system Python interpreter.
#[arg(long)]
pub(crate) target: Option<PathBuf>,

/// Use legacy `setuptools` behavior when building source distributions without a
/// `pyproject.toml`.
#[arg(long, overrides_with("no_legacy_setup_py"))]
Expand Down Expand Up @@ -1132,6 +1137,11 @@ pub(crate) struct PipInstallArgs {
#[arg(long, overrides_with("break_system_packages"))]
pub(crate) no_break_system_packages: bool,

/// Install packages into the specified directory, rather than into the virtual environment
/// or system Python interpreter.
#[arg(long)]
pub(crate) target: Option<PathBuf>,

/// Use legacy `setuptools` behavior when building source distributions without a
/// `pyproject.toml`.
#[arg(long, overrides_with("no_legacy_setup_py"))]
Expand Down Expand Up @@ -1335,6 +1345,11 @@ pub(crate) struct PipUninstallArgs {
#[arg(long, overrides_with("break_system_packages"))]
pub(crate) no_break_system_packages: bool,

/// Uninstall packages from the specified directory, rather than from the virtual environment
/// or system Python interpreter.
#[arg(long)]
pub(crate) target: Option<PathBuf>,

/// Run offline, i.e., without accessing the network.
#[arg(long, overrides_with("no_offline"))]
pub(crate) offline: bool,
Expand Down
13 changes: 10 additions & 3 deletions crates/uv/src/commands/pip_install.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
use std::borrow::Cow;
use std::fmt::Write;

use std::path::Path;

use anstream::eprint;
use anyhow::{anyhow, Context, Result};

use itertools::Itertools;
use owo_colors::OwoColorize;
use tempfile::tempdir_in;
Expand Down Expand Up @@ -33,7 +31,7 @@ use uv_configuration::{KeyringProviderType, TargetTriple};
use uv_dispatch::BuildDispatch;
use uv_fs::Simplified;
use uv_installer::{BuiltEditable, Downloader, Plan, Planner, ResolvedEditable, SitePackages};
use uv_interpreter::{Interpreter, PythonEnvironment};
use uv_interpreter::{Interpreter, PythonEnvironment, Target};
use uv_normalize::PackageName;
use uv_requirements::{
ExtrasSpecification, LookaheadResolver, NamedRequirementsResolver, RequirementsSource,
Expand Down Expand Up @@ -84,6 +82,7 @@ pub(crate) async fn pip_install(
python: Option<String>,
system: bool,
break_system_packages: bool,
target: Option<Target>,
native_tls: bool,
cache: Cache,
dry_run: bool,
Expand Down Expand Up @@ -134,6 +133,14 @@ pub(crate) async fn pip_install(
venv.python_executable().user_display().cyan()
);

// Apply any `--target` directory.
let venv = if let Some(target) = target {
target.init()?;
venv.with_target(target)
} else {
venv
};

// If the environment is externally managed, abort.
if let Some(externally_managed) = venv.interpreter().is_externally_managed() {
if break_system_packages {
Expand Down
Loading

0 comments on commit ed8f6e4

Please sign in to comment.