Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Preserve verbatim URLs for --find-links #4838

Merged
merged 1 commit into from
Jul 5, 2024
Merged

Preserve verbatim URLs for --find-links #4838

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Preserve verbatim URLs for --find-links
  • Loading branch information
@charliermarsh
charliermarsh committed Jul 5, 2024
commit 53716c4799b02c078b9e877c287ba3706a0d987f
161 changes: 73 additions & 88 deletions crates/distribution-types/src/index_url.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
use std::borrow::Cow;
use std::fmt::{Display, Formatter};
use std::ops::Deref;
use std::path::{Path, PathBuf};
use std::path::Path;
use std::str::FromStr;

use itertools::Either;
use once_cell::sync::Lazy;
use thiserror::Error;
use url::{ParseError, Url};

use pep508_rs::{expand_env_vars, split_scheme, strip_host, Scheme, VerbatimUrl, VerbatimUrlError};
use uv_fs::normalize_url_path;
use pep508_rs::{VerbatimUrl, VerbatimUrlError};

use crate::Verbatim;

Expand Down Expand Up @@ -91,6 +90,15 @@ impl Verbatim for IndexUrl {
}
}

impl From<FlatIndexLocation> for IndexUrl {
fn from(location: FlatIndexLocation) -> Self {
match location {
FlatIndexLocation::Path(url) => Self::Path(url),
FlatIndexLocation::Url(url) => Self::Url(url),
}
}
}

/// An error that can occur when parsing an [`IndexUrl`].
#[derive(Error, Debug)]
pub enum IndexUrlError {
Expand Down Expand Up @@ -173,8 +181,8 @@ impl Deref for IndexUrl {
/// Also known as `--find-links`.
#[derive(Debug, Clone, Hash, Eq, PartialEq)]
pub enum FlatIndexLocation {
Path(PathBuf),
Url(Url),
Path(VerbatimUrl),
Url(VerbatimUrl),
}

#[cfg(feature = "schemars")]
Expand All @@ -197,6 +205,60 @@ impl schemars::JsonSchema for FlatIndexLocation {
}
}

impl FlatIndexLocation {
/// Return the raw URL for the `--find-links` index.
pub fn url(&self) -> &Url {
match self {
Self::Url(url) => url.raw(),
Self::Path(url) => url.raw(),
}
}

/// Return the redacted URL for the `--find-links` index, omitting any sensitive credentials.
pub fn redacted(&self) -> Cow<'_, Url> {
let url = self.url();
if url.username().is_empty() && url.password().is_none() {
Cow::Borrowed(url)
} else {
let mut url = url.clone();
let _ = url.set_username("");
let _ = url.set_password(None);
Cow::Owned(url)
}
}
}

impl Display for FlatIndexLocation {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
Self::Url(url) => Display::fmt(url, f),
Self::Path(url) => Display::fmt(url, f),
}
}
}

impl Verbatim for FlatIndexLocation {
fn verbatim(&self) -> Cow<'_, str> {
match self {
Self::Url(url) => url.verbatim(),
Self::Path(url) => url.verbatim(),
}
}
}

impl FromStr for FlatIndexLocation {
type Err = IndexUrlError;

fn from_str(s: &str) -> Result<Self, Self::Err> {
let url = if let Ok(path) = Path::new(s).canonicalize() {
VerbatimUrl::from_path(path)?
} else {
VerbatimUrl::parse_url(s)?
};
Ok(Self::from(url.with_given(s)))
}
}

impl serde::ser::Serialize for FlatIndexLocation {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
Expand All @@ -216,60 +278,12 @@ impl<'de> serde::de::Deserialize<'de> for FlatIndexLocation {
}
}

impl FromStr for FlatIndexLocation {
type Err = url::ParseError;

/// Parse a raw string for a `--find-links` entry, which could be a URL or a local path.
///
/// For example:
/// - `file:///home/ferris/project/scripts/...`
/// - `file:../ferris/`
/// - `../ferris/`
/// - `https://download.pytorch.org/whl/torch_stable.html`
fn from_str(s: &str) -> Result<Self, Self::Err> {
// Expand environment variables.
let expanded = expand_env_vars(s);

// Parse the expanded path.
if let Some((scheme, path)) = split_scheme(&expanded) {
match Scheme::parse(scheme) {
// Ex) `file:///home/ferris/project/scripts/...`, `file://localhost/home/ferris/project/scripts/...`, or `file:../ferris/`
Some(Scheme::File) => {
// Strip the leading slashes, along with the `localhost` host, if present.
let path = strip_host(path);

// Transform, e.g., `/C:/Users/ferris/wheel-0.42.0.tar.gz` to `C:\Users\ferris\wheel-0.42.0.tar.gz`.
let path = normalize_url_path(path);

let path = PathBuf::from(path.as_ref());
Ok(Self::Path(path))
}

// Ex) `https://download.pytorch.org/whl/torch_stable.html`
Some(_) => {
let url = Url::parse(expanded.as_ref())?;
Ok(Self::Url(url))
}

// Ex) `C:\Users\ferris\wheel-0.42.0.tar.gz`
None => {
let path = PathBuf::from(expanded.as_ref());
Ok(Self::Path(path))
}
}
impl From<VerbatimUrl> for FlatIndexLocation {
fn from(url: VerbatimUrl) -> Self {
if url.scheme() == "file" {
Self::Path(url)
} else {
// Ex) `../ferris/`
let path = PathBuf::from(expanded.as_ref());
Ok(Self::Path(path))
}
}
}

impl Display for FlatIndexLocation {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
Self::Path(path) => Display::fmt(&path.display(), f),
Self::Url(url) => Display::fmt(url, f),
Self::Url(url)
}
}
}
Expand Down Expand Up @@ -387,7 +401,7 @@ impl<'a> IndexLocations {
.map(IndexUrl::url)
.chain(self.flat_index.iter().filter_map(|index| match index {
FlatIndexLocation::Path(_) => None,
FlatIndexLocation::Url(url) => Some(url),
FlatIndexLocation::Url(url) => Some(url.raw()),
}))
}
}
Expand Down Expand Up @@ -459,32 +473,3 @@ impl From<IndexLocations> for IndexUrls {
}
}
}

#[cfg(test)]
#[cfg(unix)]
mod test {
use super::*;

#[test]
fn parse_find_links() {
assert_eq!(
FlatIndexLocation::from_str("file:///home/ferris/project/scripts/...").unwrap(),
FlatIndexLocation::Path(PathBuf::from("/home/ferris/project/scripts/..."))
);
assert_eq!(
FlatIndexLocation::from_str("file:../ferris/").unwrap(),
FlatIndexLocation::Path(PathBuf::from("../ferris/"))
);
assert_eq!(
FlatIndexLocation::from_str("../ferris/").unwrap(),
FlatIndexLocation::Path(PathBuf::from("../ferris/"))
);
assert_eq!(
FlatIndexLocation::from_str("https://download.pytorch.org/whl/torch_stable.html")
.unwrap(),
FlatIndexLocation::Url(
Url::parse("https://download.pytorch.org/whl/torch_stable.html").unwrap()
)
);
}
}
105 changes: 23 additions & 82 deletions crates/requirements-txt/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,15 +45,13 @@ use unscanny::{Pattern, Scanner};
use url::Url;

use distribution_types::{UnresolvedRequirement, UnresolvedRequirementSpecification};
use pep508_rs::{
expand_env_vars, split_scheme, strip_host, Pep508Error, RequirementOrigin, Scheme, VerbatimUrl,
};
use pep508_rs::{expand_env_vars, Pep508Error, RequirementOrigin, VerbatimUrl};
use pypi_types::{Requirement, VerbatimParsedUrl};
#[cfg(feature = "http")]
use uv_client::BaseClient;
use uv_client::BaseClientBuilder;
use uv_configuration::{NoBinary, NoBuild, PackageNameSpecifier};
use uv_fs::{normalize_url_path, Simplified};
use uv_fs::Simplified;
use uv_warnings::warn_user;

use crate::requirement::EditableError;
Expand Down Expand Up @@ -84,7 +82,7 @@ enum RequirementsTxtStatement {
/// `--extra-index-url`
ExtraIndexUrl(VerbatimUrl),
/// `--find-links`
FindLinks(FindLink),
FindLinks(VerbatimUrl),
/// `--no-index`
NoIndex,
/// `--no-binary`
Expand All @@ -93,73 +91,6 @@ enum RequirementsTxtStatement {
OnlyBinary(NoBuild),
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FindLink {
Path(PathBuf),
Url(Url),
}

impl FindLink {
/// Parse a raw string for a `--find-links` entry, which could be a URL or a local path.
///
/// For example:
/// - `file:///home/ferris/project/scripts/...`
/// - `file:../ferris/`
/// - `../ferris/`
/// - `https://download.pytorch.org/whl/torch_stable.html`
pub fn parse(given: &str, working_dir: impl AsRef<Path>) -> Result<Self, url::ParseError> {
// Expand environment variables.
let expanded = expand_env_vars(given);

if let Some((scheme, path)) = split_scheme(&expanded) {
match Scheme::parse(scheme) {
// Ex) `file:///home/ferris/project/scripts/...`, `file://localhost/home/ferris/project/scripts/...`, or `file:../ferris/`
Some(Scheme::File) => {
// Strip the leading slashes, along with the `localhost` host, if present.
let path = strip_host(path);

// Transform, e.g., `/C:/Users/ferris/wheel-0.42.0.tar.gz` to `C:\Users\ferris\wheel-0.42.0.tar.gz`.
let path = normalize_url_path(path);

let path = PathBuf::from(path.as_ref());
let path = if path.is_absolute() {
path
} else {
working_dir.as_ref().join(path)
};
Ok(Self::Path(path))
}

// Ex) `https://download.pytorch.org/whl/torch_stable.html`
Some(_) => {
let url = Url::parse(&expanded)?;
Ok(Self::Url(url))
}

// Ex) `C:/Users/ferris/wheel-0.42.0.tar.gz`
_ => {
let path = PathBuf::from(expanded.as_ref());
let path = if path.is_absolute() {
path
} else {
working_dir.as_ref().join(path)
};
Ok(Self::Path(path))
}
}
} else {
// Ex) `../ferris/`
let path = PathBuf::from(expanded.as_ref());
let path = if path.is_absolute() {
path
} else {
working_dir.as_ref().join(path)
};
Ok(Self::Path(path))
}
}
}

/// A [Requirement] with additional metadata from the `requirements.txt`, currently only hashes but in
/// the future also editable and similar information.
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
Expand Down Expand Up @@ -212,7 +143,7 @@ pub struct RequirementsTxt {
/// The extra index URLs, specified with `--extra-index-url`.
pub extra_index_urls: Vec<VerbatimUrl>,
/// The find links locations, specified with `--find-links`.
pub find_links: Vec<FindLink>,
pub find_links: Vec<VerbatimUrl>,
/// Whether to ignore the index, specified with `--no-index`.
pub no_index: bool,
/// Whether to disallow wheels, specified with `--no-binary`.
Expand Down Expand Up @@ -445,8 +376,8 @@ impl RequirementsTxt {
RequirementsTxtStatement::ExtraIndexUrl(url) => {
data.extra_index_urls.push(url);
}
RequirementsTxtStatement::FindLinks(path_or_url) => {
data.find_links.push(path_or_url);
RequirementsTxtStatement::FindLinks(url) => {
data.find_links.push(url);
}
RequirementsTxtStatement::NoIndex => {
data.no_index = true;
Expand Down Expand Up @@ -592,16 +523,26 @@ fn parse_entry(
} else if s.eat_if("--no-index") {
RequirementsTxtStatement::NoIndex
} else if s.eat_if("--find-links") || s.eat_if("-f") {
let path_or_url = parse_value(content, s, |c: char| !['\n', '\r', '#'].contains(&c))?;
let path_or_url = FindLink::parse(path_or_url, working_dir).map_err(|err| {
RequirementsTxtParserError::Url {
let given = parse_value(content, s, |c: char| !['\n', '\r', '#'].contains(&c))?;
let expanded = expand_env_vars(given);
let url = if let Ok(path) = Path::new(expanded.as_ref()).canonicalize() {
VerbatimUrl::from_path(path).map_err(|err| RequirementsTxtParserError::VerbatimUrl {
source: err,
url: path_or_url.to_string(),
url: given.to_string(),
start,
end: s.cursor(),
}
})?;
RequirementsTxtStatement::FindLinks(path_or_url)
})?
} else {
VerbatimUrl::parse_url(expanded.as_ref()).map_err(|err| {
RequirementsTxtParserError::Url {
source: err,
url: given.to_string(),
start,
end: s.cursor(),
}
})?
};
RequirementsTxtStatement::FindLinks(url.with_given(given))
} else if s.eat_if("--no-binary") {
let given = parse_value(content, s, |c: char| !['\n', '\r', '#'].contains(&c))?;
let specifier = PackageNameSpecifier::from_str(given).map_err(|err| {
Expand Down
Loading
Loading