From 31fcba372fa9d769e49a6c58078db89fc4f13fd9 Mon Sep 17 00:00:00 2001 From: Weihang Lo Date: Thu, 18 Feb 2021 20:47:18 +0800 Subject: [PATCH 001/129] fix: add `.*` pattern in the same gitignore instance --- src/cargo/sources/path.rs | 49 ++++++++++++--------------------------- 1 file changed, 15 insertions(+), 34 deletions(-) diff --git a/src/cargo/sources/path.rs b/src/cargo/sources/path.rs index 64b0f77ed5a..a379cbfafaa 100644 --- a/src/cargo/sources/path.rs +++ b/src/cargo/sources/path.rs @@ -107,8 +107,17 @@ impl<'cfg> PathSource<'cfg> { fn _list_files(&self, pkg: &Package) -> CargoResult> { let root = pkg.root(); let no_include_option = pkg.manifest().include().is_empty(); + let git_repo = if no_include_option { + self.discover_git_repo(root)? + } else { + None + }; let mut exclude_builder = GitignoreBuilder::new(root); + if no_include_option && git_repo.is_none() { + // no include option and not git repo discovered (see rust-lang/cargo#7183). + exclude_builder.add_line(None, ".*")?; + } for rule in pkg.manifest().exclude() { exclude_builder.add_line(None, rule)?; } @@ -160,23 +169,16 @@ impl<'cfg> PathSource<'cfg> { // Attempt Git-prepopulate only if no `include` (see rust-lang/cargo#4135). if no_include_option { - if let Some(result) = self.discover_git_and_list_files(pkg, root, &mut filter)? { - return Ok(result); + if let Some(repo) = git_repo { + return self.list_files_git(pkg, &repo, &mut filter); } - // no include option and not git repo discovered (see rust-lang/cargo#7183). - return self.list_files_walk_except_dot_files_and_dirs(pkg, &mut filter); } self.list_files_walk(pkg, &mut filter) } - // Returns `Some(_)` if found sibling `Cargo.toml` and `.git` directory; - // otherwise, caller should fall back on full file list. - fn discover_git_and_list_files( - &self, - pkg: &Package, - root: &Path, - filter: &mut dyn FnMut(&Path, bool) -> CargoResult, - ) -> CargoResult>> { + /// Returns `Some(git2::Repository)` if found sibling `Cargo.toml` and `.git` + /// directory; otherwise, caller should fall back on full file list. + fn discover_git_repo(&self, root: &Path) -> CargoResult> { let repo = match git2::Repository::discover(root) { Ok(repo) => repo, Err(e) => { @@ -211,7 +213,7 @@ impl<'cfg> PathSource<'cfg> { }; let manifest_path = repo_relative_path.join("Cargo.toml"); if index.get_path(&manifest_path, 0).is_some() { - return Ok(Some(self.list_files_git(pkg, &repo, filter)?)); + return Ok(Some(repo)); } // Package Cargo.toml is not in git, don't use git to guide our selection. Ok(None) @@ -355,27 +357,6 @@ impl<'cfg> PathSource<'cfg> { } } - fn list_files_walk_except_dot_files_and_dirs( - &self, - pkg: &Package, - filter: &mut dyn FnMut(&Path, bool) -> CargoResult, - ) -> CargoResult> { - let root = pkg.root(); - let mut exclude_dot_files_dir_builder = GitignoreBuilder::new(root); - exclude_dot_files_dir_builder.add_line(None, ".*")?; - let ignore_dot_files_and_dirs = exclude_dot_files_dir_builder.build()?; - - let mut filter_ignore_dot_files_and_dirs = - |path: &Path, is_dir: bool| -> CargoResult { - let relative_path = path.strip_prefix(root)?; - match ignore_dot_files_and_dirs.matched_path_or_any_parents(relative_path, is_dir) { - Match::Ignore(_) => Ok(false), - _ => filter(path, is_dir), - } - }; - self.list_files_walk(pkg, &mut filter_ignore_dot_files_and_dirs) - } - fn list_files_walk( &self, pkg: &Package, From 153146ecc5264f3066caf314b37c8091c0eb7595 Mon Sep 17 00:00:00 2001 From: Weihang Lo Date: Thu, 18 Feb 2021 20:48:17 +0800 Subject: [PATCH 002/129] test(vendor): respect to Cargo.toml [package.exclude] --- tests/testsuite/vendor.rs | 44 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tests/testsuite/vendor.rs b/tests/testsuite/vendor.rs index 63872392e5f..09968aa949e 100644 --- a/tests/testsuite/vendor.rs +++ b/tests/testsuite/vendor.rs @@ -50,6 +50,50 @@ fn add_vendor_config(p: &Project) { ); } +#[cargo_test] +fn package_exclude() { + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + bar = "0.1.0" + "#, + ) + .file("src/lib.rs", "") + .build(); + + Package::new("bar", "0.1.0") + .file( + "Cargo.toml", + r#" + [package] + name = "bar" + version = "0.1.0" + exclude = [".*", "!.include", "!.dotdir/include"] + "#, + ) + .file("src/lib.rs", "") + .file(".exclude", "") + .file(".include", "") + .file(".dotdir/exclude", "") + .file(".dotdir/include", "") + .publish(); + + p.cargo("vendor --respect-source-config").run(); + let csum = dbg!(p.read_file("vendor/bar/.cargo-checksum.json")); + assert!(csum.contains(".include")); + assert!(!csum.contains(".exclude")); + assert!(!csum.contains(".dotdir/exclude")); + // Gitignore doesn't re-include a file in an excluded parent directory, + // even if negating it explicitly. + assert!(!csum.contains(".dotdir/include")); +} + #[cargo_test] fn two_versions() { let p = project() From 77d993cb86b928de664c24e66fa92dec0b8c6144 Mon Sep 17 00:00:00 2001 From: Weihang Lo Date: Fri, 23 Apr 2021 21:29:24 +0800 Subject: [PATCH 003/129] refactor: progress.tick support attaching message --- src/cargo/sources/git/utils.rs | 4 ++-- src/cargo/util/progress.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cargo/sources/git/utils.rs b/src/cargo/sources/git/utils.rs index 5b0d60a9887..ab7849ddc39 100644 --- a/src/cargo/sources/git/utils.rs +++ b/src/cargo/sources/git/utils.rs @@ -677,7 +677,7 @@ fn reset(repo: &git2::Repository, obj: &git2::Object<'_>, config: &Config) -> Ca let mut pb = Progress::new("Checkout", config); let mut opts = git2::build::CheckoutBuilder::new(); opts.progress(|_, cur, max| { - drop(pb.tick(cur, max)); + drop(pb.tick(cur, max, "")); }); debug!("doing reset"); repo.reset(obj, git2::ResetType::Hard, Some(&mut opts))?; @@ -699,7 +699,7 @@ pub fn with_fetch_options( rcb.transfer_progress(|stats| { progress - .tick(stats.indexed_objects(), stats.total_objects()) + .tick(stats.indexed_objects(), stats.total_objects(), "") .is_ok() }); diff --git a/src/cargo/util/progress.rs b/src/cargo/util/progress.rs index 7996d8a959d..45bb4d89eaa 100644 --- a/src/cargo/util/progress.rs +++ b/src/cargo/util/progress.rs @@ -96,7 +96,7 @@ impl<'cfg> Progress<'cfg> { Self::with_style(name, ProgressStyle::Percentage, cfg) } - pub fn tick(&mut self, cur: usize, max: usize) -> CargoResult<()> { + pub fn tick(&mut self, cur: usize, max: usize, msg: &str) -> CargoResult<()> { let s = match &mut self.state { Some(s) => s, None => return Ok(()), @@ -118,7 +118,7 @@ impl<'cfg> Progress<'cfg> { return Ok(()); } - s.tick(cur, max, "") + s.tick(cur, max, msg) } pub fn tick_now(&mut self, cur: usize, max: usize, msg: &str) -> CargoResult<()> { From d58aa8ab4034fe96253666167bb02df9def4bbe2 Mon Sep 17 00:00:00 2001 From: Weihang Lo Date: Fri, 23 Apr 2021 21:32:27 +0800 Subject: [PATCH 004/129] feat: show transfer rate and total bytes received --- src/cargo/sources/git/utils.rs | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/src/cargo/sources/git/utils.rs b/src/cargo/sources/git/utils.rs index ab7849ddc39..49b3b3946ed 100644 --- a/src/cargo/sources/git/utils.rs +++ b/src/cargo/sources/git/utils.rs @@ -15,6 +15,7 @@ use std::env; use std::fmt; use std::path::{Path, PathBuf}; use std::process::Command; +use std::time::{Duration, Instant}; use url::Url; fn serialize_str(t: &T, s: S) -> Result @@ -694,12 +695,40 @@ pub fn with_fetch_options( let mut progress = Progress::new("Fetch", config); network::with_retry(config, || { with_authentication(url, git_config, |f| { + let mut last_recv = 0.0; // in Byte + let mut last_rate = 0.0; // in Byte/s + let mut last_update = Instant::now(); let mut rcb = git2::RemoteCallbacks::new(); rcb.credentials(f); - rcb.transfer_progress(|stats| { + let indexed_deltas = stats.indexed_deltas(); + let msg = if indexed_deltas > 0 { + // Resolving deltas. + format!(" ({}/{})", indexed_deltas, stats.total_deltas()) + } else { + // Receiving objects. + let duration = last_update.elapsed(); + let (recv, rate) = if duration > Duration::from_secs(1) { + let recv = stats.received_bytes() as f32; + let rate = (recv - last_recv) / duration.as_secs_f32(); + last_recv = recv; + last_rate = rate; + last_update = Instant::now(); + (recv, rate) + } else { + (last_recv, last_rate) + }; + fn format_bytes(bytes: f32) -> (&'static str, f32) { + static UNITS: [&str; 5] = ["", "K", "M", "G", "T"]; + let i = (bytes.log2() / 10.0).min(4.0) as usize; + (UNITS[i], bytes / 1024_f32.powi(i as i32)) + } + let (rate_unit, rate) = format_bytes(rate); + let (unit, recv) = format_bytes(recv); + format!(" | {:.2}{}iB | {:.2}{}iB/s", recv, unit, rate, rate_unit) + }; progress - .tick(stats.indexed_objects(), stats.total_objects(), "") + .tick(stats.indexed_objects(), stats.total_objects(), &msg) .is_ok() }); From 7e3f7d64b016445f078f448eb62d75c49286fa81 Mon Sep 17 00:00:00 2001 From: Weihang Lo Date: Sat, 24 Apr 2021 00:42:45 +0800 Subject: [PATCH 005/129] fix: remove unnecessary progress tick rate limiting --- src/cargo/sources/git/utils.rs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/cargo/sources/git/utils.rs b/src/cargo/sources/git/utils.rs index 49b3b3946ed..81a08d71412 100644 --- a/src/cargo/sources/git/utils.rs +++ b/src/cargo/sources/git/utils.rs @@ -696,7 +696,6 @@ pub fn with_fetch_options( network::with_retry(config, || { with_authentication(url, git_config, |f| { let mut last_recv = 0.0; // in Byte - let mut last_rate = 0.0; // in Byte/s let mut last_update = Instant::now(); let mut rcb = git2::RemoteCallbacks::new(); rcb.credentials(f); @@ -708,16 +707,12 @@ pub fn with_fetch_options( } else { // Receiving objects. let duration = last_update.elapsed(); - let (recv, rate) = if duration > Duration::from_secs(1) { - let recv = stats.received_bytes() as f32; - let rate = (recv - last_recv) / duration.as_secs_f32(); + let recv = stats.received_bytes() as f32; + let rate = (recv - last_recv) / duration.as_secs_f32(); + if duration > Duration::from_secs(3) { last_recv = recv; - last_rate = rate; last_update = Instant::now(); - (recv, rate) - } else { - (last_recv, last_rate) - }; + } fn format_bytes(bytes: f32) -> (&'static str, f32) { static UNITS: [&str; 5] = ["", "K", "M", "G", "T"]; let i = (bytes.log2() / 10.0).min(4.0) as usize; @@ -725,7 +720,7 @@ pub fn with_fetch_options( } let (rate_unit, rate) = format_bytes(rate); let (unit, recv) = format_bytes(recv); - format!(" | {:.2}{}iB | {:.2}{}iB/s", recv, unit, rate, rate_unit) + format!(", {:.2}{}iB | {:.2}{}iB/s", recv, unit, rate, rate_unit) }; progress .tick(stats.indexed_objects(), stats.total_objects(), &msg) From 507d7e42b064cd7a10e1f9b0957031e3a2da456f Mon Sep 17 00:00:00 2001 From: Weihang Lo Date: Sat, 24 Apr 2021 00:56:34 +0800 Subject: [PATCH 006/129] fix: remove total bytes received from progress message --- src/cargo/sources/git/utils.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/cargo/sources/git/utils.rs b/src/cargo/sources/git/utils.rs index 81a08d71412..5dcdc3df53a 100644 --- a/src/cargo/sources/git/utils.rs +++ b/src/cargo/sources/git/utils.rs @@ -695,7 +695,7 @@ pub fn with_fetch_options( let mut progress = Progress::new("Fetch", config); network::with_retry(config, || { with_authentication(url, git_config, |f| { - let mut last_recv = 0.0; // in Byte + let mut last_recv = 0; // in Byte let mut last_update = Instant::now(); let mut rcb = git2::RemoteCallbacks::new(); rcb.credentials(f); @@ -707,8 +707,8 @@ pub fn with_fetch_options( } else { // Receiving objects. let duration = last_update.elapsed(); - let recv = stats.received_bytes() as f32; - let rate = (recv - last_recv) / duration.as_secs_f32(); + let recv = stats.received_bytes(); + let rate = (recv - last_recv) as f32 / duration.as_secs_f32(); if duration > Duration::from_secs(3) { last_recv = recv; last_update = Instant::now(); @@ -718,9 +718,8 @@ pub fn with_fetch_options( let i = (bytes.log2() / 10.0).min(4.0) as usize; (UNITS[i], bytes / 1024_f32.powi(i as i32)) } - let (rate_unit, rate) = format_bytes(rate); - let (unit, recv) = format_bytes(recv); - format!(", {:.2}{}iB | {:.2}{}iB/s", recv, unit, rate, rate_unit) + let (unit, rate) = format_bytes(rate); + format!(", {:.2}{}iB/s", rate, unit) }; progress .tick(stats.indexed_objects(), stats.total_objects(), &msg) From f2172db6da85e0b42d727ae4089c93c123256936 Mon Sep 17 00:00:00 2001 From: Weihang Lo Date: Sat, 24 Apr 2021 03:18:30 +0800 Subject: [PATCH 007/129] fix: message of progress when resolving deltas --- src/cargo/sources/git/utils.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/cargo/sources/git/utils.rs b/src/cargo/sources/git/utils.rs index 5dcdc3df53a..e2462528d1a 100644 --- a/src/cargo/sources/git/utils.rs +++ b/src/cargo/sources/git/utils.rs @@ -703,7 +703,11 @@ pub fn with_fetch_options( let indexed_deltas = stats.indexed_deltas(); let msg = if indexed_deltas > 0 { // Resolving deltas. - format!(" ({}/{})", indexed_deltas, stats.total_deltas()) + format!( + ", ({}/{}) resolving deltas", + indexed_deltas, + stats.total_deltas() + ) } else { // Receiving objects. let duration = last_update.elapsed(); From 083cc9e8d0cc369658963399fe517492ca8fafb7 Mon Sep 17 00:00:00 2001 From: James Hilliard Date: Thu, 1 Apr 2021 21:06:31 -0600 Subject: [PATCH 008/129] Configure hosts separately from targets when --target is specified. This prevents target configs from accidentally being picked up when cross compiling from hosts that have the same architecture as their targets. closes #3349 --- .../compiler/build_context/target_info.rs | 20 +- src/cargo/util/config/mod.rs | 5 + src/cargo/util/config/target.rs | 31 ++- tests/testsuite/build_script.rs | 240 ++++++++++++++++++ 4 files changed, 292 insertions(+), 4 deletions(-) diff --git a/src/cargo/core/compiler/build_context/target_info.rs b/src/cargo/core/compiler/build_context/target_info.rs index 325d4168bc3..0692e0c1ca6 100644 --- a/src/cargo/core/compiler/build_context/target_info.rs +++ b/src/cargo/core/compiler/build_context/target_info.rs @@ -682,7 +682,6 @@ impl<'cfg> RustcTargetData<'cfg> { ) -> CargoResult> { let config = ws.config(); let rustc = config.load_global_rustc(Some(ws))?; - let host_config = config.target_cfg_triple(&rustc.host)?; let host_info = TargetInfo::new(config, requested_kinds, &rustc, CompileKind::Host)?; let mut target_config = HashMap::new(); let mut target_info = HashMap::new(); @@ -692,10 +691,25 @@ impl<'cfg> RustcTargetData<'cfg> { // `--target` flag is not specified. Since the unit_dependency code // needs access to the target config data, create a copy so that it // can be found. See `rebuild_unit_graph_shared` for why this is done. - if requested_kinds.iter().any(CompileKind::is_host) { + let host_config = if requested_kinds.iter().any(CompileKind::is_host) { let ct = CompileTarget::new(&rustc.host)?; target_info.insert(ct, host_info.clone()); - target_config.insert(ct, host_config.clone()); + let target_host_config = config.target_cfg_triple(&rustc.host)?; + target_config.insert(ct, target_host_config.clone()); + target_host_config + } else { + config.host_cfg_triple(&rustc.host)? + }; + + for kind in requested_kinds { + if let CompileKind::Target(target) = *kind { + let tcfg = config.target_cfg_triple(target.short_name())?; + target_config.insert(target, tcfg); + target_info.insert( + target, + TargetInfo::new(config, requested_kinds, &rustc, *kind)?, + ); + } } let mut res = RustcTargetData { diff --git a/src/cargo/util/config/mod.rs b/src/cargo/util/config/mod.rs index b9921dfee26..c5b9d7c62da 100644 --- a/src/cargo/util/config/mod.rs +++ b/src/cargo/util/config/mod.rs @@ -1486,6 +1486,11 @@ impl Config { .try_borrow_with(|| self.get::("doc.extern-map")) } + /// Returns the `[host]` table definition for the given target triple. + pub fn host_cfg_triple(&self, target: &str) -> CargoResult { + target::load_host_triple(self, target) + } + /// Returns the `[target]` table definition for the given target triple. pub fn target_cfg_triple(&self, target: &str) -> CargoResult { target::load_target_triple(self, target) diff --git a/src/cargo/util/config/target.rs b/src/cargo/util/config/target.rs index e22cab92ee5..11f6085c7d5 100644 --- a/src/cargo/util/config/target.rs +++ b/src/cargo/util/config/target.rs @@ -19,7 +19,7 @@ pub struct TargetCfgConfig { pub other: BTreeMap, } -/// Config definition of a `[target]` table. +/// Config definition of a `[target]` table or `[host]`. #[derive(Debug, Clone)] pub struct TargetConfig { /// Process to run as a wrapper for `cargo run`, `test`, and `bench` commands. @@ -64,6 +64,35 @@ pub(super) fn load_target_cfgs(config: &Config) -> CargoResult CargoResult { + // This needs to get each field individually because it cannot fetch the + // struct all at once due to `links_overrides`. Can't use `serde(flatten)` + // because it causes serde to use `deserialize_map` which means the config + // deserializer does not know which keys to deserialize, which means + // environment variables would not work. + let host_triple_key = ConfigKey::from_str(&format!("host.{}", triple)); + let host_prefix = match config.get_cv(&host_triple_key)? { + Some(_) => format!("host.{}", triple), + None => "host".to_string(), + }; + let runner: OptValue = config.get(&format!("{}.runner", host_prefix))?; + let rustflags: OptValue = config.get(&format!("{}.rustflags", host_prefix))?; + let linker: OptValue = config.get(&format!("{}.linker", host_prefix))?; + // Links do not support environment variables. + let target_key = ConfigKey::from_str(&host_prefix); + let links_overrides = match config.get_table(&target_key)? { + Some(links) => parse_links_overrides(&target_key, links.val, config)?, + None => BTreeMap::new(), + }; + Ok(TargetConfig { + runner, + rustflags, + linker, + links_overrides, + }) +} + /// Loads a single `[target]` table for the given triple. pub(super) fn load_target_triple(config: &Config, triple: &str) -> CargoResult { // This needs to get each field individually because it cannot fetch the diff --git a/tests/testsuite/build_script.rs b/tests/testsuite/build_script.rs index a43856548b0..d3126e1cad5 100644 --- a/tests/testsuite/build_script.rs +++ b/tests/testsuite/build_script.rs @@ -164,6 +164,246 @@ fn custom_build_env_var_rustc_linker() { p.cargo("build --target").arg(&target).run(); } +#[cargo_test] +fn custom_build_env_var_rustc_linker_bad_host_target() { + let target = rustc_host(); + let p = project() + .file( + ".cargo/config", + &format!( + r#" + [target.{}] + linker = "/path/to/linker" + "#, + target + ), + ) + .file( + "build.rs", + r#" + use std::env; + + fn main() { + assert!(env::var("RUSTC_LINKER").unwrap().ends_with("/path/to/linker")); + } + "#, + ) + .file("src/lib.rs", "") + .build(); + + // build.rs should fail since host == target when no target is set + p.cargo("build --verbose") + .with_status(101) + .with_stderr_contains( + "\ +[COMPILING] foo v0.0.1 ([CWD]) +[RUNNING] `rustc --crate-name build_script_build build.rs [..]--crate-type bin [..]-C linker=[..]/path/to/linker [..]` +[ERROR] linker `[..]/path/to/linker` not found +" + ) + .run(); +} + +#[cargo_test] +fn custom_build_env_var_rustc_linker_host_target() { + let target = rustc_host(); + let p = project() + .file( + ".cargo/config", + &format!( + r#" + [target.{}] + linker = "/path/to/linker" + "#, + target + ), + ) + .file( + "build.rs", + r#" + use std::env; + + fn main() { + assert!(env::var("RUSTC_LINKER").unwrap().ends_with("/path/to/linker")); + } + "#, + ) + .file("src/lib.rs", "") + .build(); + + // no crate type set => linker never called => build succeeds if and + // only if build.rs succeeds, despite linker binary not existing. + p.cargo("build --target").arg(&target).run(); +} + +#[cargo_test] +fn custom_build_env_var_rustc_linker_bad_host() { + let target = rustc_host(); + let p = project() + .file( + ".cargo/config", + &format!( + r#" + [host] + linker = "/path/to/host/linker" + [target.{}] + linker = "/path/to/target/linker" + "#, + target + ), + ) + .file( + "build.rs", + r#" + use std::env; + + fn main() { + assert!(env::var("RUSTC_LINKER").unwrap().ends_with("/path/to/target/linker")); + } + "#, + ) + .file("src/lib.rs", "") + .build(); + + // build.rs should fail due to bad host linker being set + p.cargo("build --verbose --target") + .arg(&target) + .with_status(101) + .with_stderr_contains( + "\ +[COMPILING] foo v0.0.1 ([CWD]) +[RUNNING] `rustc --crate-name build_script_build build.rs [..]--crate-type bin [..]-C linker=[..]/path/to/host/linker [..]` +[ERROR] linker `[..]/path/to/host/linker` not found +" + ) + .run(); +} + +#[cargo_test] +fn custom_build_env_var_rustc_linker_bad_host_with_arch() { + let target = rustc_host(); + let p = project() + .file( + ".cargo/config", + &format!( + r#" + [host] + linker = "/path/to/host/linker" + [host.{}] + linker = "/path/to/host/arch/linker" + [target.{}] + linker = "/path/to/target/linker" + "#, + target, target + ), + ) + .file( + "build.rs", + r#" + use std::env; + + fn main() { + assert!(env::var("RUSTC_LINKER").unwrap().ends_with("/path/to/target/linker")); + } + "#, + ) + .file("src/lib.rs", "") + .build(); + + // build.rs should fail due to bad host linker being set + p.cargo("build --verbose --target") + .arg(&target) + .with_status(101) + .with_stderr_contains( + "\ +[COMPILING] foo v0.0.1 ([CWD]) +[RUNNING] `rustc --crate-name build_script_build build.rs [..]--crate-type bin [..]-C linker=[..]/path/to/host/arch/linker [..]` +[ERROR] linker `[..]/path/to/host/arch/linker` not found +" + ) + .run(); +} + +#[cargo_test] +fn custom_build_env_var_rustc_linker_cross_arch_host() { + let target = rustc_host(); + let cross_target = cross_compile::alternate(); + let p = project() + .file( + ".cargo/config", + &format!( + r#" + [host.{}] + linker = "/path/to/host/arch/linker" + [target.{}] + linker = "/path/to/target/linker" + "#, + cross_target, target + ), + ) + .file( + "build.rs", + r#" + use std::env; + + fn main() { + assert!(env::var("RUSTC_LINKER").unwrap().ends_with("/path/to/target/linker")); + } + "#, + ) + .file("src/lib.rs", "") + .build(); + + // build.rs should fail due to bad host linker being set + p.cargo("build --verbose --target").arg(&target).run(); +} + +#[cargo_test] +fn custom_build_env_var_rustc_linker_bad_cross_arch_host() { + let target = rustc_host(); + let cross_target = cross_compile::alternate(); + let p = project() + .file( + ".cargo/config", + &format!( + r#" + [host] + linker = "/path/to/host/linker" + [host.{}] + linker = "/path/to/host/arch/linker" + [target.{}] + linker = "/path/to/target/linker" + "#, + cross_target, target + ), + ) + .file( + "build.rs", + r#" + use std::env; + + fn main() { + assert!(env::var("RUSTC_LINKER").unwrap().ends_with("/path/to/target/linker")); + } + "#, + ) + .file("src/lib.rs", "") + .build(); + + // build.rs should fail due to bad host linker being set + p.cargo("build --verbose --target") + .arg(&target) + .with_status(101) + .with_stderr_contains( + "\ +[COMPILING] foo v0.0.1 ([CWD]) +[RUNNING] `rustc --crate-name build_script_build build.rs [..]--crate-type bin [..]-C linker=[..]/path/to/host/linker [..]` +[ERROR] linker `[..]/path/to/host/linker` not found +" + ) + .run(); +} + #[cargo_test] fn custom_build_script_wrong_rustc_flags() { let p = project() From e51d1511a661a3e15bff8b152c515ca514bf708b Mon Sep 17 00:00:00 2001 From: James Hilliard Date: Tue, 20 Apr 2021 09:44:02 -0600 Subject: [PATCH 009/129] Refactor host and target config table loads to deduplicate logic. --- src/cargo/util/config/target.rs | 34 ++++++++++----------------------- 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/src/cargo/util/config/target.rs b/src/cargo/util/config/target.rs index 11f6085c7d5..59bf7c1866a 100644 --- a/src/cargo/util/config/target.rs +++ b/src/cargo/util/config/target.rs @@ -66,45 +66,31 @@ pub(super) fn load_target_cfgs(config: &Config) -> CargoResult CargoResult { - // This needs to get each field individually because it cannot fetch the - // struct all at once due to `links_overrides`. Can't use `serde(flatten)` - // because it causes serde to use `deserialize_map` which means the config - // deserializer does not know which keys to deserialize, which means - // environment variables would not work. let host_triple_key = ConfigKey::from_str(&format!("host.{}", triple)); let host_prefix = match config.get_cv(&host_triple_key)? { Some(_) => format!("host.{}", triple), None => "host".to_string(), }; - let runner: OptValue = config.get(&format!("{}.runner", host_prefix))?; - let rustflags: OptValue = config.get(&format!("{}.rustflags", host_prefix))?; - let linker: OptValue = config.get(&format!("{}.linker", host_prefix))?; - // Links do not support environment variables. - let target_key = ConfigKey::from_str(&host_prefix); - let links_overrides = match config.get_table(&target_key)? { - Some(links) => parse_links_overrides(&target_key, links.val, config)?, - None => BTreeMap::new(), - }; - Ok(TargetConfig { - runner, - rustflags, - linker, - links_overrides, - }) + load_config_table(config, &host_prefix) } /// Loads a single `[target]` table for the given triple. pub(super) fn load_target_triple(config: &Config, triple: &str) -> CargoResult { + load_config_table(config, &format!("target.{}", triple)) +} + +/// Loads a single table for the given prefix. +fn load_config_table(config: &Config, prefix: &str) -> CargoResult { // This needs to get each field individually because it cannot fetch the // struct all at once due to `links_overrides`. Can't use `serde(flatten)` // because it causes serde to use `deserialize_map` which means the config // deserializer does not know which keys to deserialize, which means // environment variables would not work. - let runner: OptValue = config.get(&format!("target.{}.runner", triple))?; - let rustflags: OptValue = config.get(&format!("target.{}.rustflags", triple))?; - let linker: OptValue = config.get(&format!("target.{}.linker", triple))?; + let runner: OptValue = config.get(&format!("{}.runner", prefix))?; + let rustflags: OptValue = config.get(&format!("{}.rustflags", prefix))?; + let linker: OptValue = config.get(&format!("{}.linker", prefix))?; // Links do not support environment variables. - let target_key = ConfigKey::from_str(&format!("target.{}", triple)); + let target_key = ConfigKey::from_str(prefix); let links_overrides = match config.get_table(&target_key)? { Some(links) => parse_links_overrides(&target_key, links.val, config)?, None => BTreeMap::new(), From 0a603fdf6eddf7ed21874fc846af14ac7120d4e9 Mon Sep 17 00:00:00 2001 From: James Hilliard Date: Wed, 21 Apr 2021 00:18:30 -0600 Subject: [PATCH 010/129] Document and feature-gate host-config tables. --- src/cargo/core/features.rs | 2 ++ src/cargo/util/config/target.rs | 21 +++++++---- src/doc/src/reference/unstable.md | 33 ++++++++++++++++++ tests/testsuite/build_script.rs | 58 +++++++++++++++++++------------ 4 files changed, 86 insertions(+), 28 deletions(-) diff --git a/src/cargo/core/features.rs b/src/cargo/core/features.rs index 56321cdde80..b5298f19e6c 100644 --- a/src/cargo/core/features.rs +++ b/src/cargo/core/features.rs @@ -600,6 +600,7 @@ unstable_cli_options!( namespaced_features: bool = ("Allow features with `dep:` prefix"), no_index_update: bool = ("Do not update the registry index even if the cache is outdated"), panic_abort_tests: bool = ("Enable support to run tests with -Cpanic=abort"), + host_config: bool = ("Enable the [host] section in the .cargo/config.toml file"), patch_in_config: bool = ("Allow `[patch]` sections in .cargo/config.toml files"), rustdoc_map: bool = ("Allow passing external documentation mappings to rustdoc"), separate_nightlies: bool = (HIDDEN), @@ -787,6 +788,7 @@ impl CliUnstable { "panic-abort-tests" => self.panic_abort_tests = parse_empty(k, v)?, "jobserver-per-rustc" => self.jobserver_per_rustc = parse_empty(k, v)?, "configurable-env" => self.configurable_env = parse_empty(k, v)?, + "host-config" => self.host_config = parse_empty(k, v)?, "patch-in-config" => self.patch_in_config = parse_empty(k, v)?, "features" => { // For now this is still allowed (there are still some diff --git a/src/cargo/util/config/target.rs b/src/cargo/util/config/target.rs index 59bf7c1866a..1af36c12ac1 100644 --- a/src/cargo/util/config/target.rs +++ b/src/cargo/util/config/target.rs @@ -66,12 +66,21 @@ pub(super) fn load_target_cfgs(config: &Config) -> CargoResult CargoResult { - let host_triple_key = ConfigKey::from_str(&format!("host.{}", triple)); - let host_prefix = match config.get_cv(&host_triple_key)? { - Some(_) => format!("host.{}", triple), - None => "host".to_string(), - }; - load_config_table(config, &host_prefix) + if config.cli_unstable().host_config { + let host_triple_key = ConfigKey::from_str(&format!("host.{}", triple)); + let host_prefix = match config.get_cv(&host_triple_key)? { + Some(_) => format!("host.{}", triple), + None => "host".to_string(), + }; + load_config_table(config, &host_prefix) + } else { + Ok(TargetConfig { + runner: None, + rustflags: None, + linker: None, + links_overrides: BTreeMap::new(), + }) + } } /// Loads a single `[target]` table for the given triple. diff --git a/src/doc/src/reference/unstable.md b/src/doc/src/reference/unstable.md index 8e7927f8308..f5b671d4f7d 100644 --- a/src/doc/src/reference/unstable.md +++ b/src/doc/src/reference/unstable.md @@ -641,6 +641,39 @@ cargo +nightly -Zunstable-options -Zconfig-include --config somefile.toml build CLI paths are relative to the current working directory. +### host-config +* Original Pull Request: [#9322](https://github.com/rust-lang/cargo/pull/9322) +* Tracking Issue: [#3349](https://github.com/rust-lang/cargo/issues/3349) + +The `host` key in a config file can be used pass flags to host build targets +such as build scripts that must run on the host system instead of the target +system when cross compiling. It supports both generic and host arch specific +tables. Matching host arch tables take precedence over generic host tables. + +It requires the `-Zhost-config` command-line option. + +```toml +# .cargo/config +cargo-features = ["host-config"] + +[host] +linker = "/path/to/host/linker" +[host.x86_64-unknown-linux-gnu] +linker = "/path/to/host/arch/linker" +[target.x86_64-unknown-linux-gnu] +linker = "/path/to/target/linker" +``` + +The generic `host` table above will be entirely ignored when building on a +`x86_64-unknown-linux-gnu` host as the `host.x86_64-unknown-linux-gnu` table +takes precedence. + +This feature requires a `--target` to be specified. + +```console +cargo +nightly -Zunstable-options -Zhost-config --config somefile.toml build --target x86_64-unknown-linux-gnu +``` + ### unit-graph * Tracking Issue: [#8002](https://github.com/rust-lang/cargo/issues/8002) diff --git a/tests/testsuite/build_script.rs b/tests/testsuite/build_script.rs index d3126e1cad5..d383004f084 100644 --- a/tests/testsuite/build_script.rs +++ b/tests/testsuite/build_script.rs @@ -266,17 +266,20 @@ fn custom_build_env_var_rustc_linker_bad_host() { .build(); // build.rs should fail due to bad host linker being set - p.cargo("build --verbose --target") - .arg(&target) - .with_status(101) - .with_stderr_contains( - "\ + if cargo_test_support::is_nightly() { + p.cargo("build -Z host-config --verbose --target") + .arg(&target) + .masquerade_as_nightly_cargo() + .with_status(101) + .with_stderr_contains( + "\ [COMPILING] foo v0.0.1 ([CWD]) [RUNNING] `rustc --crate-name build_script_build build.rs [..]--crate-type bin [..]-C linker=[..]/path/to/host/linker [..]` [ERROR] linker `[..]/path/to/host/linker` not found " - ) - .run(); + ) + .run(); + } } #[cargo_test] @@ -311,17 +314,20 @@ fn custom_build_env_var_rustc_linker_bad_host_with_arch() { .build(); // build.rs should fail due to bad host linker being set - p.cargo("build --verbose --target") - .arg(&target) - .with_status(101) - .with_stderr_contains( - "\ + if cargo_test_support::is_nightly() { + p.cargo("build -Z host-config --verbose --target") + .arg(&target) + .masquerade_as_nightly_cargo() + .with_status(101) + .with_stderr_contains( + "\ [COMPILING] foo v0.0.1 ([CWD]) [RUNNING] `rustc --crate-name build_script_build build.rs [..]--crate-type bin [..]-C linker=[..]/path/to/host/arch/linker [..]` [ERROR] linker `[..]/path/to/host/arch/linker` not found " - ) - .run(); + ) + .run(); + } } #[cargo_test] @@ -355,7 +361,12 @@ fn custom_build_env_var_rustc_linker_cross_arch_host() { .build(); // build.rs should fail due to bad host linker being set - p.cargo("build --verbose --target").arg(&target).run(); + if cargo_test_support::is_nightly() { + p.cargo("build -Z host-config --verbose --target") + .arg(&target) + .masquerade_as_nightly_cargo() + .run(); + } } #[cargo_test] @@ -391,17 +402,20 @@ fn custom_build_env_var_rustc_linker_bad_cross_arch_host() { .build(); // build.rs should fail due to bad host linker being set - p.cargo("build --verbose --target") - .arg(&target) - .with_status(101) - .with_stderr_contains( - "\ + if cargo_test_support::is_nightly() { + p.cargo("build -Z host-config --verbose --target") + .arg(&target) + .masquerade_as_nightly_cargo() + .with_status(101) + .with_stderr_contains( + "\ [COMPILING] foo v0.0.1 ([CWD]) [RUNNING] `rustc --crate-name build_script_build build.rs [..]--crate-type bin [..]-C linker=[..]/path/to/host/linker [..]` [ERROR] linker `[..]/path/to/host/linker` not found " - ) - .run(); + ) + .run(); + } } #[cargo_test] From 46f9541740201ca3b095de016f7a4226ce24e717 Mon Sep 17 00:00:00 2001 From: James Hilliard Date: Fri, 23 Apr 2021 07:23:13 -0600 Subject: [PATCH 011/129] Add target-applies-to-host and default to true unless host-config is enabled. --- .../compiler/build_context/target_info.rs | 16 +++- src/cargo/util/config/mod.rs | 5 ++ src/cargo/util/config/target.rs | 14 ++++ tests/testsuite/build_script.rs | 83 +++++++++++++++++++ 4 files changed, 115 insertions(+), 3 deletions(-) diff --git a/src/cargo/core/compiler/build_context/target_info.rs b/src/cargo/core/compiler/build_context/target_info.rs index 0692e0c1ca6..e663745b26b 100644 --- a/src/cargo/core/compiler/build_context/target_info.rs +++ b/src/cargo/core/compiler/build_context/target_info.rs @@ -694,11 +694,21 @@ impl<'cfg> RustcTargetData<'cfg> { let host_config = if requested_kinds.iter().any(CompileKind::is_host) { let ct = CompileTarget::new(&rustc.host)?; target_info.insert(ct, host_info.clone()); - let target_host_config = config.target_cfg_triple(&rustc.host)?; - target_config.insert(ct, target_host_config.clone()); + let target_host_config = if config.target_applies_to_host() { + let target_cfg_clone = config.target_cfg_triple(&rustc.host)?; + target_config.insert(ct, target_cfg_clone.clone()); + target_cfg_clone + } else { + target_config.insert(ct, config.target_cfg_triple(&rustc.host)?); + config.host_cfg_triple(&rustc.host)? + }; target_host_config } else { - config.host_cfg_triple(&rustc.host)? + if config.target_applies_to_host() { + config.target_cfg_triple(&rustc.host)? + } else { + config.host_cfg_triple(&rustc.host)? + } }; for kind in requested_kinds { diff --git a/src/cargo/util/config/mod.rs b/src/cargo/util/config/mod.rs index c5b9d7c62da..9b1284fedde 100644 --- a/src/cargo/util/config/mod.rs +++ b/src/cargo/util/config/mod.rs @@ -1486,6 +1486,11 @@ impl Config { .try_borrow_with(|| self.get::("doc.extern-map")) } + /// Returns true if the `[target]` table should be applied to host targets. + pub fn target_applies_to_host(&self) -> bool { + target::get_target_applies_to_host(self) + } + /// Returns the `[host]` table definition for the given target triple. pub fn host_cfg_triple(&self, target: &str) -> CargoResult { target::load_host_triple(self, target) diff --git a/src/cargo/util/config/target.rs b/src/cargo/util/config/target.rs index 1af36c12ac1..0f9ea32af72 100644 --- a/src/cargo/util/config/target.rs +++ b/src/cargo/util/config/target.rs @@ -64,6 +64,20 @@ pub(super) fn load_target_cfgs(config: &Config) -> CargoResult bool { + let target_applies_to_host = config.get::("target-applies-to-host"); + if target_applies_to_host.is_ok() { + target_applies_to_host.unwrap() + } else { + if config.cli_unstable().host_config { + false + } else { + true + } + } +} + /// Loads a single `[host]` table for the given triple. pub(super) fn load_host_triple(config: &Config, triple: &str) -> CargoResult { if config.cli_unstable().host_config { diff --git a/tests/testsuite/build_script.rs b/tests/testsuite/build_script.rs index d383004f084..7bc1d71598f 100644 --- a/tests/testsuite/build_script.rs +++ b/tests/testsuite/build_script.rs @@ -212,6 +212,7 @@ fn custom_build_env_var_rustc_linker_host_target() { ".cargo/config", &format!( r#" + target-applies-to-host = false [target.{}] linker = "/path/to/linker" "#, @@ -236,6 +237,88 @@ fn custom_build_env_var_rustc_linker_host_target() { p.cargo("build --target").arg(&target).run(); } +#[cargo_test] +fn custom_build_env_var_rustc_linker_host_target_env() { + let target = rustc_host(); + let p = project() + .file( + ".cargo/config", + &format!( + r#" + [target.{}] + linker = "/path/to/linker" + "#, + target + ), + ) + .file( + "build.rs", + r#" + use std::env; + + fn main() { + assert!(env::var("RUSTC_LINKER").unwrap().ends_with("/path/to/linker")); + } + "#, + ) + .file("src/lib.rs", "") + .build(); + + // no crate type set => linker never called => build succeeds if and + // only if build.rs succeeds, despite linker binary not existing. + p.cargo("build --target") + .env("CARGO_TARGET_APPLIES_TO_HOST", "false") + .arg(&target) + .run(); +} + +#[cargo_test] +fn custom_build_env_var_rustc_linker_host_target_with_bad_host_config() { + let target = rustc_host(); + let p = project() + .file( + ".cargo/config", + &format!( + r#" + target-applies-to-host = true + [host] + linker = "/path/to/host/linker" + [target.{}] + linker = "/path/to/target/linker" + "#, + target + ), + ) + .file( + "build.rs", + r#" + use std::env; + + fn main() { + assert!(env::var("RUSTC_LINKER").unwrap().ends_with("/path/to/target/linker")); + } + "#, + ) + .file("src/lib.rs", "") + .build(); + + // build.rs should fail due to bad target linker being set + if cargo_test_support::is_nightly() { + p.cargo("build -Z host-config --verbose --target") + .arg(&target) + .masquerade_as_nightly_cargo() + .with_status(101) + .with_stderr_contains( + "\ +[COMPILING] foo v0.0.1 ([CWD]) +[RUNNING] `rustc --crate-name build_script_build build.rs [..]--crate-type bin [..]-C linker=[..]/path/to/target/linker [..]` +[ERROR] linker `[..]/path/to/target/linker` not found +" + ) + .run(); + } +} + #[cargo_test] fn custom_build_env_var_rustc_linker_bad_host() { let target = rustc_host(); From e24bf92219b2296121c9306fde3e9f9436d01d61 Mon Sep 17 00:00:00 2001 From: James Hilliard Date: Mon, 26 Apr 2021 21:03:01 -0600 Subject: [PATCH 012/129] Add feature gate for target-applies-to-host. --- .../compiler/build_context/target_info.rs | 8 ++++-- src/cargo/core/features.rs | 2 ++ src/cargo/util/config/mod.rs | 2 +- src/cargo/util/config/target.rs | 22 +++++++++++---- tests/testsuite/build_script.rs | 28 ++++++++++++------- 5 files changed, 43 insertions(+), 19 deletions(-) diff --git a/src/cargo/core/compiler/build_context/target_info.rs b/src/cargo/core/compiler/build_context/target_info.rs index e663745b26b..08b6098cc4d 100644 --- a/src/cargo/core/compiler/build_context/target_info.rs +++ b/src/cargo/core/compiler/build_context/target_info.rs @@ -691,10 +691,14 @@ impl<'cfg> RustcTargetData<'cfg> { // `--target` flag is not specified. Since the unit_dependency code // needs access to the target config data, create a copy so that it // can be found. See `rebuild_unit_graph_shared` for why this is done. + let target_applies_to_host = config.target_applies_to_host(); + if target_applies_to_host.is_err() { + return Err(target_applies_to_host.unwrap_err()); + } let host_config = if requested_kinds.iter().any(CompileKind::is_host) { let ct = CompileTarget::new(&rustc.host)?; target_info.insert(ct, host_info.clone()); - let target_host_config = if config.target_applies_to_host() { + let target_host_config = if target_applies_to_host.unwrap() { let target_cfg_clone = config.target_cfg_triple(&rustc.host)?; target_config.insert(ct, target_cfg_clone.clone()); target_cfg_clone @@ -704,7 +708,7 @@ impl<'cfg> RustcTargetData<'cfg> { }; target_host_config } else { - if config.target_applies_to_host() { + if target_applies_to_host.unwrap() { config.target_cfg_triple(&rustc.host)? } else { config.host_cfg_triple(&rustc.host)? diff --git a/src/cargo/core/features.rs b/src/cargo/core/features.rs index b5298f19e6c..d78eed8ccd8 100644 --- a/src/cargo/core/features.rs +++ b/src/cargo/core/features.rs @@ -601,6 +601,7 @@ unstable_cli_options!( no_index_update: bool = ("Do not update the registry index even if the cache is outdated"), panic_abort_tests: bool = ("Enable support to run tests with -Cpanic=abort"), host_config: bool = ("Enable the [host] section in the .cargo/config.toml file"), + target_applies_to_host: bool = ("Enable the `target-applies-to-host` key in the .cargo/config.toml file"), patch_in_config: bool = ("Allow `[patch]` sections in .cargo/config.toml files"), rustdoc_map: bool = ("Allow passing external documentation mappings to rustdoc"), separate_nightlies: bool = (HIDDEN), @@ -789,6 +790,7 @@ impl CliUnstable { "jobserver-per-rustc" => self.jobserver_per_rustc = parse_empty(k, v)?, "configurable-env" => self.configurable_env = parse_empty(k, v)?, "host-config" => self.host_config = parse_empty(k, v)?, + "target-applies-to-host" => self.target_applies_to_host = parse_empty(k, v)?, "patch-in-config" => self.patch_in_config = parse_empty(k, v)?, "features" => { // For now this is still allowed (there are still some diff --git a/src/cargo/util/config/mod.rs b/src/cargo/util/config/mod.rs index 9b1284fedde..71bbf78b456 100644 --- a/src/cargo/util/config/mod.rs +++ b/src/cargo/util/config/mod.rs @@ -1487,7 +1487,7 @@ impl Config { } /// Returns true if the `[target]` table should be applied to host targets. - pub fn target_applies_to_host(&self) -> bool { + pub fn target_applies_to_host(&self) -> CargoResult { target::get_target_applies_to_host(self) } diff --git a/src/cargo/util/config/target.rs b/src/cargo/util/config/target.rs index 0f9ea32af72..8fa7b6cc97f 100644 --- a/src/cargo/util/config/target.rs +++ b/src/cargo/util/config/target.rs @@ -65,15 +65,25 @@ pub(super) fn load_target_cfgs(config: &Config) -> CargoResult bool { - let target_applies_to_host = config.get::("target-applies-to-host"); - if target_applies_to_host.is_ok() { - target_applies_to_host.unwrap() +pub(super) fn get_target_applies_to_host(config: &Config) -> CargoResult { + if config.cli_unstable().target_applies_to_host { + let target_applies_to_host = config.get::("target-applies-to-host"); + if target_applies_to_host.is_ok() { + Ok(target_applies_to_host.unwrap()) + } else { + if config.cli_unstable().host_config { + Ok(false) + } else { + Ok(true) + } + } } else { if config.cli_unstable().host_config { - false + anyhow::bail!( + "the -Zhost-config flag requires the -Ztarget-applies-to-host flag to be set" + ); } else { - true + Ok(true) } } } diff --git a/tests/testsuite/build_script.rs b/tests/testsuite/build_script.rs index 7bc1d71598f..48112e56568 100644 --- a/tests/testsuite/build_script.rs +++ b/tests/testsuite/build_script.rs @@ -234,7 +234,12 @@ fn custom_build_env_var_rustc_linker_host_target() { // no crate type set => linker never called => build succeeds if and // only if build.rs succeeds, despite linker binary not existing. - p.cargo("build --target").arg(&target).run(); + if cargo_test_support::is_nightly() { + p.cargo("build -Z target-applies-to-host --target") + .arg(&target) + .masquerade_as_nightly_cargo() + .run(); + } } #[cargo_test] @@ -266,10 +271,13 @@ fn custom_build_env_var_rustc_linker_host_target_env() { // no crate type set => linker never called => build succeeds if and // only if build.rs succeeds, despite linker binary not existing. - p.cargo("build --target") - .env("CARGO_TARGET_APPLIES_TO_HOST", "false") - .arg(&target) - .run(); + if cargo_test_support::is_nightly() { + p.cargo("build -Z target-applies-to-host --target") + .env("CARGO_TARGET_APPLIES_TO_HOST", "false") + .arg(&target) + .masquerade_as_nightly_cargo() + .run(); + } } #[cargo_test] @@ -304,7 +312,7 @@ fn custom_build_env_var_rustc_linker_host_target_with_bad_host_config() { // build.rs should fail due to bad target linker being set if cargo_test_support::is_nightly() { - p.cargo("build -Z host-config --verbose --target") + p.cargo("build -Z target-applies-to-host -Z host-config --verbose --target") .arg(&target) .masquerade_as_nightly_cargo() .with_status(101) @@ -350,7 +358,7 @@ fn custom_build_env_var_rustc_linker_bad_host() { // build.rs should fail due to bad host linker being set if cargo_test_support::is_nightly() { - p.cargo("build -Z host-config --verbose --target") + p.cargo("build -Z target-applies-to-host -Z host-config --verbose --target") .arg(&target) .masquerade_as_nightly_cargo() .with_status(101) @@ -398,7 +406,7 @@ fn custom_build_env_var_rustc_linker_bad_host_with_arch() { // build.rs should fail due to bad host linker being set if cargo_test_support::is_nightly() { - p.cargo("build -Z host-config --verbose --target") + p.cargo("build -Z target-applies-to-host -Z host-config --verbose --target") .arg(&target) .masquerade_as_nightly_cargo() .with_status(101) @@ -445,7 +453,7 @@ fn custom_build_env_var_rustc_linker_cross_arch_host() { // build.rs should fail due to bad host linker being set if cargo_test_support::is_nightly() { - p.cargo("build -Z host-config --verbose --target") + p.cargo("build -Z target-applies-to-host -Z host-config --verbose --target") .arg(&target) .masquerade_as_nightly_cargo() .run(); @@ -486,7 +494,7 @@ fn custom_build_env_var_rustc_linker_bad_cross_arch_host() { // build.rs should fail due to bad host linker being set if cargo_test_support::is_nightly() { - p.cargo("build -Z host-config --verbose --target") + p.cargo("build -Z target-applies-to-host -Z host-config --verbose --target") .arg(&target) .masquerade_as_nightly_cargo() .with_status(101) From a813ba0070171cfdf0495f07889a3a808892b3ff Mon Sep 17 00:00:00 2001 From: Weihang Lo Date: Tue, 27 Apr 2021 13:47:36 +0800 Subject: [PATCH 013/129] feat: add metrics counter utility --- src/cargo/util/counter.rs | 61 +++++++++++++++++++++++++++++++++++++++ src/cargo/util/mod.rs | 2 ++ 2 files changed, 63 insertions(+) create mode 100644 src/cargo/util/counter.rs diff --git a/src/cargo/util/counter.rs b/src/cargo/util/counter.rs new file mode 100644 index 00000000000..04dd1647006 --- /dev/null +++ b/src/cargo/util/counter.rs @@ -0,0 +1,61 @@ +use std::time::Instant; + +/// A metrics counter storing only latest `N` records. +pub struct MetricsCounter { + /// Slots to store metrics. + slots: [(usize, Instant); N], + /// The slot of the oldest record. + /// Also the next slot to store the new record. + index: usize, +} + +impl MetricsCounter { + /// Creates a new counter with an initial value. + pub fn new(init: usize) -> Self { + debug_assert!(N > 0, "number of slots must be greater than zero"); + Self { + slots: [(init, Instant::now()); N], + index: 0, + } + } + + /// Adds record to the counter. + pub fn add(&mut self, data: usize) { + self.slots[self.index] = (data, Instant::now()); + self.index = (self.index + 1) % N; + } + + /// Calculates per-second average rate of all slots. + pub fn rate(&self) -> f32 { + let latest = self.slots[self.index.checked_sub(1).unwrap_or(N - 1)]; + let oldest = self.slots[self.index]; + let duration = (latest.1 - oldest.1).as_secs_f32(); + let avg = (latest.0 - oldest.0) as f32 / duration; + if f32::is_nan(avg) { + 0f32 + } else { + avg + } + } +} + +#[cfg(test)] +mod tests { + use super::MetricsCounter; + + #[test] + fn counter() { + let mut counter = MetricsCounter::<3>::new(0); + assert_eq!(counter.rate(), 0f32); + for i in 1..=5 { + counter.add(i); + assert!(counter.rate() > 0f32); + } + } + + #[test] + #[should_panic(expected = "number of slots must be greater than zero")] + fn counter_zero_slot() { + let _counter = MetricsCounter::<0>::new(0); + } +} diff --git a/src/cargo/util/mod.rs b/src/cargo/util/mod.rs index f7759174a79..e00727c4968 100644 --- a/src/cargo/util/mod.rs +++ b/src/cargo/util/mod.rs @@ -3,6 +3,7 @@ use std::time::Duration; pub use self::canonical_url::CanonicalUrl; pub use self::config::{homedir, Config, ConfigValue}; +pub(crate) use self::counter::MetricsCounter; pub use self::dependency_queue::DependencyQueue; pub use self::diagnostic_server::RustfixDiagnosticServer; pub use self::errors::{internal, CargoResult, CliResult, Test}; @@ -29,6 +30,7 @@ pub use self::workspace::{ mod canonical_url; pub mod command_prelude; pub mod config; +mod counter; pub mod cpu; mod dependency_queue; pub mod diagnostic_server; From de84f352e42f58b4dff43ead13e8412c8923e685 Mon Sep 17 00:00:00 2001 From: Weihang Lo Date: Tue, 27 Apr 2021 13:48:36 +0800 Subject: [PATCH 014/129] refactor: use metrics counter as an abstaction --- src/cargo/sources/git/utils.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/cargo/sources/git/utils.rs b/src/cargo/sources/git/utils.rs index e2462528d1a..de2c7f70dbb 100644 --- a/src/cargo/sources/git/utils.rs +++ b/src/cargo/sources/git/utils.rs @@ -3,7 +3,7 @@ use crate::core::GitReference; use crate::util::errors::CargoResult; -use crate::util::{network, Config, IntoUrl, Progress}; +use crate::util::{network, Config, IntoUrl, MetricsCounter, Progress}; use anyhow::{anyhow, Context as _}; use cargo_util::{paths, ProcessBuilder}; use curl::easy::List; @@ -695,9 +695,9 @@ pub fn with_fetch_options( let mut progress = Progress::new("Fetch", config); network::with_retry(config, || { with_authentication(url, git_config, |f| { - let mut last_recv = 0; // in Byte let mut last_update = Instant::now(); let mut rcb = git2::RemoteCallbacks::new(); + let mut counter = MetricsCounter::<10>::new(0); rcb.credentials(f); rcb.transfer_progress(|stats| { let indexed_deltas = stats.indexed_deltas(); @@ -711,10 +711,8 @@ pub fn with_fetch_options( } else { // Receiving objects. let duration = last_update.elapsed(); - let recv = stats.received_bytes(); - let rate = (recv - last_recv) as f32 / duration.as_secs_f32(); - if duration > Duration::from_secs(3) { - last_recv = recv; + if duration > Duration::from_millis(300) { + counter.add(stats.received_bytes()); last_update = Instant::now(); } fn format_bytes(bytes: f32) -> (&'static str, f32) { @@ -722,7 +720,7 @@ pub fn with_fetch_options( let i = (bytes.log2() / 10.0).min(4.0) as usize; (UNITS[i], bytes / 1024_f32.powi(i as i32)) } - let (unit, rate) = format_bytes(rate); + let (unit, rate) = format_bytes(counter.rate()); format!(", {:.2}{}iB/s", rate, unit) }; progress From 0847630d073f0d24fd14258bf284a619a7b5d04c Mon Sep 17 00:00:00 2001 From: Alik Aslanyan Date: Tue, 27 Apr 2021 13:09:44 +0400 Subject: [PATCH 015/129] Implement suggestions for unknown features in workspace --- Cargo.toml | 1 + src/cargo/core/workspace.rs | 363 +++++++++++++++++++++------- tests/testsuite/package_features.rs | 6 +- 3 files changed, 287 insertions(+), 83 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index eaf8e8307e8..c7f17c03039 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ clap = "2.31.2" unicode-width = "0.1.5" openssl = { version = '0.10.11', optional = true } im-rc = "15.0.0" +itertools = "0.10.0" # A noop dependency that changes in the Rust repository, it's a bit of a hack. # See the `src/tools/rustc-workspace-hack/README.md` file in `rust-lang/rust` diff --git a/src/cargo/core/workspace.rs b/src/cargo/core/workspace.rs index 9a2d5920a74..3227203520a 100644 --- a/src/cargo/core/workspace.rs +++ b/src/cargo/core/workspace.rs @@ -7,6 +7,7 @@ use std::slice; use anyhow::{bail, Context as _}; use glob::glob; +use itertools::*; use log::debug; use url::Url; @@ -20,6 +21,7 @@ use crate::ops; use crate::sources::{PathSource, CRATES_IO_INDEX, CRATES_IO_REGISTRY}; use crate::util::errors::{CargoResult, ManifestError}; use crate::util::interning::InternedString; +use crate::util::lev_distance; use crate::util::toml::{read_manifest, TomlDependency, TomlProfiles}; use crate::util::{config::ConfigRelativePath, Config, Filesystem, IntoUrl}; use cargo_util::paths; @@ -1075,89 +1077,298 @@ impl<'cfg> Workspace<'cfg> { } } - /// New command-line feature selection behavior with resolver = "2" or the - /// root of a virtual workspace. See `allows_new_cli_feature_behavior`. - fn members_with_features_new( + /// Returns the requested features for the given member. + /// This filters out any named features that the member does not have. + fn collect_matching_features( + member: &Package, + cli_features: &CliFeatures, + found_features: &mut BTreeSet, + ) -> CliFeatures { + if cli_features.features.is_empty() || cli_features.all_features { + return cli_features.clone(); + } + + // Only include features this member defines. + let summary = member.summary(); + + // Features defined in the manifest + let summary_features = summary.features(); + + // Dependency name -> dependency + let dependencies: BTreeMap = summary + .dependencies() + .iter() + .map(|dep| (dep.name_in_toml(), dep)) + .collect(); + + // Features that enable optional dependencies + let optional_dependency_names: BTreeSet<_> = dependencies + .iter() + .filter(|(_, dep)| dep.is_optional()) + .map(|(name, _)| name) + .copied() + .collect(); + + let mut features = BTreeSet::new(); + + // Checks if a member contains the given feature. + let summary_or_opt_dependency_feature = |feature: &InternedString| -> bool { + summary_features.contains_key(feature) || optional_dependency_names.contains(feature) + }; + + for feature in cli_features.features.iter() { + match feature { + FeatureValue::Feature(f) => { + if summary_or_opt_dependency_feature(f) { + // feature exists in this member. + features.insert(feature.clone()); + found_features.insert(feature.clone()); + } + } + // This should be enforced by CliFeatures. + FeatureValue::Dep { .. } + | FeatureValue::DepFeature { + dep_prefix: true, .. + } => panic!("unexpected dep: syntax {}", feature), + FeatureValue::DepFeature { + dep_name, + dep_feature, + dep_prefix: _, + weak: _, + } => { + if dependencies.contains_key(dep_name) { + // pkg/feat for a dependency. + // Will rely on the dependency resolver to validate `dep_feature`. + features.insert(feature.clone()); + found_features.insert(feature.clone()); + } else if *dep_name == member.name() + && summary_or_opt_dependency_feature(dep_feature) + { + // member/feat where "feat" is a feature in member. + // + // `weak` can be ignored here, because the member + // either is or isn't being built. + features.insert(FeatureValue::Feature(*dep_feature)); + found_features.insert(feature.clone()); + } + } + } + } + CliFeatures { + features: Rc::new(features), + all_features: false, + uses_default_features: cli_features.uses_default_features, + } + } + + fn report_unknown_features_error( &self, specs: &[PackageIdSpec], cli_features: &CliFeatures, - ) -> CargoResult> { - // Keep track of which features matched *any* member, to produce an error - // if any of them did not match anywhere. - let mut found: BTreeSet = BTreeSet::new(); + found_features: &BTreeSet, + ) -> CargoResult<()> { + // Keeps track of which features were contained in summary of `member` to suggest similar features in errors + let mut summary_features: Vec = Default::default(); - // Returns the requested features for the given member. - // This filters out any named features that the member does not have. - let mut matching_features = |member: &Package| -> CliFeatures { - if cli_features.features.is_empty() || cli_features.all_features { - return cli_features.clone(); - } + // Keeps track of `member` dependencies (`dep/feature`) and their features names to suggest similar features in error + let mut dependencies_features: BTreeMap = + Default::default(); + + // Keeps track of `member` optional dependencies names (which can be enabled with feature) to suggest similar features in error + let mut optional_dependency_names: Vec = Default::default(); + + // Keeps track of which features were contained in summary of `member` to suggest similar features in errors + let mut summary_features_per_member: BTreeMap<&Package, BTreeSet> = + Default::default(); + + // Keeps track of `member` optional dependencies (which can be enabled with feature) to suggest similar features in error + let mut optional_dependency_names_per_member: BTreeMap<&Package, BTreeSet> = + Default::default(); + + for member in self + .members() + .filter(|m| specs.iter().any(|spec| spec.matches(m.package_id()))) + { // Only include features this member defines. let summary = member.summary(); - let member_features = summary.features(); - let mut features = BTreeSet::new(); - - // Checks if a member contains the given feature. - let contains = |feature: InternedString| -> bool { - member_features.contains_key(&feature) - || summary - .dependencies() + + // Features defined in the manifest + summary_features.extend(summary.features().keys()); + summary_features_per_member + .insert(member, summary.features().keys().copied().collect()); + + // Dependency name -> dependency + let dependencies: BTreeMap = summary + .dependencies() + .iter() + .map(|dep| (dep.name_in_toml(), dep)) + .collect(); + + dependencies_features.extend( + dependencies + .iter() + .map(|(name, dep)| (*name, dep.features())), + ); + + // Features that enable optional dependencies + let optional_dependency_names_raw: BTreeSet<_> = dependencies + .iter() + .filter(|(_, dep)| dep.is_optional()) + .map(|(name, _)| name) + .copied() + .collect(); + + optional_dependency_names.extend(optional_dependency_names_raw.iter()); + optional_dependency_names_per_member.insert(member, optional_dependency_names_raw); + } + + let levenshtein_test = + |a: InternedString, b: InternedString| lev_distance(a.as_str(), b.as_str()) < 4; + + let suggestions: Vec<_> = cli_features + .features + .difference(found_features) + .map(|feature| match feature { + // Simple feature, check if any of the optional dependency features or member features are close enough + FeatureValue::Feature(typo) => { + // Finds member features which are similar to the requested feature. + let summary_features = summary_features .iter() - .any(|dep| dep.is_optional() && dep.name_in_toml() == feature) - }; + .filter(move |feature| levenshtein_test(**feature, *typo)); - for feature in cli_features.features.iter() { - match feature { - FeatureValue::Feature(f) => { - if contains(*f) { - // feature exists in this member. - features.insert(feature.clone()); - found.insert(feature.clone()); - } - } - // This should be enforced by CliFeatures. - FeatureValue::Dep { .. } - | FeatureValue::DepFeature { - dep_prefix: true, .. - } => panic!("unexpected dep: syntax {}", feature), - FeatureValue::DepFeature { - dep_name, - dep_feature, - dep_prefix: _, - weak: _, - } => { - if summary - .dependencies() - .iter() - .any(|dep| dep.name_in_toml() == *dep_name) - { - // pkg/feat for a dependency. - // Will rely on the dependency resolver to validate `dep_feature`. - features.insert(feature.clone()); - found.insert(feature.clone()); - } else if *dep_name == member.name() && contains(*dep_feature) { - // member/feat where "feat" is a feature in member. - // - // `weak` can be ignored here, because the member - // either is or isn't being built. - features.insert(FeatureValue::Feature(*dep_feature)); - found.insert(feature.clone()); - } - } + // Finds optional dependencies which name is similar to the feature + let optional_dependency_features = optional_dependency_names + .iter() + .filter(move |feature| levenshtein_test(**feature, *typo)); + + summary_features + .chain(optional_dependency_features) + .map(|s| s.to_string()) + .collect::>() } - } - CliFeatures { - features: Rc::new(features), - all_features: false, - uses_default_features: cli_features.uses_default_features, - } - }; + FeatureValue::Dep { .. } + | FeatureValue::DepFeature { + dep_prefix: true, .. + } => panic!("unexpected dep: syntax {}", feature), + FeatureValue::DepFeature { + dep_name, + dep_feature, + dep_prefix: _, + weak: _, + } => { + // Finds set of `pkg/feat` that are very similar to current `pkg/feat`. + let pkg_feat_similar = dependencies_features + .iter() + .filter(|(name, _)| levenshtein_test(**name, *dep_name)) + .map(|(name, features)| { + ( + name, + features + .iter() + .filter(|feature| levenshtein_test(**feature, *dep_feature)) + .collect::>(), + ) + }) + .map(|(name, features)| { + features + .into_iter() + .map(move |feature| format!("{}/{}", name, feature)) + }) + .flatten(); + + // Finds set of `member/optional_dep` features which name is similar to current `pkg/feat`. + let optional_dependency_features = optional_dependency_names_per_member + .iter() + .filter(|(package, _)| levenshtein_test(package.name(), *dep_name)) + .map(|(package, optional_dependencies)| { + optional_dependencies + .into_iter() + .filter(|optional_dependency| { + levenshtein_test(**optional_dependency, *dep_name) + }) + .map(move |optional_dependency| { + format!("{}/{}", package.name(), optional_dependency) + }) + }) + .flatten(); + + // Finds set of `member/feat` features which name is similar to current `pkg/feat`. + let summary_features = summary_features_per_member + .iter() + .filter(|(package, _)| levenshtein_test(package.name(), *dep_name)) + .map(|(package, summary_features)| { + summary_features + .into_iter() + .filter(|summary_feature| { + levenshtein_test(**summary_feature, *dep_name) + }) + .map(move |summary_feature| { + format!("{}/{}", package.name(), summary_feature) + }) + }) + .flatten(); + + pkg_feat_similar + .chain(optional_dependency_features) + .chain(summary_features) + .collect::>() + } + }) + .map(|v| v.into_iter()) + .flatten() + .unique() + .filter(|element| { + !cli_features + .features + .contains(&FeatureValue::new(InternedString::new(element))) + }) + .sorted() + .take(5) + .collect(); + + let unknown: Vec<_> = cli_features + .features + .difference(found_features) + .map(|feature| feature.to_string()) + .sorted() + .collect(); + + if suggestions.is_empty() { + bail!( + "none of the selected packages contains these features: {}", + unknown.join(", ") + ); + } else { + bail!( + "none of the selected packages contains these features: {}, did you mean: {}?", + unknown.join(", "), + suggestions.join(", ") + ); + } + } + + /// New command-line feature selection behavior with resolver = "2" or the + /// root of a virtual workspace. See `allows_new_cli_feature_behavior`. + fn members_with_features_new( + &self, + specs: &[PackageIdSpec], + cli_features: &CliFeatures, + ) -> CargoResult> { + // Keeps track of which features matched `member` to produce an error + // if any of them did not match anywhere. + let mut found_features = Default::default(); let members: Vec<(&Package, CliFeatures)> = self .members() .filter(|m| specs.iter().any(|spec| spec.matches(m.package_id()))) - .map(|m| (m, matching_features(m))) + .map(|m| { + ( + m, + Workspace::collect_matching_features(m, cli_features, &mut found_features), + ) + }) .collect(); + if members.is_empty() { // `cargo build -p foo`, where `foo` is not a member. // Do not allow any command-line flags (defaults only). @@ -1174,18 +1385,8 @@ impl<'cfg> Workspace<'cfg> { .map(|m| (m, CliFeatures::new_all(false))) .collect()); } - if *cli_features.features != found { - let mut missing: Vec<_> = cli_features - .features - .difference(&found) - .map(|fv| fv.to_string()) - .collect(); - missing.sort(); - // TODO: typo suggestions would be good here. - bail!( - "none of the selected packages contains these features: {}", - missing.join(", ") - ); + if *cli_features.features != found_features { + self.report_unknown_features_error(specs, cli_features, &found_features)?; } Ok(members) } diff --git a/tests/testsuite/package_features.rs b/tests/testsuite/package_features.rs index 957ed921cf3..e7605581b35 100644 --- a/tests/testsuite/package_features.rs +++ b/tests/testsuite/package_features.rs @@ -67,13 +67,15 @@ fn virtual_no_default_features() { p.cargo("check --features foo") .masquerade_as_nightly_cargo() .with_status(101) - .with_stderr("[ERROR] none of the selected packages contains these features: foo") + .with_stderr( + "[ERROR] none of the selected packages contains these features: foo, did you mean: f1?", + ) .run(); p.cargo("check --features a/dep1,b/f1,b/f2,f2") .masquerade_as_nightly_cargo() .with_status(101) - .with_stderr("[ERROR] none of the selected packages contains these features: b/f2, f2") + .with_stderr("[ERROR] none of the selected packages contains these features: b/f2, f2, did you mean: b/f1, f1?") .run(); } From 3786b27367e339c514cc387d3f878078f9d191ba Mon Sep 17 00:00:00 2001 From: Alik Aslanyan Date: Tue, 27 Apr 2021 09:36:07 +0000 Subject: [PATCH 016/129] Update test --- tests/testsuite/package_features.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testsuite/package_features.rs b/tests/testsuite/package_features.rs index e7605581b35..83c217f0fce 100644 --- a/tests/testsuite/package_features.rs +++ b/tests/testsuite/package_features.rs @@ -75,7 +75,7 @@ fn virtual_no_default_features() { p.cargo("check --features a/dep1,b/f1,b/f2,f2") .masquerade_as_nightly_cargo() .with_status(101) - .with_stderr("[ERROR] none of the selected packages contains these features: b/f2, f2, did you mean: b/f1, f1?") + .with_stderr("[ERROR] none of the selected packages contains these features: b/f2, f2, did you mean: f1?") .run(); } From 0d7fe77eff7333474886af0a9979843796516d0b Mon Sep 17 00:00:00 2001 From: Alik Aslanyan Date: Tue, 27 Apr 2021 14:49:19 +0400 Subject: [PATCH 017/129] Fix test case --- src/cargo/core/workspace.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/cargo/core/workspace.rs b/src/cargo/core/workspace.rs index 3227203520a..234e7e8d949 100644 --- a/src/cargo/core/workspace.rs +++ b/src/cargo/core/workspace.rs @@ -1318,9 +1318,8 @@ impl<'cfg> Workspace<'cfg> { .flatten() .unique() .filter(|element| { - !cli_features - .features - .contains(&FeatureValue::new(InternedString::new(element))) + let feature = FeatureValue::new(InternedString::new(element)); + !cli_features.features.contains(&feature) && !found_features.contains(&feature) }) .sorted() .take(5) From fc5840d17534559b5155a70667b70a7b1431cf1e Mon Sep 17 00:00:00 2001 From: Jade Date: Tue, 27 Apr 2021 04:25:46 -0700 Subject: [PATCH 018/129] Fix dep-info files emitting paths relative to deps' roots Sample `shoo.d` file prior to this change is below, note the `build.rs` at the end, which was not from my package. From booping the debugger, I found this was coming from `compiler_builtins`. This is not really their bug though: if a build.rs asks for rerun-if-changed on some crate relative path, this will happen in general. So I've fixed it in Cargo and added a test to prevent it regressing. ``` target/riscv64imac-mu-shoo-elf/release/shoo: /home/jade/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/../../stdarch/crates/core_arch/src/core_arch_docs.md /home/jade/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/../../stdarch/crates/core_arch/src/macros.rs /home/jade/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/../../stdarch/crates/core_arch/src/mod.rs /home/jade/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/../../stdarch/crates/core_arch/src/simd.rs /home/jade/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/../../stdarch/crates/core_arch/src/simd_llvm.rs crates/build_bits/src/lib.rs shoo/src/main.rs shoo/src/task.rs shoo/src/vectors.s build.rs ``` This change fixes it so it's like: ``` target/riscv64imac-mu-shoo-elf/release/shoo: /home/jade/.cargo/registry/src/github.com-1ecc6299db9ec823/compiler_builtins-0.1.39/build.rs /home/jade/.cargo/registry/src/github.com-1ecc6299db9ec823/log-0.4.14/build.rs /home/jade/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/../../stdarch/crates/core_arch/src/core_arch_docs.md /home/jade/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/../../stdarch/crates/core_arch/src/macros.rs /home/jade/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/../../stdarch/crates/core_arch/src/mod.rs /home/jade/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/../../stdarch/crates/core_arch/src/simd.rs /home/jade/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/../../stdarch/crates/core_arch/src/simd_llvm.rs crates/build_bits/src/lib.rs shoo/src/main.rs shoo/src/task.rs shoo/src/vectors.s ``` --- src/cargo/core/compiler/output_depinfo.rs | 3 + tests/testsuite/build_script.rs | 95 ++++++++++++++++++++++- 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/src/cargo/core/compiler/output_depinfo.rs b/src/cargo/core/compiler/output_depinfo.rs index 8804c7ba1dc..b55dd8fa530 100644 --- a/src/cargo/core/compiler/output_depinfo.rs +++ b/src/cargo/core/compiler/output_depinfo.rs @@ -81,6 +81,9 @@ fn add_deps_for_unit( if let Some(metadata) = cx.find_build_script_metadata(unit) { if let Some(output) = cx.build_script_outputs.lock().unwrap().get(metadata) { for path in &output.rerun_if_changed { + // The paths we have saved from the unit are of arbitrary relativeness and may be + // relative to the crate root of the dependency. + let path = unit.pkg.root().join(path); deps.insert(path.into()); } } diff --git a/tests/testsuite/build_script.rs b/tests/testsuite/build_script.rs index a43856548b0..29082d96141 100644 --- a/tests/testsuite/build_script.rs +++ b/tests/testsuite/build_script.rs @@ -1,8 +1,9 @@ //! Tests for build.rs scripts. -use cargo_test_support::paths::CargoPathExt; +use cargo_test_support::project_in_home; use cargo_test_support::registry::Package; use cargo_test_support::{basic_manifest, cross_compile, is_coarse_mtime, project}; +use cargo_test_support::{lines_match, paths::CargoPathExt}; use cargo_test_support::{rustc_host, sleep_ms, slow_cpu_multiplier, symlink_supported}; use cargo_util::paths::remove_dir_all; use std::env; @@ -2602,6 +2603,98 @@ fn fresh_builds_possible_with_multiple_metadata_overrides() { .run(); } +#[cargo_test] +fn generate_good_d_files() { + // this is here to stop regression on an issue where build.rs rerun-if-changed paths aren't + // made absolute properly, which in turn interacts poorly with the dep-info-basedir setting, + // and the dep-info files have other-crate-relative paths spat out in them + let dep = project_in_home("awoo") + .file( + "Cargo.toml", + r#" + [project] + name = "awoo" + version = "0.5.0" + build = "build.rs" + "#, + ) + .file("src/lib.rs", "") + .file( + "build.rs", + r#" + fn main() { + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-changed=barkbarkbark"); + } + "#, + ) + .build(); + + let p = project_in_home("meow") + .file( + "Cargo.toml", + &format!( + r#" + [project] + name = "meow" + version = "0.5.0" + [dependencies] + awoo = {{ path = "{path}" }} + "#, + path = dep.root().to_str().unwrap(), + ), + ) + .file("src/main.rs", "fn main() {}") + .build(); + + p.cargo("build -v").run(); + + let dot_d_path = p.bin("meow").with_extension("d"); + println!("*meow at* {:?}", dot_d_path); + let dot_d = fs::read_to_string(&dot_d_path).unwrap(); + + println!("*.d file content*: {}", &dot_d); + + assert!( + lines_match( + "[..]/target/debug/meow: [..]/awoo/barkbarkbark [..]/awoo/build.rs[..]", + &dot_d + ) || lines_match( + "[..]/target/debug/meow: [..]/awoo/build.rs [..]/awoo/barkbarkbark[..]", + &dot_d + ) + ); + + // paths relative to dependency roots should not be allowed + assert!(!dot_d.split_whitespace().any(|v| v == "build.rs")); + + p.change_file( + ".cargo/config.toml", + r#" + [build] + dep-info-basedir="." + "#, + ); + p.cargo("build -v").run(); + + let dot_d = fs::read_to_string(&dot_d_path).unwrap(); + + println!("*.d file content with dep-info-basedir*: {}", &dot_d); + + assert!( + lines_match( + "target/debug/meow: [..]/awoo/barkbarkbark [..]/awoo/build.rs[..]", + &dot_d + ) || lines_match( + "target/debug/meow: [..]/awoo/build.rs [..]/awoo/barkbarkbark[..]", + &dot_d + ) + ); + + // paths relative to dependency roots should not be allowed + assert!(!dot_d.split_whitespace().any(|v| v == "build.rs")); +} + #[cargo_test] fn rebuild_only_on_explicit_paths() { let p = project() From a89b1e8a9f4ebc3490fd518bd8488929275f083c Mon Sep 17 00:00:00 2001 From: Weihang Lo Date: Tue, 27 Apr 2021 22:46:46 +0800 Subject: [PATCH 019/129] refactor: pass Instant to MetricsCounter --- src/cargo/sources/git/utils.rs | 13 ++++++++----- src/cargo/util/counter.rs | 26 ++++++++++++++++---------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/cargo/sources/git/utils.rs b/src/cargo/sources/git/utils.rs index de2c7f70dbb..54ff436a19f 100644 --- a/src/cargo/sources/git/utils.rs +++ b/src/cargo/sources/git/utils.rs @@ -697,7 +697,9 @@ pub fn with_fetch_options( with_authentication(url, git_config, |f| { let mut last_update = Instant::now(); let mut rcb = git2::RemoteCallbacks::new(); - let mut counter = MetricsCounter::<10>::new(0); + // We choose `N=10` here to make a `300ms * 10slots ~= 3000ms` + // sliding window for tracking the data transfer rate (in bytes/s). + let mut counter = MetricsCounter::<10>::new(0, last_update); rcb.credentials(f); rcb.transfer_progress(|stats| { let indexed_deltas = stats.indexed_deltas(); @@ -710,10 +712,11 @@ pub fn with_fetch_options( ) } else { // Receiving objects. - let duration = last_update.elapsed(); - if duration > Duration::from_millis(300) { - counter.add(stats.received_bytes()); - last_update = Instant::now(); + let now = Instant::now(); + // Scrape a `received_bytes` to the counter every 300ms. + if now - last_update > Duration::from_millis(300) { + counter.add(stats.received_bytes(), now); + last_update = now; } fn format_bytes(bytes: f32) -> (&'static str, f32) { static UNITS: [&str; 5] = ["", "K", "M", "G", "T"]; diff --git a/src/cargo/util/counter.rs b/src/cargo/util/counter.rs index 04dd1647006..26132afbaad 100644 --- a/src/cargo/util/counter.rs +++ b/src/cargo/util/counter.rs @@ -11,17 +11,17 @@ pub struct MetricsCounter { impl MetricsCounter { /// Creates a new counter with an initial value. - pub fn new(init: usize) -> Self { + pub fn new(init: usize, init_at: Instant) -> Self { debug_assert!(N > 0, "number of slots must be greater than zero"); Self { - slots: [(init, Instant::now()); N], + slots: [(init, init_at); N], index: 0, } } /// Adds record to the counter. - pub fn add(&mut self, data: usize) { - self.slots[self.index] = (data, Instant::now()); + pub fn add(&mut self, data: usize, added_at: Instant) { + self.slots[self.index] = (data, added_at); self.index = (self.index + 1) % N; } @@ -42,20 +42,26 @@ impl MetricsCounter { #[cfg(test)] mod tests { use super::MetricsCounter; + use std::time::{Duration, Instant}; #[test] fn counter() { - let mut counter = MetricsCounter::<3>::new(0); + let now = Instant::now(); + let mut counter = MetricsCounter::<3>::new(0, now); assert_eq!(counter.rate(), 0f32); - for i in 1..=5 { - counter.add(i); - assert!(counter.rate() > 0f32); - } + counter.add(1, now + Duration::from_secs(1)); + assert_eq!(counter.rate(), 1f32); + counter.add(4, now + Duration::from_secs(2)); + assert_eq!(counter.rate(), 2f32); + counter.add(7, now + Duration::from_secs(3)); + assert_eq!(counter.rate(), 3f32); + counter.add(12, now + Duration::from_secs(4)); + assert_eq!(counter.rate(), 4f32); } #[test] #[should_panic(expected = "number of slots must be greater than zero")] fn counter_zero_slot() { - let _counter = MetricsCounter::<0>::new(0); + let _counter = MetricsCounter::<0>::new(0, Instant::now()); } } From 5bba21afe40312dbdba9122430642f4c28611c3b Mon Sep 17 00:00:00 2001 From: Jade Date: Tue, 27 Apr 2021 21:21:56 -0700 Subject: [PATCH 020/129] Fix the test on Windows --- tests/testsuite/build_script.rs | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/tests/testsuite/build_script.rs b/tests/testsuite/build_script.rs index 29082d96141..0c216bf217b 100644 --- a/tests/testsuite/build_script.rs +++ b/tests/testsuite/build_script.rs @@ -1,6 +1,5 @@ //! Tests for build.rs scripts. -use cargo_test_support::project_in_home; use cargo_test_support::registry::Package; use cargo_test_support::{basic_manifest, cross_compile, is_coarse_mtime, project}; use cargo_test_support::{lines_match, paths::CargoPathExt}; @@ -2608,9 +2607,9 @@ fn generate_good_d_files() { // this is here to stop regression on an issue where build.rs rerun-if-changed paths aren't // made absolute properly, which in turn interacts poorly with the dep-info-basedir setting, // and the dep-info files have other-crate-relative paths spat out in them - let dep = project_in_home("awoo") + let p = project() .file( - "Cargo.toml", + "awoo/Cargo.toml", r#" [project] name = "awoo" @@ -2618,9 +2617,9 @@ fn generate_good_d_files() { build = "build.rs" "#, ) - .file("src/lib.rs", "") + .file("awoo/src/lib.rs", "") .file( - "build.rs", + "awoo/build.rs", r#" fn main() { println!("cargo:rerun-if-changed=build.rs"); @@ -2628,21 +2627,15 @@ fn generate_good_d_files() { } "#, ) - .build(); - - let p = project_in_home("meow") .file( "Cargo.toml", - &format!( - r#" + r#" [project] name = "meow" version = "0.5.0" [dependencies] - awoo = {{ path = "{path}" }} + awoo = { path = "awoo" } "#, - path = dep.root().to_str().unwrap(), - ), ) .file("src/main.rs", "fn main() {}") .build(); @@ -2666,7 +2659,9 @@ fn generate_good_d_files() { ); // paths relative to dependency roots should not be allowed - assert!(!dot_d.split_whitespace().any(|v| v == "build.rs")); + assert!(!dot_d + .split_whitespace() + .any(|v| v == "barkbarkbark" || v == "build.rs")); p.change_file( ".cargo/config.toml", @@ -2683,16 +2678,18 @@ fn generate_good_d_files() { assert!( lines_match( - "target/debug/meow: [..]/awoo/barkbarkbark [..]/awoo/build.rs[..]", + "target/debug/meow: [..]awoo/barkbarkbark [..]awoo/build.rs[..]", &dot_d ) || lines_match( - "target/debug/meow: [..]/awoo/build.rs [..]/awoo/barkbarkbark[..]", + "target/debug/meow: [..]awoo/build.rs [..]awoo/barkbarkbark[..]", &dot_d ) ); // paths relative to dependency roots should not be allowed - assert!(!dot_d.split_whitespace().any(|v| v == "build.rs")); + assert!(!dot_d + .split_whitespace() + .any(|v| v == "barkbarkbark" || v == "build.rs")); } #[cargo_test] From b998364e6b6070efac4493bfc80a07871d896b50 Mon Sep 17 00:00:00 2001 From: Jade Date: Tue, 27 Apr 2021 22:30:35 -0700 Subject: [PATCH 021/129] Fix test on Windows: reprise --- tests/testsuite/build_script.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/testsuite/build_script.rs b/tests/testsuite/build_script.rs index 0c216bf217b..0f99f73c2b9 100644 --- a/tests/testsuite/build_script.rs +++ b/tests/testsuite/build_script.rs @@ -2648,6 +2648,17 @@ fn generate_good_d_files() { println!("*.d file content*: {}", &dot_d); + #[cfg(windows)] + assert!( + lines_match( + "[..]\\target\\debug\\meow.exe: [..]\\awoo\\barkbarkbark [..]\\awoo\\build.rs[..]", + &dot_d + ) || lines_match( + "[..]\\target\\debug\\meow.exe: [..]\\awoo\\build.rs [..]\\awoo\\barkbarkbark[..]", + &dot_d + ) + ); + #[cfg(not(windows))] assert!( lines_match( "[..]/target/debug/meow: [..]/awoo/barkbarkbark [..]/awoo/build.rs[..]", @@ -2676,6 +2687,17 @@ fn generate_good_d_files() { println!("*.d file content with dep-info-basedir*: {}", &dot_d); + #[cfg(windows)] + assert!( + lines_match( + "target\\debug\\meow.exe: [..]awoo\\barkbarkbark [..]awoo\\build.rs[..]", + &dot_d + ) || lines_match( + "target\\debug\\meow.exe: [..]awoo\\build.rs [..]awoo\\barkbarkbark[..]", + &dot_d + ) + ); + #[cfg(not(windows))] assert!( lines_match( "target/debug/meow: [..]awoo/barkbarkbark [..]awoo/build.rs[..]", From e5ab39145f18bf69f8d684c849a22c8aa97d61c1 Mon Sep 17 00:00:00 2001 From: Yuki Okushi Date: Wed, 28 Apr 2021 17:59:57 +0900 Subject: [PATCH 022/129] Only deny the `unused_mut` lint --- tests/testsuite/fix.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testsuite/fix.rs b/tests/testsuite/fix.rs index e40e6a04f88..e6a733a6539 100644 --- a/tests/testsuite/fix.rs +++ b/tests/testsuite/fix.rs @@ -553,7 +553,7 @@ fn fix_deny_warnings_but_not_others() { .file( "src/lib.rs", " - #![deny(warnings)] + #![deny(unused_mut)] pub fn foo() -> u32 { let mut x = 3; From 9df531b223e169c6f4b46a7188ca5cae341a5e03 Mon Sep 17 00:00:00 2001 From: Weihang Lo Date: Thu, 29 Apr 2021 00:58:58 +0800 Subject: [PATCH 023/129] fix: bytes per second should not prefix with `i` --- src/cargo/sources/git/utils.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cargo/sources/git/utils.rs b/src/cargo/sources/git/utils.rs index 54ff436a19f..c72fb8e1dac 100644 --- a/src/cargo/sources/git/utils.rs +++ b/src/cargo/sources/git/utils.rs @@ -719,12 +719,12 @@ pub fn with_fetch_options( last_update = now; } fn format_bytes(bytes: f32) -> (&'static str, f32) { - static UNITS: [&str; 5] = ["", "K", "M", "G", "T"]; + static UNITS: [&str; 5] = ["", "Ki", "Mi", "Gi", "Ti"]; let i = (bytes.log2() / 10.0).min(4.0) as usize; (UNITS[i], bytes / 1024_f32.powi(i as i32)) } let (unit, rate) = format_bytes(counter.rate()); - format!(", {:.2}{}iB/s", rate, unit) + format!(", {:.2}{}B/s", rate, unit) }; progress .tick(stats.indexed_objects(), stats.total_objects(), &msg) From e4d4347223cfef58b7a2820c97dfcaebfe800535 Mon Sep 17 00:00:00 2001 From: Weihang Lo Date: Thu, 29 Apr 2021 01:00:14 +0800 Subject: [PATCH 024/129] comment about caveat of current transfer rate refresh --- src/cargo/sources/git/utils.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/cargo/sources/git/utils.rs b/src/cargo/sources/git/utils.rs index c72fb8e1dac..434fc35300e 100644 --- a/src/cargo/sources/git/utils.rs +++ b/src/cargo/sources/git/utils.rs @@ -712,6 +712,16 @@ pub fn with_fetch_options( ) } else { // Receiving objects. + // + // # Caveat + // + // Progress bar relies on git2 calling `transfer_progress` + // to update its transfer rate, but we cannot guarantee a + // periodic call of that callback. Thus if we don't receive + // any data for, say, 10 seconds, the rate will get stuck + // and never go down to 0B/s. + // In the future, we need to find away to update the rate + // even when the callback is not called. let now = Instant::now(); // Scrape a `received_bytes` to the counter every 300ms. if now - last_update > Duration::from_millis(300) { From 55f7d93e88dc6575ada9ced111b330d5191c7e7f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 28 Apr 2021 21:38:01 +0000 Subject: [PATCH 025/129] Upgrade to GitHub-native Dependabot --- .github/dependabot.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000000..913586440c2 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ +version: 2 +updates: +- package-ecosystem: cargo + directory: "/" + schedule: + interval: daily + open-pull-requests-limit: 10 + reviewers: + - alexcrichton + assignees: + - alexcrichton + ignore: + - dependency-name: proptest + versions: + - ">= 0.9.a, < 0.10" From 06fa940b470228d65d29325b1936935012631c20 Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Wed, 28 Apr 2021 14:39:55 -0700 Subject: [PATCH 026/129] Add missing tracking issues and unstable docs. --- src/doc/src/reference/unstable.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/doc/src/reference/unstable.md b/src/doc/src/reference/unstable.md index 8e7927f8308..bbf39e40464 100644 --- a/src/doc/src/reference/unstable.md +++ b/src/doc/src/reference/unstable.md @@ -85,6 +85,7 @@ to any Rust tools that cargo ends up calling (like `rustc` or Cargo _or_ Rust features can be used. ### extra-link-arg +* Tracking Issue: [#9426](https://github.com/rust-lang/cargo/issues/9426) * Original Pull Request: [#7811](https://github.com/rust-lang/cargo/pull/7811) The `-Z extra-link-arg` flag makes the following two instructions available @@ -138,7 +139,7 @@ invocations of nightly cargo. (the config flag is ignored by stable) ### avoid-dev-deps * Original Issue: [#4988](https://github.com/rust-lang/cargo/issues/4988) -* Stabilization Issue: [#5133](https://github.com/rust-lang/cargo/issues/5133) +* Tracking Issue: [#5133](https://github.com/rust-lang/cargo/issues/5133) When running commands such as `cargo install` or `cargo build`, Cargo currently requires dev-dependencies to be downloaded, even if they are not @@ -569,6 +570,8 @@ itself, which has implicit dependencies on the standard library that would otherwise be untracked for change-detection. ### panic-abort-tests +* Tracking Issue: [#67650](https://github.com/rust-lang/rust/issues/67650) +* Original Pull Request: [#7460](https://github.com/rust-lang/cargo/pull/7460) The `-Z panic-abort-tests` flag will enable nightly support to compile test harness crates with `-Cpanic=abort`. Without this flag Cargo will compile tests, @@ -839,6 +842,9 @@ The default value is `"remote"`. The value may also take a URL for a custom location. ### terminal-width + +* Tracking Issue: [#84673](https://github.com/rust-lang/rust/issues/84673) + This feature provides a new flag, `-Z terminal-width`, which is used to pass a terminal width to `rustc` so that error messages containing long lines can be intelligently truncated. @@ -896,6 +902,9 @@ dependency. However, unlike the normal `serde/std` syntax, it will not enable the optional dependency `serde` unless something else has included it. ### per-package-target +* Tracking Issue: [#9406](https://github.com/rust-lang/cargo/pull/9406) +* Original Pull Request: [#9030](https://github.com/rust-lang/cargo/pull/9030) +* Original Issue: [#7004](https://github.com/rust-lang/cargo/pull/7004) The `per-package-target` feature adds two keys to the manifest: `package.default-target` and `package.forced-target`. The first makes @@ -1212,6 +1221,23 @@ cargo +nightly -Zunstable-options config get build.rustflags If no config value is included, it will display all config values. See the `--help` output for more options available. +### `doctest-in-workspace` + +* Tracking Issue: [#9427](https://github.com/rust-lang/cargo/issues/9427) + +The `-Z doctest-in-workspace` flag changes the behavior of the current working +directory used when running doctests. Historically, Cargo has run `rustdoc +--test` relative to the root of the package, with paths relative from that +root. However, this is inconsistent with how `rustc` and `rustdoc` are +normally run in a workspace, where they are run relative to the workspace +root. This inconsistency causes problems in various ways, such as when passing +RUSTDOCFLAGS with relative paths, or dealing with diagnostic output. + +The `-Z doctest-in-workspace` flag causes cargo to switch to running `rustdoc` +from the root of the workspace. It also passes the `--test-run-directory` to +`rustdoc` so that when *running* the tests, they are run from the root of the +package. This preserves backwards compatibility and is consistent with how +normal unittests are run.