diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 02b0aa6e..f4007ccf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -133,19 +133,6 @@ jobs: - name: cargo publish run: cargo publish --dry-run - msrv-check: - name: Minimum Stable Rust Version Check - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 - with: - toolchain: "1.56.1" - override: true - - run: cargo fetch - - name: cargo check - run: cargo check --all-targets - release: name: Release #needs: [test, self, doc-book] diff --git a/Cargo.lock b/Cargo.lock index eca3ba0e..38b51c1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -118,6 +118,12 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "bytes" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" + [[package]] name = "bytesize" version = "1.1.0" @@ -135,9 +141,9 @@ dependencies = [ [[package]] name = "cargo" -version = "0.60.0" +version = "0.61.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc194fab2f0394703f2794faeb9fcca34301af33eee96fa943b856028f279a77" +checksum = "f76f22dfcbc8e5aaa4e150373354723efe22b6b2280805f1fb6b1363005e7bab" dependencies = [ "anyhow", "atty", @@ -171,6 +177,7 @@ dependencies = [ "memchr", "num_cpus", "opener", + "openssl", "os_info", "percent-encoding", "rustc-workspace-hack", @@ -184,7 +191,7 @@ dependencies = [ "tar", "tempfile", "termcolor", - "toml", + "toml_edit", "unicode-width", "unicode-xid", "url", @@ -202,16 +209,16 @@ dependencies = [ "atty", "bitvec", "cargo", + "clap", "codespan", "codespan-reporting", + "crates-index", "crossbeam", "fern", "git2", "home", "krates", - "lazy_static", "log", - "memchr", "rayon", "rustsec", "semver", @@ -219,7 +226,6 @@ dependencies = [ "serde_json", "smallvec", "spdx", - "structopt", "tempfile", "time", "toml", @@ -310,17 +316,41 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "2.34.0" +version = "3.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +checksum = "7c167e37342afc5f33fd87bbc870cedd020d2a6dffa05d45ccd9241fbdd146db" dependencies = [ - "ansi_term", "atty", "bitflags", + "clap_derive", + "clap_lex", + "indexmap", + "lazy_static", "strsim", + "termcolor", "textwrap", - "unicode-width", - "vec_map", +] + +[[package]] +name = "clap_derive" +version = "3.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3aab4734e083b809aaf5794e14e756d1c798d2c69c7f7de7a09a2f5214993c1" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "189ddd3b5d32a70b35e7686054371742a937b0d99128e76dde6340210e966669" +dependencies = [ + "os_str_bytes", ] [[package]] @@ -342,6 +372,16 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "combine" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a604e93b79d1808327a6fca85a6f2d69de66461e7620f5a4cbf5fb4d1d7c948" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "commoncrypto" version = "0.2.0" @@ -378,15 +418,17 @@ checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "crates-index" -version = "0.17.0" +version = "0.18.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ad4af5c8dd9940a497ef4473e6e558b660a4a1b6e5ce2cb9d85454e2aaaf947" +checksum = "0044896374c388ccbf1497dad6384bf6111dbcad9d7069506df7450ce9b62ea3" dependencies = [ "git2", - "glob", "hex 0.4.3", "home", "memchr", + "num_cpus", + "rayon", + "rustc-hash", "semver", "serde", "serde_derive", @@ -396,9 +438,9 @@ dependencies = [ [[package]] name = "crates-io" -version = "0.33.1" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2d7714dc2b336c5a579a1a2aa2d41c7cd7a31ccb25e2ea908dba8934cfeb75a" +checksum = "6b4a87459133b2e708195eaab34be55039bc30e0d120658bd40794bb00b6328d" dependencies = [ "anyhow", "curl", @@ -661,9 +703,9 @@ dependencies = [ [[package]] name = "git2" -version = "0.13.25" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29229cc1b24c0e6062f6e742aa3e256492a5323365e5ed3413599f8a5eff7d6" +checksum = "d0155506aab710a86160ddb504a480d2964d7ab5b9e62419be69e0032bc5931c" dependencies = [ "bitflags", "libc", @@ -676,9 +718,9 @@ dependencies = [ [[package]] name = "git2-curl" -version = "0.14.1" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "883539cb0ea94bab3f8371a98cd8e937bbe9ee7c044499184aa4c17deb643a50" +checksum = "1ee51709364c341fbb6fe2a385a290fb9196753bdde2fc45447d27cd31b11b13" dependencies = [ "curl", "git2", @@ -713,12 +755,9 @@ checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" [[package]] name = "heck" -version = "0.3.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" -dependencies = [ - "unicode-segmentation", -] +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" [[package]] name = "hermit-abi" @@ -759,16 +798,6 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" -[[package]] -name = "humantime-serde" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a3db5ea5923d99402c94e9feb261dc5ee9b4efa158b0315f788cf549cc200c" -dependencies = [ - "humantime", - "serde", -] - [[package]] name = "idna" version = "0.2.3" @@ -867,6 +896,15 @@ dependencies = [ "semver", ] +[[package]] +name = "kstring" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b310ccceade8121d7d77fee406160e457c2f4e7c7982d589da3499bc7ea4526" +dependencies = [ + "serde", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -887,9 +925,9 @@ checksum = "ec647867e2bf0772e28c8bcde4f0d19a9216916e890543b5a03ed8ef27b8f259" [[package]] name = "libgit2-sys" -version = "0.12.26+1.3.0" +version = "0.13.4+1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e1c899248e606fbfe68dcb31d8b0176ebab833b103824af31bddf4b7457494" +checksum = "d0fa6563431ede25f5cc7f6d803c6afbc1c5d3ad3d4925d12c882bf2b526f5d1" dependencies = [ "cc", "libc", @@ -1082,6 +1120,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "os_str_bytes" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64" + [[package]] name = "percent-encoding" version = "2.1.0" @@ -1263,6 +1307,12 @@ dependencies = [ "serde", ] +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-workspace-hack" version = "1.0.0" @@ -1288,13 +1338,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6136976fbabcd3ca37a12ae8ecc3408e8d7a94916d1cabdabd86aa4464e0887" dependencies = [ "cargo-lock", - "crates-index", "cvss", "fs-err", - "git2", - "home", - "humantime", - "humantime-serde", "platforms", "semver", "serde", @@ -1407,12 +1452,14 @@ checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" [[package]] name = "smartstring" -version = "0.2.10" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e714dff2b33f2321fdcd475b71cec79781a692d846f37f415fb395a1d2bcd48e" +checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" dependencies = [ + "autocfg", "serde", "static_assertions", + "version_check", ] [[package]] @@ -1451,33 +1498,9 @@ dependencies = [ [[package]] name = "strsim" -version = "0.8.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" - -[[package]] -name = "structopt" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" -dependencies = [ - "clap", - "lazy_static", - "structopt-derive", -] - -[[package]] -name = "structopt-derive" -version = "0.4.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" -dependencies = [ - "heck", - "proc-macro-error", - "proc-macro2", - "quote", - "syn", -] +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" @@ -1537,12 +1560,9 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.11.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -dependencies = [ - "unicode-width", -] +checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" [[package]] name = "thiserror" @@ -1615,6 +1635,19 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_edit" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744e9ed5b352340aa47ce033716991b5589e23781acb97cad37d4ea70560f55b" +dependencies = [ + "combine", + "indexmap", + "itertools", + "kstring", + "serde", +] + [[package]] name = "twox-hash" version = "1.6.2" @@ -1646,12 +1679,6 @@ dependencies = [ "tinyvec", ] -[[package]] -name = "unicode-segmentation" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" - [[package]] name = "unicode-width" version = "0.1.9" @@ -1689,12 +1716,6 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" -[[package]] -name = "vec_map" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" - [[package]] name = "version_check" version = "0.9.4" diff --git a/Cargo.toml b/Cargo.toml index f780551d..28d2d7b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ homepage = "https://github.com/EmbarkStudios/cargo-deny" categories = ["development-tools::cargo-plugins"] keywords = ["cargo", "license", "spdx", "ci", "advisories"] exclude = ["examples/", ".github/"] +rust-version = "1.60.0" [badges] maintenance = { status = "actively-developed" } @@ -27,7 +28,7 @@ path = "src/cargo-deny/main.rs" default = ["vendored-openssl"] # Allows the use of a vendored version openssl when compiling libgit, which allows # us to compile static executables (eg musl) and avoid system dependencies -vendored-openssl = ["rustsec/vendored-openssl", "git2/vendored-openssl"] +vendored-openssl = ["cargo?/vendored-openssl", "crates-index/vendored-openssl", "git2/vendored-openssl"] # Allows embedding cargo as a library so that we can run in minimal (eg container) # environments that don't need to have cargo/rust installed on them for cargo-deny # to still function @@ -45,28 +46,32 @@ atty = "0.2" # Used to track various things during check runs bitvec = { version = "1.0", features = ["alloc"] } # Allows us to do eg cargo metadata operations without relying on an external cargo -cargo = { version = "0.60", optional = true } +cargo = { version = "0.61", optional = true } +# Argument parsing +clap = { version = "3.1", features = ["derive", "env"] } # Used for diagnostic reporting codespan = "0.11" codespan-reporting = "0.11" +# Fetching and reading of crates.io (or other indices) +crates-index = { version = "0.18", default-features = false, features = [ + "parallel", +] } # Brrrrr crossbeam = "0.8" # Logging utilities fern = "0.6" # We directly interact with git when doing index operations eg during fix -git2 = "0.13" # must be kept in sync with cargo and rustsec +git2 = "0.14" # We need to figure out HOME/CARGO_HOME in some cases home = "0.5" # Provides graphs on top of cargo_metadata krates = { version = "0.10", features = ["targets"] } # Log macros log = "0.4" -# Used when parsing binary files in registry index caches -memchr = "2.3" # Moar brrrr rayon = "1.4" # Used for interacting with advisory databases -rustsec = "0.25" +rustsec = { version = "0.25", default-features = false } # Parsing and checking of versions/version requirements semver = "1.0" # Gee what could it be @@ -76,8 +81,6 @@ serde_json = "1.0" smallvec = "1.6" # Used for parsing and checking SPDX license expressions spdx = "0.8" -# Handles all of the argument parsing -structopt = "0.3" # Timestamp emission time = { version = "0.3", default-features = false, features = [ "formatting", @@ -91,7 +94,5 @@ twox-hash = { version = "1.5", default-features = false } url = "2.1" [dev-dependencies] -# Avoid loading license check many times -lazy_static = "1.4.0" # We use this for creating fake crate directories for crawling license files on disk tempfile = "3.1.0" diff --git a/src/advisories.rs b/src/advisories.rs index b1d3424e..f795ccbc 100644 --- a/src/advisories.rs +++ b/src/advisories.rs @@ -43,12 +43,18 @@ pub fn check( let (report, yanked) = rayon::join( || Report::generate(advisory_dbs, &lockfile, emit_audit_compatible_reports), || { - let index = rustsec::registry::Index::open()?; + // TODO: Once rustsec fully supports non-crates.io sources we'll want + // to also fetch those as well + let index = crates_index::Index::new_cargo_default()?; let mut yanked = Vec::new(); for package in &lockfile.0.packages { - if let Ok(index_entry) = index.find(&package.name, &package.version) { - if index_entry.is_yanked { + if let Some(krate) = index.crate_(package.name.as_str()) { + if krate + .versions() + .iter() + .any(|kv| kv.version() == package.version.to_string() && kv.is_yanked()) + { yanked.push(package); } } @@ -59,7 +65,7 @@ pub fn check( ); // rust is having trouble doing type inference - let yanked: Result<_, rustsec::Error> = yanked; + let yanked: Result<_, anyhow::Error> = yanked; use bitvec::prelude::*; let mut ignore_hits: BitVec = BitVec::repeat(false, ctx.cfg.ignore.len()); diff --git a/src/advisories/helpers.rs b/src/advisories/helpers.rs index 15875890..54c76713 100644 --- a/src/advisories/helpers.rs +++ b/src/advisories/helpers.rs @@ -5,6 +5,9 @@ pub use rustsec::{advisory::Id, lockfile::Lockfile, Database, Vulnerability}; use std::path::{Path, PathBuf}; use url::Url; +// The default, official, rustsec advisory database +const DEFAULT_URL: &str = "https://github.com/RustSec/advisory-db"; + /// Whether the database will be fetched or not #[derive(Copy, Clone)] pub enum Fetch { @@ -16,7 +19,7 @@ pub enum Fetch { /// A collection of [`Database`]s that is used to query advisories /// in many different databases. /// -/// [`Database`]: https://docs.rs/rustsec/0.21.0/rustsec/database/struct.Database.html +/// [`Database`]: https://docs.rs/rustsec/0.25.0/rustsec/database/struct.Database.html pub struct DbSet { dbs: Vec<(Url, Database)>, } @@ -57,9 +60,9 @@ impl DbSet { if urls.is_empty() { info!( "No advisory database configured, falling back to default '{}'", - rustsec::repository::git::DEFAULT_URL + DEFAULT_URL ); - urls.push(Url::parse(rustsec::repository::git::DEFAULT_URL).unwrap()); + urls.push(Url::parse(DEFAULT_URL).unwrap()); } use rayon::prelude::*; @@ -178,34 +181,30 @@ fn url_to_path(mut db_path: PathBuf, url: &Url) -> Result { } fn load_db(db_url: &Url, root_db_path: PathBuf, fetch: Fetch) -> Result { - use rustsec::repository::git::Repository; let db_path = url_to_path(root_db_path, db_url)?; - let db_repo = match fetch { + match fetch { Fetch::Allow => { debug!("Fetching advisory database from '{}'", db_url); - - Repository::fetch(db_url.as_str(), &db_path, true /* ensure_fresh */) - .context("failed to fetch advisory database")? + fetch_via_git(db_url, &db_path)?; } Fetch::AllowWithGitCli => { debug!("Fetching advisory database with git cli from '{}'", db_url); - fetch_with_cli(db_url.as_str(), &db_path) + fetch_via_cli(db_url.as_str(), &db_path) .context("failed to fetch advisory database with cli")?; - - Repository::open(&db_path).context("failed to open advisory database")? } Fetch::Disallow => { debug!("Opening advisory database at '{}'", db_path.display()); - - Repository::open(&db_path).context("failed to open advisory database")? } - }; + } + + // Verify that the repository is actually valid + git2::Repository::open(&db_path).context("failed to open advisory database")?; debug!("loading advisory database from {}", db_path.display()); - let res = Database::load_from_repo(&db_repo).context("failed to load advisory database"); + let res = Database::open(&db_path).context("failed to load advisory database"); debug!( "finished loading advisory database from {}", @@ -215,7 +214,143 @@ fn load_db(db_url: &Url, root_db_path: PathBuf, fetch: Fetch) -> Result Result<(), Error> { +fn fetch_via_git(url: &Url, db_path: &Path) -> Result<(), Error> { + anyhow::ensure!( + url.scheme() == "https" || url.scheme() == "ssh", + "expected '{}' to be an `https` or `ssh` url", + url + ); + + // Ensure the parent directory chain is created, git2 won't do it for us + { + let parent = db_path + .parent() + .with_context(|| format!("invalid directory: {}", db_path.display()))?; + + if !parent.is_dir() { + std::fs::create_dir_all(parent)?; + } + } + + // Avoid libgit2 errors in the case the directory exists but is + // otherwise empty. + // + // See: https://github.com/RustSec/cargo-audit/issues/32 + if db_path.is_dir() && std::fs::read_dir(&db_path)?.next().is_none() { + std::fs::remove_dir(&db_path)?; + } + + /// Ref for the `main` branch in the local repository + const LOCAL_REF: &str = "refs/heads/main"; + + /// Ref for the `main` branch in the remote repository + const REMOTE_REF: &str = "refs/remotes/origin/main"; + + let git_config = git2::Config::new()?; + + with_authentication(url.as_str(), &git_config, |f| { + let mut callbacks = git2::RemoteCallbacks::new(); + callbacks.credentials(f); + + let mut proxy_opts = git2::ProxyOptions::new(); + proxy_opts.auto(); + + let mut fetch_opts = git2::FetchOptions::new(); + fetch_opts.remote_callbacks(callbacks); + fetch_opts.proxy_options(proxy_opts); + + if db_path.exists() { + let repo = git2::Repository::open(&db_path)?; + let refspec = format!("{LOCAL_REF}:{REMOTE_REF}"); + + // Fetch remote packfiles and update tips + let mut remote = repo.remote_anonymous(url.as_str())?; + remote.fetch(&[refspec.as_str()], Some(&mut fetch_opts), None)?; + + // Get the current remote tip (as an updated local reference) + let remote_main_ref = repo.find_reference(REMOTE_REF)?; + let remote_target = remote_main_ref.target().unwrap(); + + // Set the local main ref to match the remote + match repo.find_reference(LOCAL_REF) { + Ok(mut local_main_ref) => { + local_main_ref.set_target( + remote_target, + &format!("moving `main` to {}: {}", REMOTE_REF, &remote_target), + )?; + } + Err(e) if e.code() == git2::ErrorCode::NotFound => { + anyhow::bail!("unable to find reference '{}'", LOCAL_REF); + } + Err(e) => { + return Err(e.into()); + } + }; + } else { + git2::build::RepoBuilder::new() + .fetch_options(fetch_opts) + .clone(url.as_str(), db_path)?; + } + + Ok(()) + })?; + + let repo = git2::Repository::open(&db_path).context("failed to open repository")?; + + // Retrieve the HEAD commit + let head = repo.head()?; + + let oid = head + .target() + .with_context(|| format!("no ref target for '{}'", db_path.display()))?; + + let commit_id = oid.to_string(); + let commit_object = repo.find_object(oid, Some(git2::ObjectType::Commit))?; + let commit = commit_object + .as_commit() + .context("HEAD OID was not a reference to a commit")?; + + // Reset the state of the repository to the latest commit + repo.reset(&commit_object, git2::ResetType::Hard, None)?; + + let author = commit.author().to_string(); + + let summary = commit + .summary() + .with_context(|| format!("no commit summary for {}", commit_id))?; + + // Commits to the official rustsec database should always be signed, but we + // may have to relax this requirement for non-official/private databases + // TODO: verify signatures against GitHub's public key + repo.extract_signature(&oid, None).with_context(|| { + format!( + "no signature on commit {}: {} ({})", + commit_id, summary, author, + ) + })?; + + let timestamp = time::OffsetDateTime::from_unix_timestamp(commit.time().seconds()) + .context("commit timestamp is invalid")?; + + // 90 days + const MINIMUM_FRESHNESS: time::Duration = time::Duration::seconds(90 * 24 * 60 * 60); + + // Ensure that the upstream repository hasn't gone stale, ie, they've + // configured cargo-deny to not fetch the remote database(s), but they've + // failed to update the databases manuallly + anyhow::ensure!( + timestamp + > time::OffsetDateTime::now_utc() + .checked_sub(MINIMUM_FRESHNESS) + .expect("this should never happen"), + "repository is stale (last commit: {})", + timestamp + ); + + Ok(()) +} + +fn fetch_via_cli(url: &str, db_path: &Path) -> Result<(), Error> { use std::{fs, process::Command}; if let Some(parent) = db_path.parent() { @@ -251,6 +386,221 @@ fn fetch_with_cli(url: &str, db_path: &Path) -> Result<(), Error> { Ok(()) } +/// Prepare the authentication callbacks for cloning a git repository. +/// +/// The main purpose of this function is to construct the "authentication +/// callback" which is used to clone a repository. This callback will attempt to +/// find the right authentication on the system (without user input) and will +/// guide libgit2 in doing so. +/// +/// The callback is provided `allowed` types of credentials, and we try to do as +/// much as possible based on that: +/// +/// * Prioritize SSH keys from the local ssh agent as they're likely the most +/// reliable. The username here is prioritized from the credential +/// callback, then from whatever is configured in git itself, and finally +/// we fall back to the generic user of `git`. +/// +/// * If a username/password is allowed, then we fallback to git2-rs's +/// implementation of the credential helper. This is what is configured +/// with `credential.helper` in git, and is the interface for the macOS +/// keychain, for example. +/// +/// * After the above two have failed, we just kinda grapple attempting to +/// return *something*. +/// +/// If any form of authentication fails, libgit2 will repeatedly ask us for +/// credentials until we give it a reason to not do so. To ensure we don't +/// just sit here looping forever we keep track of authentications we've +/// attempted and we don't try the same ones again. +pub fn with_authentication(url: &str, cfg: &git2::Config, mut f: F) -> Result +where + F: FnMut(&mut git2::Credentials<'_>) -> Result, +{ + let mut cred_helper = git2::CredentialHelper::new(url); + cred_helper.config(cfg); + + let mut ssh_username_requested = false; + let mut cred_helper_bad = None; + let mut ssh_agent_attempts = Vec::new(); + let mut any_attempts = false; + let mut tried_sshkey = false; + + let mut res = f(&mut |url, username, allowed| { + any_attempts = true; + // libgit2's "USERNAME" authentication actually means that it's just + // asking us for a username to keep going. This is currently only really + // used for SSH authentication and isn't really an authentication type. + // The logic currently looks like: + // + // let user = ...; + // if (user.is_null()) + // user = callback(USERNAME, null, ...); + // + // callback(SSH_KEY, user, ...) + // + // So if we're being called here then we know that (a) we're using ssh + // authentication and (b) no username was specified in the URL that + // we're trying to clone. We need to guess an appropriate username here, + // but that may involve a few attempts. Unfortunately we can't switch + // usernames during one authentication session with libgit2, so to + // handle this we bail out of this authentication session after setting + // the flag `ssh_username_requested`, and then we handle this below. + if allowed.contains(git2::CredentialType::USERNAME) { + debug_assert!(username.is_none()); + ssh_username_requested = true; + return Err(git2::Error::from_str("gonna try usernames later")); + } + + // An "SSH_KEY" authentication indicates that we need some sort of SSH + // authentication. This can currently either come from the ssh-agent + // process or from a raw in-memory SSH key. Cargo only supports using + // ssh-agent currently. + // + // If we get called with this then the only way that should be possible + // is if a username is specified in the URL itself (e.g., `username` is + // Some), hence the unwrap() here. We try custom usernames down below. + if allowed.contains(git2::CredentialType::SSH_KEY) && !tried_sshkey { + // If ssh-agent authentication fails, libgit2 will keep + // calling this callback asking for other authentication + // methods to try. Make sure we only try ssh-agent once, + // to avoid looping forever. + tried_sshkey = true; + let username = username.unwrap(); + debug_assert!(!ssh_username_requested); + ssh_agent_attempts.push(username.to_string()); + return git2::Cred::ssh_key_from_agent(username); + } + + // Sometimes libgit2 will ask for a username/password in plaintext. This + // is where Cargo would have an interactive prompt if we supported it, + // but we currently don't! Right now the only way we support fetching a + // plaintext password is through the `credential.helper` support, so + // fetch that here. + // + // If ssh-agent authentication fails, libgit2 will keep calling this + // callback asking for other authentication methods to try. Check + // cred_helper_bad to make sure we only try the git credential helper + // once, to avoid looping forever. + if allowed.contains(git2::CredentialType::USER_PASS_PLAINTEXT) && cred_helper_bad.is_none() + { + let r = git2::Cred::credential_helper(cfg, url, username); + cred_helper_bad = Some(r.is_err()); + return r; + } + + // I'm... not sure what the DEFAULT kind of authentication is, but seems + // easy to support? + if allowed.contains(git2::CredentialType::DEFAULT) { + return git2::Cred::default(); + } + + // Whelp, we tried our best + Err(git2::Error::from_str("no authentication available")) + }); + + // Ok, so if it looks like we're going to be doing ssh authentication, we + // want to try a few different usernames as one wasn't specified in the URL + // for us to use. In order, we'll try: + // + // * A credential helper's username for this URL, if available. + // * This account's username. + // * "git" + // + // We have to restart the authentication session each time (due to + // constraints in libssh2 I guess? maybe this is inherent to ssh?), so we + // call our callback, `f`, in a loop here. + if ssh_username_requested { + debug_assert!(res.is_err()); + let mut attempts = vec!["git".to_owned()]; + if let Ok(s) = std::env::var("USER").or_else(|_| std::env::var("USERNAME")) { + attempts.push(s); + } + if let Some(ref s) = cred_helper.username { + attempts.push(s.clone()); + } + + while let Some(s) = attempts.pop() { + // We should get `USERNAME` first, where we just return our attempt, + // and then after that we should get `SSH_KEY`. If the first attempt + // fails we'll get called again, but we don't have another option so + // we bail out. + let mut attempts = 0; + res = f(&mut |_url, username, allowed| { + if allowed.contains(git2::CredentialType::USERNAME) { + return git2::Cred::username(&s); + } + if allowed.contains(git2::CredentialType::SSH_KEY) { + debug_assert_eq!(Some(&s[..]), username); + attempts += 1; + if attempts == 1 { + ssh_agent_attempts.push(s.clone()); + return git2::Cred::ssh_key_from_agent(&s); + } + } + Err(git2::Error::from_str("no authentication available")) + }); + + // If we made two attempts then that means: + // + // 1. A username was requested, we returned `s`. + // 2. An ssh key was requested, we returned to look up `s` in the + // ssh agent. + // 3. For whatever reason that lookup failed, so we were asked again + // for another mode of authentication. + // + // Essentially, if `attempts == 2` then in theory the only error was + // that this username failed to authenticate (e.g., no other network + // errors happened). Otherwise something else is funny so we bail + // out. + if attempts != 2 { + break; + } + } + } + + if res.is_ok() || !any_attempts { + return res.map_err(From::from); + } + + // In the case of an authentication failure (where we tried something) then + // we try to give a more helpful error message about precisely what we + // tried. + let res = res.map_err(|_e| { + let mut msg = "failed to authenticate when downloading repository".to_owned(); + if !ssh_agent_attempts.is_empty() { + let names = ssh_agent_attempts + .iter() + .map(|s| format!("`{}`", s)) + .collect::>() + .join(", "); + msg.push_str(&format!( + "\nattempted ssh-agent authentication, but \ + none of the usernames {} succeeded", + names + )); + } + if let Some(failed_cred_helper) = cred_helper_bad { + if failed_cred_helper { + msg.push_str( + "\nattempted to find username/password via \ + git's `credential.helper` support, but failed", + ); + } else { + msg.push_str( + "\nattempted to find username/password via \ + `credential.helper`, but maybe the found \ + credentials were incorrect", + ); + } + } + + anyhow::anyhow!(msg) + })?; + + Ok(res) +} + pub fn load_lockfile(path: &krates::Utf8Path) -> Result { let mut lockfile = Lockfile::load(path)?; @@ -260,6 +610,10 @@ pub fn load_lockfile(path: &krates::Utf8Path) -> Result { Ok(lockfile) } +/// A wrapper around a rustsec `Lockfile`, this is used to filter out all of +/// the crates that are not part of the crate graph for some reason, eg. a target +/// specific dependency for a target the user doesn't actually target, so that +/// any advisories that affect crates not in the graph are triggered pub struct PrunedLockfile(pub(crate) Lockfile); impl PrunedLockfile { diff --git a/src/cargo-deny/check.rs b/src/cargo-deny/check.rs index 72a72da2..b0c809ab 100644 --- a/src/cargo-deny/check.rs +++ b/src/cargo-deny/check.rs @@ -8,56 +8,50 @@ use cargo_deny::{ use log::error; use serde::Deserialize; use std::{path::PathBuf, time::Instant}; -use structopt::{clap::arg_enum, StructOpt}; - -arg_enum! { - #[derive(Debug, PartialEq, Copy, Clone)] - pub enum WhichCheck { - Advisories, - Ban, - Bans, - License, - Licenses, - Sources, - All, - } + +#[derive(clap::ArgEnum, Debug, PartialEq, Copy, Clone)] +pub enum WhichCheck { + Advisories, + Ban, + Bans, + License, + Licenses, + Sources, + All, } -#[derive(StructOpt, Debug)] +#[derive(clap::Parser, Debug)] pub struct Args { /// Path to the config to use /// /// Defaults to /deny.toml if not specified - #[structopt(short, long, parse(from_os_str))] + #[clap(short, long, parse(from_os_str))] pub config: Option, /// Path to graph_output root directory /// /// If set, a dotviz graph will be created for whenever multiple versions of the same crate are detected. /// /// Each file will be created at /graph_output/.dot. /graph_output/* is deleted and recreated each run. - #[structopt(short, long, parse(from_os_str))] + #[clap(short, long, parse(from_os_str))] pub graph: Option, /// Hides the inclusion graph when printing out info for a crate - #[structopt(long)] + #[clap(long)] pub hide_inclusion_graph: bool, /// Disable fetching of the advisory database /// /// When running the `advisories` check, the configured advisory database will be fetched and opened. If this flag is passed, the database won't be fetched, but an error will occur if it doesn't already exist locally. - #[structopt(short, long)] + #[clap(short, long)] pub disable_fetch: bool, /// To ease transition from cargo-audit to cargo-deny, this flag will tell cargo-deny to output the exact same output as cargo-audit would, to `stdout` instead of `stderr`, just as with cargo-audit. /// /// Note that this flag only applies when the output format is JSON, and note that since cargo-deny supports multiple advisory databases, instead of a single JSON object, there will be 1 for each unique advisory database. - #[structopt(long)] + #[clap(long)] pub audit_compatible_output: bool, /// Show stats for all the checks, regardless of the log-level - #[structopt(short, long = "show-stats")] + #[clap(short, long = "show-stats")] pub show_stats: bool, /// The check(s) to perform - #[structopt( - possible_values = &WhichCheck::variants(), - case_insensitive = true, - )] + #[clap(arg_enum)] pub which: Vec, } diff --git a/src/cargo-deny/fetch.rs b/src/cargo-deny/fetch.rs index 03eebc78..a08b5486 100644 --- a/src/cargo-deny/fetch.rs +++ b/src/cargo-deny/fetch.rs @@ -4,29 +4,23 @@ use cargo_deny::{ diag::{Diagnostic, Files}, }; use std::path::PathBuf; -use structopt::{clap::arg_enum, StructOpt}; - -arg_enum! { - #[derive(Debug, PartialEq, Copy, Clone)] - pub enum FetchSource { - Db, - Index, - All, - } + +#[derive(clap::ArgEnum, Debug, PartialEq, Copy, Clone)] +pub enum FetchSource { + Db, + Index, + All, } -#[derive(StructOpt, Debug, Clone)] +#[derive(clap::Parser, Debug, Clone)] pub struct Args { /// Path to the config to use /// /// Defaults to /deny.toml if not specified - #[structopt(short, long, parse(from_os_str))] + #[clap(short, long, parse(from_os_str))] config: Option, /// The sources to fetch - #[structopt( - possible_values = &FetchSource::variants(), - case_insensitive = true, - )] + #[clap(arg_enum)] sources: Vec, } @@ -130,7 +124,16 @@ pub fn cmd( if fetch_index { s.spawn(|_| { log::info!("fetching crates.io index..."); - index = Some(rustsec::registry::Index::fetch()); + index = Some(match crates_index::Index::new_cargo_default() { + Ok(mut index) => match index.update() { + Ok(_) => Ok(index), + Err(err) => Err(anyhow::anyhow!( + "opened crates.io index but failed to fetch updates: {}", + err + )), + }, + Err(err) => Err(anyhow::anyhow!("failed to open crates.io index: {}", err)), + }); log::info!("fetched crates.io index"); }); } diff --git a/src/cargo-deny/init.rs b/src/cargo-deny/init.rs index 403ca759..a3a35b14 100644 --- a/src/cargo-deny/init.rs +++ b/src/cargo-deny/init.rs @@ -1,13 +1,12 @@ use anyhow::{ensure, Context, Error}; use std::path::PathBuf; -use structopt::StructOpt; -#[derive(StructOpt, Debug, Clone)] +#[derive(clap::Parser, Debug, Clone)] pub struct Args { /// The path to create /// /// Defaults to /deny.toml - #[structopt(parse(from_os_str))] + #[clap(parse(from_os_str))] config: Option, } diff --git a/src/cargo-deny/list.rs b/src/cargo-deny/list.rs index a9c080d3..49ed7102 100644 --- a/src/cargo-deny/list.rs +++ b/src/cargo-deny/list.rs @@ -3,56 +3,39 @@ use anyhow::{Context, Error}; use cargo_deny::{diag::Files, licenses, Kid}; use serde::Serialize; use std::path::PathBuf; -use structopt::{clap::arg_enum, StructOpt}; -arg_enum! { - #[derive(Copy, Clone, Debug)] - pub enum Layout { - Crate, - License, - } +#[derive(clap::ArgEnum, Copy, Clone, Debug)] +pub enum Layout { + Crate, + License, } -arg_enum! { - #[derive(Copy, Clone, Debug)] - pub enum OutputFormat { - Human, - Json, - Tsv, - } +#[derive(clap::ArgEnum, Copy, Clone, Debug)] +pub enum OutputFormat { + Human, + Json, + Tsv, } -#[derive(StructOpt, Debug)] +#[derive(clap::Parser, Debug)] pub struct Args { /// Path to the config to use /// /// Defaults to a deny.toml in the same folder as the manifest path, or a deny.toml in a parent directory. - #[structopt(short, long, parse(from_os_str))] + #[clap(short, long, parse(from_os_str))] config: Option, /// Minimum confidence threshold for license text /// /// When determining the license from file contents, a confidence score is assigned according to how close the contents are to the canonical license text. If the confidence score is below this threshold, they license text will ignored, which might mean the crate is treated as unlicensed. /// /// [possible values: 0.0 - 1.0] - #[structopt(short, long, default_value = "0.8")] + #[clap(short, long, default_value = "0.8")] threshold: f32, /// The format of the output - #[structopt( - short, - long, - default_value = "human", - possible_values = &OutputFormat::variants(), - case_insensitive = true, - )] + #[clap(short, long, default_value = "human", arg_enum)] format: OutputFormat, /// The layout for the output, does not apply to TSV - #[structopt( - short, - long, - default_value = "license", - possible_values = &Layout::variants(), - case_insensitive = true, - )] + #[clap(short, long, default_value = "license", arg_enum)] layout: Layout, } diff --git a/src/cargo-deny/main.rs b/src/cargo-deny/main.rs index 3b041e54..cad16eab 100644 --- a/src/cargo-deny/main.rs +++ b/src/cargo-deny/main.rs @@ -80,8 +80,8 @@ #![allow(clippy::exit, clippy::single_match_else)] use anyhow::{bail, Context, Error}; +use clap::{Parser, Subcommand}; use std::path::PathBuf; -use structopt::StructOpt; mod check; mod common; @@ -90,23 +90,23 @@ mod init; mod list; mod stats; -#[derive(StructOpt, Debug)] +#[derive(Subcommand, Debug)] enum Command { /// Checks a project's crate graph - #[structopt(name = "check")] + #[clap(name = "check")] Check(check::Args), /// Fetches remote data - #[structopt(name = "fetch")] + #[clap(name = "fetch")] Fetch(fetch::Args), /// Creates a cargo-deny config from a template - #[structopt(name = "init")] + #[clap(name = "init")] Init(init::Args), /// Outputs a listing of all licenses and the crates that use them - #[structopt(name = "list")] + #[clap(name = "list")] List(list::Args), } -#[derive(StructOpt, Copy, Clone, Debug, PartialEq)] +#[derive(Parser, Copy, Clone, Debug, PartialEq)] pub enum Format { Human, Json, @@ -132,7 +132,7 @@ impl std::str::FromStr for Format { } } -#[derive(StructOpt, Copy, Clone, Debug)] +#[derive(Parser, Copy, Clone, Debug)] pub enum Color { Auto, Always, @@ -165,58 +165,58 @@ fn parse_level(s: &str) -> Result { .with_context(|| format!("failed to parse level '{}'", s)) } -#[derive(StructOpt)] -#[structopt(rename_all = "kebab-case", max_term_width = 80)] +#[derive(Parser)] +#[clap(rename_all = "kebab-case")] pub(crate) struct GraphContext { /// The path of a Cargo.toml to use as the context for the operation. /// /// By default, the Cargo.toml in the current working directory is used. - #[structopt(long, parse(from_os_str))] + #[clap(long, parse(from_os_str))] pub(crate) manifest_path: Option, /// If passed, all workspace packages are used as roots for the crate graph. /// /// Automatically assumed if the manifest path points to a virtual manifest. /// /// Normally, if you specify a manifest path that is a member of a workspace, that crate will be the sole root of the crate graph, meaning only other workspace members that are dependencies of that workspace crate will be included in the graph. This overrides that behavior to include all workspace members. - #[structopt(long)] + #[clap(long)] pub(crate) workspace: bool, /// One or more crates to exclude from the crate graph that is used. /// /// NOTE: Unlike cargo, this does not have to be used with the `--workspace` flag. - #[structopt(long)] + #[clap(long)] pub(crate) exclude: Vec, /// One or more platforms to filter crates by /// /// If a dependency is target specific, it will be ignored if it does not match 1 or more of the specified targets. This option overrides the top-level `targets = []` configuration value. - #[structopt(short, long)] + #[clap(short, long)] pub(crate) target: Vec, /// Activate all available features - #[structopt(long)] + #[clap(long)] pub(crate) all_features: bool, /// Do not activate the `default` feature - #[structopt(long)] + #[clap(long)] pub(crate) no_default_features: bool, /// Space or comma separated list of features to activate - #[structopt(long, use_delimiter = true)] + #[clap(long, use_value_delimiter = true)] pub(crate) features: Vec, /// Require Cargo.lock and cache are up to date - #[structopt(long)] + #[clap(long)] pub(crate) frozen: bool, /// Require Cargo.lock is up to date - #[structopt(long)] + #[clap(long)] pub(crate) locked: bool, /// Run without accessing the network. If used with the `check` subcommand, this also disables advisory database fetching. - #[structopt(long)] + #[clap(long)] pub(crate) offline: bool, } /// Lints your project's crate graph -#[derive(StructOpt)] -#[structopt(rename_all = "kebab-case", max_term_width = 80)] +#[derive(Parser)] +#[clap(author, version, about, long_about = None, rename_all = "kebab-case", max_term_width = 80)] struct Opts { /// The log level for messages - #[structopt( - short = "L", + #[clap( + short = 'L', long = "log-level", default_value = "warn", parse(try_from_str = parse_level), @@ -234,13 +234,13 @@ Possible values: ")] log_level: log::LevelFilter, /// Specify the format of cargo-deny's output - #[structopt(short, long, default_value = "human", possible_values = Format::variants())] + #[clap(short, long, default_value = "human", possible_values = Format::variants())] format: Format, - #[structopt(short, long, default_value = "auto", possible_values = Color::variants(), env = "CARGO_TERM_COLOR")] + #[clap(short, long, default_value = "auto", possible_values = Color::variants(), env = "CARGO_TERM_COLOR")] color: Color, - #[structopt(flatten)] + #[clap(flatten)] ctx: GraphContext, - #[structopt(subcommand)] + #[clap(subcommand)] cmd: Command, } @@ -331,7 +331,7 @@ fn setup_logger( fn real_main() -> Result<(), Error> { let args = - Opts::from_iter({ + Opts::parse_from({ std::env::args().enumerate().filter_map(|(i, a)| { if i == 1 && a == "deny" { None diff --git a/tests/advisories.rs b/tests/advisories.rs index 76a01b45..15c96e17 100644 --- a/tests/advisories.rs +++ b/tests/advisories.rs @@ -261,7 +261,10 @@ fn downgrades_lint_levels() { #[ignore] fn detects_yanked() { // Force fetch the index just in case - rustsec::registry::Index::fetch().unwrap(); + { + let mut index = crates_index::Index::new_cargo_default().unwrap(); + index.update().unwrap(); + } let TestCtx { dbs, lock, krates } = load();