From b07e4b3c7727f1caa47b1f333c10b5d573ad04d5 Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Tue, 8 Nov 2022 14:49:01 +0000 Subject: [PATCH] Group build script failures separately and extract the build script's crate --- local-crates/build-script-fail/Cargo.toml | 7 ++ local-crates/build-script-fail/build.rs | 3 + local-crates/build-script-fail/src/main.rs | 1 + src/report/display.rs | 2 + src/results/mod.rs | 4 ++ src/runner/test.rs | 15 +++- .../full/downloads.html.context.expected.json | 2 +- .../full/full.html.context.expected.json | 69 ++++++++++++------- .../full/index.html.context.expected.json | 38 +++++----- .../full/markdown.md.context.expected.json | 6 +- tests/minicrater/full/results.expected.json | 20 +++++- .../full.html.context.expected.json | 2 +- .../index.html.context.expected.json | 2 +- .../markdown.md.context.expected.json | 2 +- .../resource-exhaustion/results.expected.json | 2 +- 15 files changed, 125 insertions(+), 50 deletions(-) create mode 100644 local-crates/build-script-fail/Cargo.toml create mode 100644 local-crates/build-script-fail/build.rs create mode 100644 local-crates/build-script-fail/src/main.rs diff --git a/local-crates/build-script-fail/Cargo.toml b/local-crates/build-script-fail/Cargo.toml new file mode 100644 index 000000000..a6efbc32b --- /dev/null +++ b/local-crates/build-script-fail/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "build-fail" +version = "0.1.0" +authors = ["Pietro Albini "] +build = "build.rs" + +[dependencies] diff --git a/local-crates/build-script-fail/build.rs b/local-crates/build-script-fail/build.rs new file mode 100644 index 000000000..1d82aa266 --- /dev/null +++ b/local-crates/build-script-fail/build.rs @@ -0,0 +1,3 @@ +fn main() { + panic!("kawoom") +} diff --git a/local-crates/build-script-fail/src/main.rs b/local-crates/build-script-fail/src/main.rs new file mode 100644 index 000000000..f328e4d9d --- /dev/null +++ b/local-crates/build-script-fail/src/main.rs @@ -0,0 +1 @@ +fn main() {} diff --git a/src/report/display.rs b/src/report/display.rs index bf03f14d8..6f4bc0e36 100644 --- a/src/report/display.rs +++ b/src/report/display.rs @@ -16,6 +16,7 @@ impl ResultName for FailureReason { FailureReason::OOM => "OOM".into(), FailureReason::ICE => "ICE".into(), FailureReason::CompilerError(_) => "compiler error".into(), + FailureReason::BuildScript => "build script failure".into(), FailureReason::DependsOn(_) => "faulty deps".into(), } } @@ -26,6 +27,7 @@ impl ResultName for FailureReason { FailureReason::Unknown | FailureReason::NetworkAccess | FailureReason::Timeout + | FailureReason::BuildScript | FailureReason::OOM | FailureReason::ICE => self.short_name(), } diff --git a/src/results/mod.rs b/src/results/mod.rs index e5d799a74..3cd5b746f 100644 --- a/src/results/mod.rs +++ b/src/results/mod.rs @@ -199,6 +199,7 @@ pub enum FailureReason { Timeout, ICE, NetworkAccess, + BuildScript, CompilerError(BTreeSet), DependsOn(BTreeSet), } @@ -213,6 +214,7 @@ impl ::std::fmt::Display for FailureReason { FailureReason::Timeout => write!(f, "timeout"), FailureReason::ICE => write!(f, "ice"), FailureReason::NetworkAccess => write!(f, "network-access"), + FailureReason::BuildScript => write!(f, "build-script"), FailureReason::CompilerError(codes) => write!( f, "compiler-error({})", @@ -261,6 +263,7 @@ impl ::std::str::FromStr for FailureReason { } else { match s { "network-access" => Ok(FailureReason::NetworkAccess), + "build-script" => Ok(FailureReason::BuildScript), "unknown" => Ok(FailureReason::Unknown), "oom" => Ok(FailureReason::OOM), "timeout" => Ok(FailureReason::Timeout), @@ -277,6 +280,7 @@ impl FailureReason { FailureReason::OOM | FailureReason::Timeout | FailureReason::NetworkAccess => true, FailureReason::CompilerError(_) | FailureReason::DependsOn(_) + | FailureReason::BuildScript | FailureReason::Unknown | FailureReason::ICE => false, } diff --git a/src/runner/test.rs b/src/runner/test.rs index 4c60863cb..8994710bb 100644 --- a/src/runner/test.rs +++ b/src/runner/test.rs @@ -5,7 +5,7 @@ use crate::results::{BrokenReason, EncodingType, FailureReason, TestResult, Writ use crate::runner::tasks::TaskCtx; use crate::runner::OverrideResult; use cargo_metadata::diagnostic::DiagnosticLevel; -use cargo_metadata::{Message, Metadata, Package, Target}; +use cargo_metadata::{Message, Metadata, Package, PackageId, Target}; use docsrs_metadata::Metadata as DocsrsMetadata; use failure::Error; use remove_dir_all::remove_dir_all; @@ -109,6 +109,7 @@ fn run_cargo( } let mut did_ice = false; + let mut build_script_failure = false; let mut did_network = false; let mut error_codes = BTreeSet::new(); let mut deps = BTreeSet::new(); @@ -123,6 +124,16 @@ fn run_cargo( if line.contains("code: 111") && line.contains("Connection refused") { did_network = true; } + if let Some(rest) = line.strip_prefix("error: failed to run custom build command for `") { + build_script_failure = true; + let krate = rest.trim_end_matches('`'); + let krate = PackageId { repr: krate.into() }; + if !local_packages_id.contains(&krate) { + if let Ok(krate) = Crate::try_from(&krate) { + deps.insert(krate); + } + } + } // Avoid trying to deserialize non JSON output if !line.starts_with('{') { @@ -198,6 +209,8 @@ fn run_cargo( Err(e.context(FailureReason::CompilerError(error_codes)).into()) } else if did_network { Err(e.context(FailureReason::NetworkAccess).into()) + } else if build_script_failure { + Err(e.context(FailureReason::BuildScript).into()) } else { Err(e.into()) } diff --git a/tests/minicrater/full/downloads.html.context.expected.json b/tests/minicrater/full/downloads.html.context.expected.json index cebdf81b2..4400ae43e 100644 --- a/tests/minicrater/full/downloads.html.context.expected.json +++ b/tests/minicrater/full/downloads.html.context.expected.json @@ -29,7 +29,7 @@ "path": "logs-archives/test-fail.tar.gz" } ], - "crates_count": 17, + "crates_count": 18, "nav": [ { "active": false, diff --git a/tests/minicrater/full/full.html.context.expected.json b/tests/minicrater/full/full.html.context.expected.json index ca7a514a8..e67943753 100644 --- a/tests/minicrater/full/full.html.context.expected.json +++ b/tests/minicrater/full/full.html.context.expected.json @@ -118,38 +118,40 @@ "fixed", { "RootResults": { - "count": 1, + "count": 2, "results": { - "build failed (unknown)": [ + "build build script failure": [ { - "name": "beta-fixed (local)", + "name": "network-access (local)", "res": "fixed", "runs": [ { - "log": "stable/local/beta-fixed", - "res": 2 + "log": "stable/local/network-access", + "res": 6 }, { - "log": "beta/local/beta-fixed", - "res": 0 + "log": "beta/local/network-access", + "res": 7 } ], - "url": "https://github.com/rust-lang/crater/tree/master/local-crates/beta-fixed" - }, + "url": "https://github.com/rust-lang/crater/tree/master/local-crates/network-access" + } + ], + "build failed (unknown)": [ { - "name": "network-access (local)", + "name": "beta-fixed (local)", "res": "fixed", "runs": [ { - "log": "stable/local/network-access", + "log": "stable/local/beta-fixed", "res": 2 }, { - "log": "beta/local/network-access", - "res": 6 + "log": "beta/local/beta-fixed", + "res": 0 } ], - "url": "https://github.com/rust-lang/crater/tree/master/local-crates/network-access" + "url": "https://github.com/rust-lang/crater/tree/master/local-crates/beta-fixed" } ] } @@ -166,11 +168,11 @@ "runs": [ { "log": "stable/local/broken-cargotoml", - "res": 7 + "res": 8 }, { "log": "beta/local/broken-cargotoml", - "res": 7 + "res": 8 } ], "url": "https://github.com/rust-lang/crater/tree/master/local-crates/broken-cargotoml" @@ -181,11 +183,11 @@ "runs": [ { "log": "stable/local/yanked-deps", - "res": 8 + "res": 9 }, { "log": "beta/local/yanked-deps", - "res": 8 + "res": 9 } ], "url": "https://github.com/rust-lang/crater/tree/master/local-crates/yanked-deps" @@ -212,17 +214,32 @@ ], "url": "https://github.com/rust-lang/crater/tree/master/local-crates/build-fail" }, + { + "name": "build-script-fail (local)", + "res": "build-fail", + "runs": [ + { + "log": "stable/local/build-script-fail", + "res": 6 + }, + { + "log": "beta/local/build-script-fail", + "res": 6 + } + ], + "url": "https://github.com/rust-lang/crater/tree/master/local-crates/build-script-fail" + }, { "name": "faulty-deps (local)", "res": "build-fail", "runs": [ { "log": "stable/local/faulty-deps", - "res": 9 + "res": 10 }, { "log": "beta/local/faulty-deps", - "res": 9 + "res": 10 } ], "url": "https://github.com/rust-lang/crater/tree/master/local-crates/faulty-deps" @@ -338,11 +355,11 @@ "runs": [ { "log": "stable/local/test-fail", - "res": 6 + "res": 7 }, { "log": "beta/local/test-fail", - "res": 6 + "res": 7 } ], "url": "https://github.com/rust-lang/crater/tree/master/local-crates/test-fail" @@ -377,11 +394,11 @@ "Single": "#72a156" } }, - "crates_count": 17, + "crates_count": 18, "full": true, "info": { "broken": 2, - "build-fail": 2, + "build-fail": 3, "fixed": 2, "regressed": 4, "skipped": 1, @@ -424,6 +441,9 @@ { "Single": "#db3026" }, + { + "Single": "#db3026" + }, { "Single": "#65461e" }, @@ -444,6 +464,7 @@ "build compiler error", "build compiler error", "build ICE", + "build build script failure", "test failed (unknown)", "broken Cargo.toml", "deps yanked", diff --git a/tests/minicrater/full/index.html.context.expected.json b/tests/minicrater/full/index.html.context.expected.json index 7763beb80..bb882f272 100644 --- a/tests/minicrater/full/index.html.context.expected.json +++ b/tests/minicrater/full/index.html.context.expected.json @@ -118,38 +118,40 @@ "fixed", { "RootResults": { - "count": 1, + "count": 2, "results": { - "build failed (unknown)": [ + "build build script failure": [ { - "name": "beta-fixed (local)", + "name": "network-access (local)", "res": "fixed", "runs": [ { - "log": "stable/local/beta-fixed", - "res": 2 + "log": "stable/local/network-access", + "res": 6 }, { - "log": "beta/local/beta-fixed", - "res": 0 + "log": "beta/local/network-access", + "res": 7 } ], - "url": "https://github.com/rust-lang/crater/tree/master/local-crates/beta-fixed" - }, + "url": "https://github.com/rust-lang/crater/tree/master/local-crates/network-access" + } + ], + "build failed (unknown)": [ { - "name": "network-access (local)", + "name": "beta-fixed (local)", "res": "fixed", "runs": [ { - "log": "stable/local/network-access", + "log": "stable/local/beta-fixed", "res": 2 }, { - "log": "beta/local/network-access", - "res": 6 + "log": "beta/local/beta-fixed", + "res": 0 } ], - "url": "https://github.com/rust-lang/crater/tree/master/local-crates/network-access" + "url": "https://github.com/rust-lang/crater/tree/master/local-crates/beta-fixed" } ] } @@ -165,11 +167,11 @@ "Single": "#db3026" } }, - "crates_count": 17, + "crates_count": 18, "full": false, "info": { "broken": 2, - "build-fail": 2, + "build-fail": 3, "fixed": 2, "regressed": 4, "skipped": 1, @@ -212,6 +214,9 @@ { "Single": "#db3026" }, + { + "Single": "#db3026" + }, { "Single": "#65461e" } @@ -223,6 +228,7 @@ "build compiler error", "build compiler error", "build ICE", + "build build script failure", "test failed (unknown)" ] } diff --git a/tests/minicrater/full/markdown.md.context.expected.json b/tests/minicrater/full/markdown.md.context.expected.json index 41d2366cd..f33ee4c7d 100644 --- a/tests/minicrater/full/markdown.md.context.expected.json +++ b/tests/minicrater/full/markdown.md.context.expected.json @@ -140,7 +140,7 @@ "runs": [ { "log": "stable/local/network-access", - "res": "build-fail:unknown" + "res": "build-fail:build-script" }, { "log": "beta/local/network-access", @@ -156,11 +156,11 @@ } ] ], - "crates_count": 17, + "crates_count": 18, "full": false, "info": { "broken": 2, - "build-fail": 2, + "build-fail": 3, "fixed": 2, "regressed": 4, "skipped": 1, diff --git a/tests/minicrater/full/results.expected.json b/tests/minicrater/full/results.expected.json index cd4be6d4c..d7d86f986 100644 --- a/tests/minicrater/full/results.expected.json +++ b/tests/minicrater/full/results.expected.json @@ -108,6 +108,24 @@ ], "url": "https://github.com/rust-lang/crater/tree/master/local-crates/build-pass" }, + { + "krate": { + "Local": "build-script-fail" + }, + "name": "build-script-fail (local)", + "res": "build-fail", + "runs": [ + { + "log": "stable/local/build-script-fail", + "res": "build-fail:build-script" + }, + { + "log": "beta/local/build-script-fail", + "res": "build-fail:build-script" + } + ], + "url": "https://github.com/rust-lang/crater/tree/master/local-crates/build-script-fail" + }, { "krate": { "Local": "clippy-warn" @@ -237,7 +255,7 @@ "runs": [ { "log": "stable/local/network-access", - "res": "build-fail:unknown" + "res": "build-fail:build-script" }, { "log": "beta/local/network-access", diff --git a/tests/minicrater/resource-exhaustion/full.html.context.expected.json b/tests/minicrater/resource-exhaustion/full.html.context.expected.json index c742e6b83..ebe09df3d 100644 --- a/tests/minicrater/resource-exhaustion/full.html.context.expected.json +++ b/tests/minicrater/resource-exhaustion/full.html.context.expected.json @@ -92,7 +92,7 @@ ], "result_names": [ "test passed", - "build OOM", + "build build script failure", "test OOM" ] } diff --git a/tests/minicrater/resource-exhaustion/index.html.context.expected.json b/tests/minicrater/resource-exhaustion/index.html.context.expected.json index 536c6039d..70025b3dc 100644 --- a/tests/minicrater/resource-exhaustion/index.html.context.expected.json +++ b/tests/minicrater/resource-exhaustion/index.html.context.expected.json @@ -63,7 +63,7 @@ } ], "result_names": [ - "build OOM", + "build build script failure", "test OOM" ] } diff --git a/tests/minicrater/resource-exhaustion/markdown.md.context.expected.json b/tests/minicrater/resource-exhaustion/markdown.md.context.expected.json index edf5221c6..17a273f62 100644 --- a/tests/minicrater/resource-exhaustion/markdown.md.context.expected.json +++ b/tests/minicrater/resource-exhaustion/markdown.md.context.expected.json @@ -13,7 +13,7 @@ "runs": [ { "log": "stable/local/memory-hungry", - "res": "build-fail:oom" + "res": "build-fail:build-script" }, { "log": "beta/local/memory-hungry", diff --git a/tests/minicrater/resource-exhaustion/results.expected.json b/tests/minicrater/resource-exhaustion/results.expected.json index 29feaada1..963b1ab81 100644 --- a/tests/minicrater/resource-exhaustion/results.expected.json +++ b/tests/minicrater/resource-exhaustion/results.expected.json @@ -27,7 +27,7 @@ "runs": [ { "log": "stable/local/memory-hungry", - "res": "build-fail:oom" + "res": "build-fail:build-script" }, { "log": "beta/local/memory-hungry",