make it possible for a test to declare should-panic

and write a really basic "meta test" of the compilertest framework

COMPILER_TESTS.md

@@ -42,6 +42,9 @@ whole, instead of just a few lines inside the test.
 * `ignore-test` always ignores the test
 * `ignore-lldb` and `ignore-gdb` will skip the debuginfo tests
 * `min-{gdb,lldb}-version`
+* `should-panic` indicates that the test should fail; used for "meta testing",
+  where we test the compiletest program itself to check that it will generate
+  errors in appropriate scenarios
 
 ## Revisions
 
@@ -73,3 +76,9 @@ fn test_foo() {
     let x: usize = 32_u32; //[foo]~ ERROR mismatched types
 }
 ```
+
+Note that not all headers have meaning when customized too a revision.
+For example, the `ignore-test` header (and all "ignore" headers)
+currently only apply to the test as a whole, not to particular
+revisions. The only headers that are intended to really work when
+customized to a revision are error patterns and compiler flags.

src/compiletest/compiletest.rs

@@ -354,11 +354,16 @@ pub fn is_test(config: &Config, testfile: &Path) -> bool {
 }
 
 pub fn make_test(config: &Config, testpaths: &TestPaths) -> test::TestDescAndFn {
+    let early_props = header::early_props(config, &testpaths.file);
     test::TestDescAndFn {
         desc: test::TestDesc {
             name: make_test_name(config, testpaths),
-            ignore: header::is_test_ignored(config, &testpaths.file),
-            should_panic: test::ShouldPanic::No,
+            ignore: early_props.ignore,
+            should_panic: if early_props.should_panic {
+                test::ShouldPanic::Yes
+            } else {
+                test::ShouldPanic::No
+            },
         },
         testfn: make_test_closure(config, testpaths),
     }

src/compiletest/header.rs

@@ -164,8 +164,6 @@ pub fn load_props_into(props: &mut TestProps, testfile: &Path, cfg: Option<&str>
         if let Some(of) = parse_forbid_output(ln) {
             props.forbid_output.push(of);
         }
-
-        true
     });
 
     for key in vec!["RUST_TEST_NOCAPTURE", "RUST_TEST_THREADS"] {
@@ -179,7 +177,42 @@ pub fn load_props_into(props: &mut TestProps, testfile: &Path, cfg: Option<&str>
     }
 }
 
-pub fn is_test_ignored(config: &Config, testfile: &Path) -> bool {
+pub struct EarlyProps {
+    pub ignore: bool,
+    pub should_panic: bool,
+}
+
+// scan the file to detect whether the test should be ignored and
+// whether it should panic; these are two things the test runner needs
+// to know early, before actually running the test
+pub fn early_props(config: &Config, testfile: &Path) -> EarlyProps {
+    let mut props = EarlyProps {
+        ignore: false,
+        should_panic: false,
+    };
+
+    iter_header(testfile, None, &mut |ln| {
+        props.ignore =
+            props.ignore ||
+            parse_name_directive(ln, "ignore-test") ||
+            parse_name_directive(ln, &ignore_target(config)) ||
+            parse_name_directive(ln, &ignore_architecture(config)) ||
+            parse_name_directive(ln, &ignore_stage(config)) ||
+            parse_name_directive(ln, &ignore_env(config)) ||
+            (config.mode == common::Pretty &&
+             parse_name_directive(ln, "ignore-pretty")) ||
+            (config.target != config.host &&
+             parse_name_directive(ln, "ignore-cross-compile")) ||
+            ignore_gdb(config, ln) ||
+            ignore_lldb(config, ln);
+
+        props.should_panic =
+            props.should_panic ||
+            parse_name_directive(ln, "should-panic");
+    });
+
+    return props;
+
     fn ignore_target(config: &Config) -> String {
         format!("ignore-{}", util::get_os(&config.target))
     }
@@ -246,26 +279,11 @@ pub fn is_test_ignored(config: &Config, testfile: &Path) -> bool {
             false
         }
     }
-
-    let val = iter_header(testfile, None, &mut |ln| {
-        !parse_name_directive(ln, "ignore-test") &&
-        !parse_name_directive(ln, &ignore_target(config)) &&
-        !parse_name_directive(ln, &ignore_architecture(config)) &&
-        !parse_name_directive(ln, &ignore_stage(config)) &&
-        !parse_name_directive(ln, &ignore_env(config)) &&
-        !(config.mode == common::Pretty && parse_name_directive(ln, "ignore-pretty")) &&
-        !(config.target != config.host && parse_name_directive(ln, "ignore-cross-compile")) &&
-        !ignore_gdb(config, ln) &&
-        !ignore_lldb(config, ln)
-    });
-
-    !val
 }
 
 fn iter_header(testfile: &Path,
                cfg: Option<&str>,
-               it: &mut FnMut(&str) -> bool)
-               -> bool {
+               it: &mut FnMut(&str)) {
     let rdr = BufReader::new(File::open(testfile).unwrap());
     for ln in rdr.lines() {
         // Assume that any directives will be found before the first
@@ -274,7 +292,7 @@ fn iter_header(testfile: &Path,
         let ln = ln.unwrap();
         let ln = ln.trim();
         if ln.starts_with("fn") || ln.starts_with("mod") {
-            return true;
+            return;
         } else if ln.starts_with("//[") {
             // A comment like `//[foo]` is specific to revision `foo`
             if let Some(close_brace) = ln.find("]") {
@@ -283,20 +301,18 @@ fn iter_header(testfile: &Path,
                     Some(s) => s == &lncfg[..],
                     None => false,
                 };
-                if matches && !it(&ln[close_brace+1..]) {
-                    return false;
+                if matches {
+                    it(&ln[close_brace+1..]);
                 }
             } else {
                 panic!("malformed condition directive: expected `//[foo]`, found `{}`",
                        ln)
             }
         } else if ln.starts_with("//") {
-            if !it(&ln[2..]) {
-                return false;
-            }
+            it(&ln[2..]);
         }
     }
-    return true;
+    return;
 }
 
 fn parse_error_pattern(line: &str) -> Option<String> {

src/test/compile-fail/meta-expected-error-correct-rev.rs

@@ -0,0 +1,21 @@
+// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// revisions: a
+// pretty-expanded FIXME #23616
+
+// Counterpart to `meta-expected-error-wrong-rev.rs`
+
+#[cfg(a)]
+fn foo() {
+    let x: u32 = 22_usize; //[a]~ ERROR mismatched types
+}
+
+fn main() { }

src/test/compile-fail/meta-expected-error-wrong-rev.rs

@@ -0,0 +1,25 @@
+// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// revisions: a
+// should-panic
+// pretty-expanded FIXME #23616
+
+// This is a "meta-test" of the compilertest framework itself.  In
+// particular, it includes the right error message, but the message
+// targets the wrong revision, so we expect the execution to fail.
+// See also `meta-expected-error-correct-rev.rs`.
+
+#[cfg(a)]
+fn foo() {
+    let x: u32 = 22_usize; //[b]~ ERROR mismatched types
+}
+
+fn main() { }