diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 7512f46988c91..36a0fda645805 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -98,21 +98,46 @@ impl<'a> Parser<'a> { } impl CreateTokenStream for LazyTokenStreamImpl { fn create_token_stream(&self) -> TokenStream { - // The token produced by the final call to `next` or `next_desugared` - // was not actually consumed by the callback. The combination - // of chaining the initial token and using `take` produces the desired - // result - we produce an empty `TokenStream` if no calls were made, - // and omit the final token otherwise. + if self.num_calls == 0 { + return TokenStream::new(vec![]); + } + let mut cursor_snapshot = self.cursor_snapshot.clone(); - let tokens = std::iter::once(self.start_token.clone()) - .chain((0..self.num_calls).map(|_| { - if self.desugar_doc_comments { + // Don't skip `None` delimiters, since we want to pass them to + // proc macros. Normally, we'll end up capturing `TokenKind::Interpolated`, + // which gets converted to a `None`-delimited group when we invoke + // a proc-macro. However, it's possible to already have a `None`-delimited + // group in the stream (such as when parsing the output of a proc-macro, + // or in certain unusual cases with cross-crate `macro_rules!` macros). + cursor_snapshot.skip_none_delims = false; + + // The token produced by the final call to `next` or `next_desugared` + // was not actually consumed by the callback. + let num_calls = self.num_calls - 1; + let mut i = 0; + let tokens = + std::iter::once(self.start_token.clone()).chain(std::iter::from_fn(|| { + if i >= num_calls { + return None; + } + + let token = if self.desugar_doc_comments { cursor_snapshot.next_desugared() } else { cursor_snapshot.next() + }; + + // When the `LazyTokenStreamImpl` was original produced, we did *not* + // include `NoDelim` tokens in `num_calls`, since they are normally ignored + // by the parser. Therefore, we only increment our counter for other types of tokens. + if !matches!( + token.0.kind, + token::OpenDelim(token::NoDelim) | token::CloseDelim(token::NoDelim) + ) { + i += 1; } - })) - .take(self.num_calls); + Some(token) + })); make_token_stream(tokens, self.append_unglued_token.clone()) } diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index f0ee76d328c40..748a8e2bb490f 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -172,6 +172,13 @@ struct TokenCursor { // appended to the captured stream when // we evaluate a `LazyTokenStream` append_unglued_token: Option, + // If `true`, skip the delimiters for `None`-delimited groups, + // and just yield the inner tokens. This is `true` during + // normal parsing, since the parser code is not currently prepared + // to handle `None` delimiters. When capturing a `TokenStream`, + // however, we want to handle `None`-delimiters, since + // proc-macros always see `None`-delimited groups. + skip_none_delims: bool, } #[derive(Clone)] @@ -184,13 +191,13 @@ struct TokenCursorFrame { } impl TokenCursorFrame { - fn new(span: DelimSpan, delim: DelimToken, tts: TokenStream) -> Self { + fn new(span: DelimSpan, delim: DelimToken, tts: TokenStream, skip_none_delims: bool) -> Self { TokenCursorFrame { delim, span, - open_delim: delim == token::NoDelim, + open_delim: delim == token::NoDelim && skip_none_delims, tree_cursor: tts.into_trees(), - close_delim: delim == token::NoDelim, + close_delim: delim == token::NoDelim && skip_none_delims, } } } @@ -218,7 +225,7 @@ impl TokenCursor { return (token, spacing); } TokenTree::Delimited(sp, delim, tts) => { - let frame = TokenCursorFrame::new(sp, delim, tts); + let frame = TokenCursorFrame::new(sp, delim, tts, self.skip_none_delims); self.stack.push(mem::replace(&mut self.frame, frame)); } } @@ -276,6 +283,7 @@ impl TokenCursor { .cloned() .collect::() }, + self.skip_none_delims, ), )); @@ -371,12 +379,19 @@ impl<'a> Parser<'a> { prev_token: Token::dummy(), restrictions: Restrictions::empty(), expected_tokens: Vec::new(), + // Skip over the delimiters for `None`-delimited groups token_cursor: TokenCursor { - frame: TokenCursorFrame::new(DelimSpan::dummy(), token::NoDelim, tokens), + frame: TokenCursorFrame::new( + DelimSpan::dummy(), + token::NoDelim, + tokens, + /* skip_none_delims */ true, + ), stack: Vec::new(), num_next_calls: 0, desugar_doc_comments, append_unglued_token: None, + skip_none_delims: true, }, desugar_doc_comments, unmatched_angle_bracket_count: 0, diff --git a/src/test/ui/proc-macro/auxiliary/nested-macro-rules.rs b/src/test/ui/proc-macro/auxiliary/nested-macro-rules.rs index 52ebe8e7fb2ef..27676a5cb8112 100644 --- a/src/test/ui/proc-macro/auxiliary/nested-macro-rules.rs +++ b/src/test/ui/proc-macro/auxiliary/nested-macro-rules.rs @@ -2,14 +2,15 @@ pub struct FirstStruct; #[macro_export] macro_rules! outer_macro { - ($name:ident) => { + ($name:ident, $attr_struct_name:ident) => { #[macro_export] macro_rules! inner_macro { - ($wrapper:ident) => { - $wrapper!($name) + ($bang_macro:ident, $attr_macro:ident) => { + $bang_macro!($name); + #[$attr_macro] struct $attr_struct_name {} } } } } -outer_macro!(FirstStruct); +outer_macro!(FirstStruct, FirstAttrStruct); diff --git a/src/test/ui/proc-macro/nested-macro-rules.rs b/src/test/ui/proc-macro/nested-macro-rules.rs index 2fef0e5fad074..25ffcfad7c7ea 100644 --- a/src/test/ui/proc-macro/nested-macro-rules.rs +++ b/src/test/ui/proc-macro/nested-macro-rules.rs @@ -1,7 +1,7 @@ // run-pass // aux-build:nested-macro-rules.rs // aux-build:test-macros.rs -// compile-flags: -Z span-debug +// compile-flags: -Z span-debug -Z macro-backtrace // edition:2018 #![no_std] // Don't load unnecessary hygiene information from std @@ -10,14 +10,14 @@ extern crate std; extern crate nested_macro_rules; extern crate test_macros; -use test_macros::print_bang; +use test_macros::{print_bang, print_attr}; use nested_macro_rules::FirstStruct; struct SecondStruct; fn main() { - nested_macro_rules::inner_macro!(print_bang); + nested_macro_rules::inner_macro!(print_bang, print_attr); - nested_macro_rules::outer_macro!(SecondStruct); - inner_macro!(print_bang); + nested_macro_rules::outer_macro!(SecondStruct, SecondAttrStruct); + inner_macro!(print_bang, print_attr); } diff --git a/src/test/ui/proc-macro/nested-macro-rules.stdout b/src/test/ui/proc-macro/nested-macro-rules.stdout index dcafe3b4bda60..8292617fc1675 100644 --- a/src/test/ui/proc-macro/nested-macro-rules.stdout +++ b/src/test/ui/proc-macro/nested-macro-rules.stdout @@ -5,10 +5,32 @@ PRINT-BANG INPUT (DEBUG): TokenStream [ stream: TokenStream [ Ident { ident: "FirstStruct", - span: $DIR/auxiliary/nested-macro-rules.rs:15:14: 15:25 (#7), + span: $DIR/auxiliary/nested-macro-rules.rs:16:14: 16:25 (#7), }, ], - span: $DIR/auxiliary/nested-macro-rules.rs:9:27: 9:32 (#6), + span: $DIR/auxiliary/nested-macro-rules.rs:9:30: 9:35 (#6), + }, +] +PRINT-ATTR INPUT (DISPLAY): struct FirstAttrStruct { } +PRINT-ATTR INPUT (DEBUG): TokenStream [ + Ident { + ident: "struct", + span: $DIR/auxiliary/nested-macro-rules.rs:10:32: 10:38 (#6), + }, + Group { + delimiter: None, + stream: TokenStream [ + Ident { + ident: "FirstAttrStruct", + span: $DIR/auxiliary/nested-macro-rules.rs:16:27: 16:42 (#7), + }, + ], + span: $DIR/auxiliary/nested-macro-rules.rs:10:39: 10:56 (#6), + }, + Group { + delimiter: Brace, + stream: TokenStream [], + span: $DIR/auxiliary/nested-macro-rules.rs:10:57: 10:59 (#6), }, ] PRINT-BANG INPUT (DISPLAY): SecondStruct @@ -18,9 +40,31 @@ PRINT-BANG INPUT (DEBUG): TokenStream [ stream: TokenStream [ Ident { ident: "SecondStruct", - span: $DIR/nested-macro-rules.rs:21:38: 21:50 (#13), + span: $DIR/nested-macro-rules.rs:21:38: 21:50 (#16), }, ], - span: $DIR/auxiliary/nested-macro-rules.rs:9:27: 9:32 (#12), + span: $DIR/auxiliary/nested-macro-rules.rs:9:30: 9:35 (#15), + }, +] +PRINT-ATTR INPUT (DISPLAY): struct SecondAttrStruct { } +PRINT-ATTR INPUT (DEBUG): TokenStream [ + Ident { + ident: "struct", + span: $DIR/auxiliary/nested-macro-rules.rs:10:32: 10:38 (#15), + }, + Group { + delimiter: None, + stream: TokenStream [ + Ident { + ident: "SecondAttrStruct", + span: $DIR/nested-macro-rules.rs:21:52: 21:68 (#16), + }, + ], + span: $DIR/auxiliary/nested-macro-rules.rs:10:39: 10:56 (#15), + }, + Group { + delimiter: Brace, + stream: TokenStream [], + span: $DIR/auxiliary/nested-macro-rules.rs:10:57: 10:59 (#15), }, ]