diff --git a/compiler/rustc_expand/src/mbe/diagnostics.rs b/compiler/rustc_expand/src/mbe/diagnostics.rs index e4ed5b9806da7..cd0c54f293e44 100644 --- a/compiler/rustc_expand/src/mbe/diagnostics.rs +++ b/compiler/rustc_expand/src/mbe/diagnostics.rs @@ -2,6 +2,7 @@ use std::borrow::Cow; use rustc_ast::token::{self, Token}; use rustc_ast::tokenstream::TokenStream; +use rustc_data_structures::fx::FxHashSet; use rustc_errors::{Applicability, Diag, DiagCtxtHandle, DiagMessage, pluralize}; use rustc_hir::attrs::diagnostic::{CustomDiagnostic, Directive, FormatArgs}; use rustc_macros::Subdiagnostic; @@ -152,7 +153,13 @@ struct CollectTrackerAndEmitter<'dcx, 'matcher> { /// The matcher currently being parsed. // // FIXME: Factor out a per-arm `Tracker` so that the `Option` is unnecessary. - current: Option, + current: Option<(WhichMatcher, &'matcher [MatcherLoc])>, + + /// Matches of [`MatcherLoc`]s that successfully consumed input from the parser. + /// + /// This accumulates all calls to [`Tracker::matched_one()`]. It is used to identify all + /// competing matches for ambiguity errors. + matches: FxHashSet, remaining_matcher: Option<&'matcher MatcherLoc>, /// Which arm's failure should we report? (the one furthest along) @@ -161,6 +168,17 @@ struct CollectTrackerAndEmitter<'dcx, 'matcher> { result: Option<(Span, ErrorGuaranteed)>, } +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +struct SuccessfulMatch { + /// The position in the parser. + /// + /// As per [`Parser::approx_token_stream_pos()`]. + input_pos: u32, + + /// The index of the [`MatcherLoc`]. + loc_index: u32, +} + struct BestFailure { token: Token, @@ -183,12 +201,12 @@ impl BestFailure { } impl<'dcx, 'matcher> Tracker<'matcher> for CollectTrackerAndEmitter<'dcx, 'matcher> { - fn prepare(&mut self, which_matcher: WhichMatcher) { + fn prepare(&mut self, which_matcher: WhichMatcher, matcher: &'matcher [MatcherLoc]) { if self.current.is_some() { bug!("`Self::after_arm()` was not called to clean up context"); } - self.current = Some(which_matcher); + self.current = Some((which_matcher, matcher)); } fn before_match_loc(&mut self, parser: &TtParser, matcher: &'matcher MatcherLoc) { @@ -199,6 +217,13 @@ impl<'dcx, 'matcher> Tracker<'matcher> for CollectTrackerAndEmitter<'dcx, 'match } } + fn matched_one(&mut self, parser: &Parser<'_>, loc_index: usize) { + let input_pos = parser.approx_token_stream_pos(); + let loc_index: u32 = loc_index.try_into().unwrap(); + let m = SuccessfulMatch { input_pos, loc_index }; + self.matches.insert(m); + } + fn after_arm(&mut self, result: &NamedParseResult) { match *result { Success(_) => { @@ -223,10 +248,11 @@ impl<'dcx, 'matcher> Tracker<'matcher> for CollectTrackerAndEmitter<'dcx, 'match } self.current = None; + self.matches.clear(); } fn failure(&mut self, parser: &Parser<'_>) { - let Some(which_matcher) = self.current else { + let Some((which_matcher, _)) = self.current else { bug!("`Self::prepare()` was not called to initialize context"); }; @@ -262,12 +288,28 @@ impl<'dcx, 'matcher> Tracker<'matcher> for CollectTrackerAndEmitter<'dcx, 'match } } - fn ambiguity( - &mut self, - parser: &Parser<'_>, - bb_locs: impl IntoIterator, - next_locs: impl IntoIterator, - ) { + fn ambiguity(&mut self, parser: &Parser<'_>) { + let Some((_, matcher)) = self.current else { + bug!("`Self::prepare()` was not called to initialize context"); + }; + + #[expect( + rustc::potential_query_instability, + reason = "sorting the results deterministically afterwards" + )] + let (mut bb_locs, mut next_locs) = self + .matches + .iter() + .filter(|m| m.input_pos == parser.approx_token_stream_pos()) + .partition::, _>(|m| { + let loc = &matcher[m.loc_index as usize]; + matches!(loc, MatcherLoc::MetaVarDecl { .. }) + }); + + // Use a reasonable and deterministic ordering for data in the error message. + bb_locs.sort_unstable_by_key(|m| m.loc_index); + next_locs.sort_unstable_by_key(|m| m.loc_index); + let span = parser.token.span.substitute_dummy(self.root_span); if parser.token == token::Eof { @@ -279,11 +321,10 @@ impl<'dcx, 'matcher> Tracker<'matcher> for CollectTrackerAndEmitter<'dcx, 'match let nts = bb_locs .into_iter() - .map(|loc| match loc { - MatcherLoc::MetaVarDecl { bind, kind, .. } => { - format!("{kind} ('{bind}')") - } - _ => unreachable!(), + .map(|m| { + let loc = &matcher[m.loc_index as usize]; + let MatcherLoc::MetaVarDecl { bind, kind, .. } = loc else { unreachable!() }; + format!("{kind} ('{bind}')") }) .collect::>() .join(" or "); @@ -291,7 +332,7 @@ impl<'dcx, 'matcher> Tracker<'matcher> for CollectTrackerAndEmitter<'dcx, 'match let msg = format!( "local ambiguity when calling macro `{}`: multiple parsing options: {}", self.macro_name, - match next_locs.into_iter().count() { + match next_locs.len() { 0 => format!("built-in NTs {nts}."), n => format!("built-in NTs {nts} or {n} other option{s}.", s = pluralize!(n)), } @@ -316,6 +357,7 @@ impl<'dcx> CollectTrackerAndEmitter<'dcx, '_> { macro_name, dcx, current: None, + matches: FxHashSet::default(), remaining_matcher: None, best_failure: None, root_span, diff --git a/compiler/rustc_expand/src/mbe/macro_parser.rs b/compiler/rustc_expand/src/mbe/macro_parser.rs index 1ff5cd6a61787..95a4ebc63d38b 100644 --- a/compiler/rustc_expand/src/mbe/macro_parser.rs +++ b/compiler/rustc_expand/src/mbe/macro_parser.rs @@ -72,6 +72,7 @@ use std::borrow::Cow; use std::fmt::Display; +use std::ops::ControlFlow; use std::rc::Rc; pub(crate) use NamedMatch::*; @@ -82,7 +83,6 @@ use rustc_errors::{Diag, ErrorGuaranteed}; use rustc_middle::span_bug; use rustc_parse::parser::{ParseNtResult, Parser, token_descr}; use rustc_span::{Ident, MacroRulesNormalizedIdent, Span}; -use smallvec::SmallVec; use crate::mbe::macro_rules::Tracker; use crate::mbe::{KleeneOp, TokenTree}; @@ -433,12 +433,12 @@ pub(crate) struct TtParser { /// `parse_tt`. next_mps: Vec, - /// The set of mps that are waiting for the black-box parser. - bb_mps: Vec, - /// Pre-allocate an empty match array, so it can be cloned cheaply for macros with many rules /// that have no metavars. empty_matches: Rc>, + + /// Whether an ambiguity error has occurred. + found_ambiguity: bool, } impl TtParser { @@ -446,8 +446,8 @@ impl TtParser { TtParser { cur_mps: vec![], next_mps: vec![], - bb_mps: vec![], empty_matches: Rc::new(vec![]), + found_ambiguity: false, } } @@ -464,52 +464,45 @@ impl TtParser { /// track of through the mps generated. fn parse_tt_inner<'matcher, T: Tracker<'matcher>>( &mut self, - parser: &Parser<'_>, + parser: &mut Cow<'_, Parser<'_>>, matcher: &'matcher [MatcherLoc], track: &mut T, ) -> Option { - // Matcher positions that would be valid if the macro invocation was over now. Only - // modified if `token == Eof`. - let mut eof_mps = SmallVec::<[MatcherPos; 1]>::new(); - while let Some(mp) = self.cur_mps.pop() { - self.match_one(parser, matcher, mp, track, &mut eof_mps); + if let Some(result) = self.match_one(parser, matcher, mp, track, false) { + return Some(result); + } } - // If we reached the end of input, check that there is EXACTLY ONE possible matcher. - // Otherwise, either the parse is ambiguous (which is an error) or there is a syntax error. - let token = &parser.token; - if *token == token::Eof { - assert!(self.next_mps.is_empty()); - assert!(self.bb_mps.is_empty()); - - Some(match *eof_mps { - [_] => { - let eof_mp = eof_mps.pop().unwrap(); - let matches = Rc::unwrap_or_clone(eof_mp.matches).into_iter(); - Success(self.nameize(matcher, matches)) - } - [] => { - track.failure(parser); - Failure - } - _ => self.ambiguity_error(parser, matcher, track), - }) - } else { - None + // FIXME: Error messages here could be improved with links to original rules. + + if self.next_mps.is_empty() { + // There are no possible next positions: syntax error. + track.failure(parser); + return Some(Failure); } + + // Dump all possible `next_mps` into `cur_mps` for the next iteration. Then + // process the next token. + self.cur_mps.append(&mut self.next_mps); + parser.to_mut().bump(); + + None } /// Match a single [`MatcherPos`]. + /// + /// If a meta-variable is encountered and `checking_for_ambiguity` is `false`, `cur_mps` will be + /// drained to eagerly check for ambiguity, and `parser` will be modified. #[inline(always)] // must be inlined in `parse_tt_inner()` fn match_one<'matcher, T: Tracker<'matcher>>( &mut self, - parser: &Parser<'_>, + parser: &mut Cow<'_, Parser<'_>>, matcher: &'matcher [MatcherLoc], mut mp: MatcherPos, track: &mut T, - eof_mps: &mut SmallVec<[MatcherPos; 1]>, - ) { + checking_for_ambiguity: bool, + ) -> Option { let matcher_loc = &matcher[mp.idx]; track.before_match_loc(self, matcher_loc); let token = &parser.token; @@ -528,6 +521,7 @@ impl TtParser { mp.idx += 1; self.cur_mps.push(mp); } else if token_name_eq(t, token) { + track.matched_one(parser, mp.idx); mp.idx += 1; self.next_mps.push(mp); } @@ -587,6 +581,7 @@ impl TtParser { if token_name_eq(token, separator) { // The separator matches the current token. Advance past it. + track.matched_one(parser, mp.idx); mp.idx += 1; self.next_mps.push(mp); } @@ -597,22 +592,96 @@ impl TtParser { mp.idx = idx_first; self.cur_mps.push(mp); } - &MatcherLoc::MetaVarDecl { kind, .. } => { + &MatcherLoc::MetaVarDecl { kind, next_metavar, seq_depth, .. } => { // Built-in nonterminals never start with these tokens, so we can eliminate them // from consideration. We use the span of the metavariable declaration to determine // any edition-specific matching behavior for non-terminals. - if Parser::nonterminal_may_begin_with(kind, token) { - self.bb_mps.push(mp); + if !Parser::nonterminal_may_begin_with(kind, token) { + return None; + } + + // EOF tokens would cause unexpected processing in `match_one()`. + debug_assert!(parser.token != token::Eof, "{kind:?} should not accept EOF tokens"); + + track.matched_one(parser, mp.idx); + + if let ControlFlow::Break(result) = + self.check_for_ambiguity(parser, matcher, track, checking_for_ambiguity) + { + return result; } + + // We use the span of the metavariable declaration to determine any + // edition-specific matching behavior for non-terminals. + let nt = match parser.to_mut().parse_nonterminal(kind) { + Err(err) => return Some(self.nt_parsing_error(matcher_loc, err)), + Ok(nt) => nt, + }; + mp.push_match(next_metavar, seq_depth, MatchedSingle(nt)); + + mp.idx += 1; + self.cur_mps.push(mp); } MatcherLoc::Eof => { // We are past the matcher's end, and not in a sequence. Try to end things. debug_assert_eq!(mp.idx, matcher.len() - 1); - if *token == token::Eof { - eof_mps.push(mp); + + if *token != token::Eof { + return None; + } + + track.matched_one(parser, mp.idx); + + if let ControlFlow::Break(result) = + self.check_for_ambiguity(parser, matcher, track, checking_for_ambiguity) + { + return result; } + + let matches = Rc::unwrap_or_clone(mp.matches).into_iter(); + return Some(Success(self.nameize(matcher, matches))); } } + + None + } + + /// Look for ambiguity before parsing a non-terminal. + /// + /// - When `checking_for_ambiguity`: immediately an ambiguity error. + /// - Otherwise: eagerly consumes [`Self::cur_mps`] to check for ambiguity. + /// + /// If [`ControlFlow::Continue`] is returned, ambiguity has not been detected. + fn check_for_ambiguity<'matcher, R, T: Tracker<'matcher>>( + &mut self, + parser: &mut Cow<'_, Parser<'_>>, + matcher: &'matcher [MatcherLoc], + track: &mut T, + checking_for_ambiguity: bool, + ) -> ControlFlow>> { + if checking_for_ambiguity { + // This was called in the context of a _different_ `MatcherLoc` that was about to be + // matched. Prevent the caller from doing more work, but don't prepare the actual error + // yet; let the outer `check_for_ambiguity()` do that. + self.found_ambiguity = true; + return ControlFlow::Break(None); + } + + assert!(!self.found_ambiguity); + + // Consume all pending mps at the current input position. + while let Some(mp) = self.cur_mps.pop() { + let result = self.match_one(parser, matcher, mp, track, true); + // A result cannot be returned when `check_for_ambiguity` is `true`. + assert!(result.is_none()); + } + + if std::mem::take(&mut self.found_ambiguity) || !self.next_mps.is_empty() { + track.ambiguity(parser); + ControlFlow::Break(Some(Ambiguity)) + } else { + ControlFlow::Continue(()) + } } /// Match the token stream from `parser` against `matcher`. @@ -631,67 +700,19 @@ impl TtParser { self.cur_mps.push(MatcherPos { idx: 0, matches: Rc::clone(&self.empty_matches) }); loop { + assert!(!self.cur_mps.is_empty()); self.next_mps.clear(); - self.bb_mps.clear(); - // Process `cur_mps` until either we have finished the input or we need to get some - // parsing from the black-box parser done. + // Parse all mps at the current input position, then progress the parser. let res = self.parse_tt_inner(parser, matcher, track); if let Some(res) = res { return res; } - - // `parse_tt_inner` handled all of `cur_mps`, so it's empty. - assert!(self.cur_mps.is_empty()); - - // Error messages here could be improved with links to original rules. - match (self.next_mps.len(), self.bb_mps.len()) { - (0, 0) => { - // There are no possible next positions AND we aren't waiting for the black-box - // parser: syntax error. - track.failure(parser); - return Failure; - } - - (_, 0) => { - // Dump all possible `next_mps` into `cur_mps` for the next iteration. Then - // process the next token. - self.cur_mps.append(&mut self.next_mps); - parser.to_mut().bump(); - } - - (0, 1) => { - // We need to call the black-box parser to get some nonterminal. - let mut mp = self.bb_mps.pop().unwrap(); - let loc = &matcher[mp.idx]; - let MatcherLoc::MetaVarDecl { kind, next_metavar, seq_depth, .. } = *loc else { - unreachable!() - }; - - // We use the span of the metavariable declaration to determine any - // edition-specific matching behavior for non-terminals. - let nt = match parser.to_mut().parse_nonterminal(kind) { - Err(err) => return self.nt_parsing_error(loc, err), - Ok(nt) => nt, - }; - mp.push_match(next_metavar, seq_depth, MatchedSingle(nt)); - - mp.idx += 1; - self.cur_mps.push(mp); - } - - (_, _) => { - // Too many possibilities! - return self.ambiguity_error(parser, matcher, track); - } - } - - assert!(!self.cur_mps.is_empty()); } } - fn nt_parsing_error(&self, loc: &MatcherLoc, err: Diag<'_>) -> NamedParseResult { + fn nt_parsing_error(&self, loc: &MatcherLoc, err: Diag<'_>) -> ParseResult { let &MatcherLoc::MetaVarDecl { span, kind, .. } = loc else { unreachable!() }; let guarantee = err .with_span_label( @@ -702,22 +723,6 @@ impl TtParser { ErrorReported(guarantee) } - fn ambiguity_error<'matcher, T: Tracker<'matcher>>( - &mut self, - parser: &Parser<'_>, - matcher: &'matcher [MatcherLoc], - track: &mut T, - ) -> NamedParseResult { - // Use a reasonable and deterministic ordering for data in the error message. - self.bb_mps.sort_unstable_by_key(|mp| mp.idx); - self.next_mps.sort_unstable_by_key(|mp| mp.idx); - - let bb_locs = self.bb_mps.iter().map(|mp| &matcher[mp.idx]); - let next_locs = self.next_mps.iter().map(|mp| &matcher[mp.idx]); - track.ambiguity(parser, bb_locs, next_locs); - Ambiguity - } - fn nameize>( &self, matcher: &[MatcherLoc], diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs index 4a0cacc977529..025558a745169 100644 --- a/compiler/rustc_expand/src/mbe/macro_rules.rs +++ b/compiler/rustc_expand/src/mbe/macro_rules.rs @@ -360,11 +360,19 @@ fn trace_macros_note(cx_expansions: &mut FxIndexMap>, sp: Span pub(super) trait Tracker<'matcher> { /// Provide context on the arm that's about to be matched. - fn prepare(&mut self, which_matcher: WhichMatcher); + fn prepare(&mut self, which_matcher: WhichMatcher, matcher: &'matcher [MatcherLoc]); /// This is called before trying to match next MatcherLoc on the current token. fn before_match_loc(&mut self, parser: &TtParser, matcher: &'matcher MatcherLoc); + /// A [`MatcherLoc`] successfully consumed input from the parser. + /// + /// This is called for [`MatcherLoc::Token`] and [`MatcherLoc::SequenceSep`], which consume + /// single tokens, when they successfully match [`Parser::token`]. It is also called for + /// [`MatcherLoc::MetaVarDecl`] when non-terminal parsing is guaranteed to occur (i.e. after + /// [`Parser::nonterminal_may_begin_with()`] returns `true`). + fn matched_one(&mut self, parser: &Parser<'_>, loc_index: usize); + /// This is called after an arm has been parsed, either successfully or unsuccessfully. When /// this is called, `before_match_loc` was called at least once (with a `MatcherLoc::Eof`). fn after_arm(&mut self, result: &NamedParseResult); @@ -381,12 +389,7 @@ pub(super) trait Tracker<'matcher> { /// An ambiguity error occurred. /// /// The parser will return [`NamedParseResult::Ambiguity`] after calling this. - fn ambiguity( - &mut self, - parser: &Parser<'_>, - bb_locs: impl IntoIterator, - next_locs: impl IntoIterator, - ); + fn ambiguity(&mut self, parser: &Parser<'_>); /// For tracing. fn description() -> &'static str; @@ -399,17 +402,13 @@ pub(super) trait Tracker<'matcher> { pub(super) struct NoopTracker; impl<'matcher> Tracker<'matcher> for NoopTracker { - fn prepare(&mut self, _which_matcher: WhichMatcher) {} + fn prepare(&mut self, _which_matcher: WhichMatcher, _matcher: &'matcher [MatcherLoc]) {} fn before_match_loc(&mut self, _parser: &TtParser, _matcher: &'matcher MatcherLoc) {} - fn ambiguity( - &mut self, - _parser: &Parser<'_>, - _bb_locs: impl IntoIterator, - _next_locs: impl IntoIterator, - ) { - } + fn matched_one(&mut self, _parser: &Parser<'_>, _loc_index: usize) {} + + fn ambiguity(&mut self, _parser: &Parser<'_>) {} fn after_arm(&mut self, _result: &NamedParseResult) {} @@ -637,7 +636,7 @@ pub(super) fn try_match_macro<'matcher, T: Tracker<'matcher>>( // are not recorded. On the first `Success(..)`ful matcher, the spans are merged. let mut gated_spans_snapshot = mem::take(&mut *psess.gated_spans.spans.borrow_mut()); - track.prepare(WhichMatcher::FOR_FUNC); + track.prepare(WhichMatcher::FOR_FUNC, lhs); let result = tt_parser.parse_tt(&mut Cow::Borrowed(&parser), lhs, track); track.after_arm(&result); @@ -695,7 +694,7 @@ pub(super) fn try_match_macro_attr<'matcher, T: Tracker<'matcher>>( let mut gated_spans_snapshot = mem::take(&mut *psess.gated_spans.spans.borrow_mut()); - track.prepare(WhichMatcher::Args); + track.prepare(WhichMatcher::Args, args); let result = tt_parser.parse_tt(&mut Cow::Borrowed(&args_parser), args, track); track.after_arm(&result); @@ -709,7 +708,7 @@ pub(super) fn try_match_macro_attr<'matcher, T: Tracker<'matcher>>( ErrorReported(guar) => return Err(CanRetry::No(guar)), }; - track.prepare(WhichMatcher::Body); + track.prepare(WhichMatcher::Body, body); let result = tt_parser.parse_tt(&mut Cow::Borrowed(&body_parser), body, track); track.after_arm(&result); @@ -750,7 +749,7 @@ pub(super) fn try_match_macro_derive<'matcher, T: Tracker<'matcher>>( let mut gated_spans_snapshot = mem::take(&mut *psess.gated_spans.spans.borrow_mut()); - track.prepare(WhichMatcher::FOR_DERIVE); + track.prepare(WhichMatcher::FOR_DERIVE, body); let result = tt_parser.parse_tt(&mut Cow::Borrowed(&body_parser), body, track); track.after_arm(&result);