From a86fc727fa9b9fa1ac60b67147736783b3376e91 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 1 Feb 2023 12:58:04 +1100 Subject: [PATCH] Rename `Cursor`/`CursorRef` as `TokenTreeCursor`/`RefTokenTreeCursor`. This makes it clear they return token trees, and makes for a nice comparison against `TokenCursor` which returns tokens. --- compiler/rustc_ast/src/tokenstream.rs | 30 ++++++++++--------- compiler/rustc_expand/src/mbe/metavar_expr.rs | 12 ++++---- compiler/rustc_parse/src/parser/mod.rs | 13 ++++---- src/tools/rustfmt/src/macros.rs | 8 ++--- 4 files changed, 34 insertions(+), 29 deletions(-) diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index ab554d70c4623..f0a6a5e072586 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -389,12 +389,12 @@ impl TokenStream { self.0.len() } - pub fn trees(&self) -> CursorRef<'_> { - CursorRef::new(self) + pub fn trees(&self) -> RefTokenTreeCursor<'_> { + RefTokenTreeCursor::new(self) } - pub fn into_trees(self) -> Cursor { - Cursor::new(self) + pub fn into_trees(self) -> TokenTreeCursor { + TokenTreeCursor::new(self) } /// Compares two `TokenStream`s, checking equality without regarding span information. @@ -552,16 +552,17 @@ impl TokenStream { } } -/// By-reference iterator over a [`TokenStream`]. +/// By-reference iterator over a [`TokenStream`], that produces `&TokenTree` +/// items. #[derive(Clone)] -pub struct CursorRef<'t> { +pub struct RefTokenTreeCursor<'t> { stream: &'t TokenStream, index: usize, } -impl<'t> CursorRef<'t> { +impl<'t> RefTokenTreeCursor<'t> { fn new(stream: &'t TokenStream) -> Self { - CursorRef { stream, index: 0 } + RefTokenTreeCursor { stream, index: 0 } } pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> { @@ -569,7 +570,7 @@ impl<'t> CursorRef<'t> { } } -impl<'t> Iterator for CursorRef<'t> { +impl<'t> Iterator for RefTokenTreeCursor<'t> { type Item = &'t TokenTree; fn next(&mut self) -> Option<&'t TokenTree> { @@ -580,15 +581,16 @@ impl<'t> Iterator for CursorRef<'t> { } } -/// Owning by-value iterator over a [`TokenStream`]. +/// Owning by-value iterator over a [`TokenStream`], that produces `TokenTree` +/// items. // FIXME: Many uses of this can be replaced with by-reference iterator to avoid clones. #[derive(Clone)] -pub struct Cursor { +pub struct TokenTreeCursor { pub stream: TokenStream, index: usize, } -impl Iterator for Cursor { +impl Iterator for TokenTreeCursor { type Item = TokenTree; fn next(&mut self) -> Option { @@ -599,9 +601,9 @@ impl Iterator for Cursor { } } -impl Cursor { +impl TokenTreeCursor { fn new(stream: TokenStream) -> Self { - Cursor { stream, index: 0 } + TokenTreeCursor { stream, index: 0 } } #[inline] diff --git a/compiler/rustc_expand/src/mbe/metavar_expr.rs b/compiler/rustc_expand/src/mbe/metavar_expr.rs index 99fe474541e9d..de34df0114a74 100644 --- a/compiler/rustc_expand/src/mbe/metavar_expr.rs +++ b/compiler/rustc_expand/src/mbe/metavar_expr.rs @@ -1,5 +1,5 @@ use rustc_ast::token::{self, Delimiter}; -use rustc_ast::tokenstream::{CursorRef, TokenStream, TokenTree}; +use rustc_ast::tokenstream::{RefTokenTreeCursor, TokenStream, TokenTree}; use rustc_ast::{LitIntType, LitKind}; use rustc_ast_pretty::pprust; use rustc_errors::{Applicability, PResult}; @@ -72,7 +72,7 @@ impl MetaVarExpr { // Checks if there are any remaining tokens. For example, `${ignore(ident ... a b c ...)}` fn check_trailing_token<'sess>( - iter: &mut CursorRef<'_>, + iter: &mut RefTokenTreeCursor<'_>, sess: &'sess ParseSess, ) -> PResult<'sess, ()> { if let Some(tt) = iter.next() { @@ -88,7 +88,7 @@ fn check_trailing_token<'sess>( /// Parse a meta-variable `count` expression: `count(ident[, depth])` fn parse_count<'sess>( - iter: &mut CursorRef<'_>, + iter: &mut RefTokenTreeCursor<'_>, sess: &'sess ParseSess, span: Span, ) -> PResult<'sess, MetaVarExpr> { @@ -99,7 +99,7 @@ fn parse_count<'sess>( /// Parses the depth used by index(depth) and length(depth). fn parse_depth<'sess>( - iter: &mut CursorRef<'_>, + iter: &mut RefTokenTreeCursor<'_>, sess: &'sess ParseSess, span: Span, ) -> PResult<'sess, usize> { @@ -126,7 +126,7 @@ fn parse_depth<'sess>( /// Parses an generic ident fn parse_ident<'sess>( - iter: &mut CursorRef<'_>, + iter: &mut RefTokenTreeCursor<'_>, sess: &'sess ParseSess, span: Span, ) -> PResult<'sess, Ident> { @@ -152,7 +152,7 @@ fn parse_ident<'sess>( /// Tries to move the iterator forward returning `true` if there is a comma. If not, then the /// iterator is not modified and the result is `false`. -fn try_eat_comma(iter: &mut CursorRef<'_>) -> bool { +fn try_eat_comma(iter: &mut RefTokenTreeCursor<'_>) -> bool { if let Some(TokenTree::Token(token::Token { kind: token::Comma, .. }, _)) = iter.look_ahead(0) { let _ = iter.next(); return true; diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 982fde727c898..2ea55f838a37e 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -19,9 +19,8 @@ pub use path::PathStyle; use rustc_ast::ptr::P; use rustc_ast::token::{self, Delimiter, Nonterminal, Token, TokenKind}; -use rustc_ast::tokenstream::AttributesData; -use rustc_ast::tokenstream::{self, DelimSpan, Spacing}; -use rustc_ast::tokenstream::{TokenStream, TokenTree}; +use rustc_ast::tokenstream::{AttributesData, DelimSpan, Spacing}; +use rustc_ast::tokenstream::{TokenStream, TokenTree, TokenTreeCursor}; use rustc_ast::util::case::Case; use rustc_ast::AttrId; use rustc_ast::DUMMY_NODE_ID; @@ -221,17 +220,21 @@ impl<'a> Drop for Parser<'a> { } } +/// Iterator over a `TokenStream` that produces `Token`s. It's a bit odd that +/// we (a) lex tokens into a nice tree structure (`TokenStream`), and then (b) +/// use this type to emit them as a linear sequence. But a linear sequence is +/// what the parser expects, for the most part. #[derive(Clone)] struct TokenCursor { // Cursor for the current (innermost) token stream. The delimiters for this // token stream are found in `self.stack.last()`; when that is `None` then // we are in the outermost token stream which never has delimiters. - tree_cursor: tokenstream::Cursor, + tree_cursor: TokenTreeCursor, // Token streams surrounding the current one. The delimiters for stack[n]'s // tokens are in `stack[n-1]`. `stack[0]` (when present) has no delimiters // because it's the outermost token stream which never has delimiters. - stack: Vec<(tokenstream::Cursor, Delimiter, DelimSpan)>, + stack: Vec<(TokenTreeCursor, Delimiter, DelimSpan)>, desugar_doc_comments: bool, diff --git a/src/tools/rustfmt/src/macros.rs b/src/tools/rustfmt/src/macros.rs index d58f7547fefb3..7978d8cba9541 100644 --- a/src/tools/rustfmt/src/macros.rs +++ b/src/tools/rustfmt/src/macros.rs @@ -13,7 +13,7 @@ use std::collections::HashMap; use std::panic::{catch_unwind, AssertUnwindSafe}; use rustc_ast::token::{BinOpToken, Delimiter, Token, TokenKind}; -use rustc_ast::tokenstream::{Cursor, TokenStream, TokenTree}; +use rustc_ast::tokenstream::{TokenStream, TokenTree, TokenTreeCursor}; use rustc_ast::{ast, ptr}; use rustc_ast_pretty::pprust; use rustc_span::{ @@ -736,7 +736,7 @@ impl MacroArgParser { self.buf.clear(); } - fn add_meta_variable(&mut self, iter: &mut Cursor) -> Option<()> { + fn add_meta_variable(&mut self, iter: &mut TokenTreeCursor) -> Option<()> { match iter.next() { Some(TokenTree::Token( Token { @@ -768,7 +768,7 @@ impl MacroArgParser { &mut self, inner: Vec, delim: Delimiter, - iter: &mut Cursor, + iter: &mut TokenTreeCursor, ) -> Option<()> { let mut buffer = String::new(); let mut first = true; @@ -1121,7 +1121,7 @@ pub(crate) fn macro_style(mac: &ast::MacCall, context: &RewriteContext<'_>) -> D // Currently we do not attempt to parse any further than that. #[derive(new)] struct MacroParser { - toks: Cursor, + toks: TokenTreeCursor, } impl MacroParser {