From fdff7defc9a21fc63a1586efab87694723144793 Mon Sep 17 00:00:00 2001
From: Tyler Mandry <tmandry@google.com>
Date: Thu, 10 Sep 2020 02:18:46 +0000
Subject: [PATCH] Revert "Rollup merge of #76285 - matklad:censor-spacing,
 r=petrochenkov"

This reverts commit 85cee57fd791d670d92dc61e0ad71594128dd45a, reversing
changes made to b4d387302416c90a3f70211770292d8d8ab5e07d.
---
 compiler/rustc_ast/src/tokenstream.rs         |  4 ++--
 .../rustc_expand/src/proc_macro_server.rs     | 22 +++++--------------
 compiler/rustc_parse/src/lexer/tokentrees.rs  |  5 ++++-
 compiler/rustc_parse/src/parser/mod.rs        | 10 ++++-----
 4 files changed, 16 insertions(+), 25 deletions(-)

diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs
index fb98f55a2154a..151acddae840e 100644
--- a/compiler/rustc_ast/src/tokenstream.rs
+++ b/compiler/rustc_ast/src/tokenstream.rs
@@ -403,8 +403,8 @@ impl Cursor {
         self.index = index;
     }
 
-    pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
-        self.stream.0[self.index..].get(n).map(|(tree, _)| tree)
+    pub fn look_ahead(&self, n: usize) -> Option<TokenTree> {
+        self.stream.0[self.index..].get(n).map(|(tree, _)| tree.clone())
     }
 }
 
diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs
index 765871a6396f3..39c82f97e0a39 100644
--- a/compiler/rustc_expand/src/proc_macro_server.rs
+++ b/compiler/rustc_expand/src/proc_macro_server.rs
@@ -47,26 +47,15 @@ impl ToInternal<token::DelimToken> for Delimiter {
     }
 }
 
-impl
-    FromInternal<(
-        TreeAndJoint,
-        Option<&'_ tokenstream::TokenTree>,
-        &'_ ParseSess,
-        &'_ mut Vec<Self>,
-    )> for TokenTree<Group, Punct, Ident, Literal>
+impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
+    for TokenTree<Group, Punct, Ident, Literal>
 {
     fn from_internal(
-        ((tree, is_joint), look_ahead, sess, stack): (
-            TreeAndJoint,
-            Option<&tokenstream::TokenTree>,
-            &ParseSess,
-            &mut Vec<Self>,
-        ),
+        ((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec<Self>),
     ) -> Self {
         use rustc_ast::token::*;
 
-        let joint = is_joint == Joint
-            && matches!(look_ahead, Some(tokenstream::TokenTree::Token(t)) if t.is_op());
+        let joint = is_joint == Joint;
         let Token { kind, span } = match tree {
             tokenstream::TokenTree::Delimited(span, delim, tts) => {
                 let delimiter = Delimiter::from_internal(delim);
@@ -456,8 +445,7 @@ impl server::TokenStreamIter for Rustc<'_> {
         loop {
             let tree = iter.stack.pop().or_else(|| {
                 let next = iter.cursor.next_with_joint()?;
-                let lookahead = iter.cursor.look_ahead(0);
-                Some(TokenTree::from_internal((next, lookahead, self.sess, &mut iter.stack)))
+                Some(TokenTree::from_internal((next, self.sess, &mut iter.stack)))
             })?;
             // A hack used to pass AST fragments to attribute and derive macros
             // as a single nonterminal token instead of a token stream.
diff --git a/compiler/rustc_parse/src/lexer/tokentrees.rs b/compiler/rustc_parse/src/lexer/tokentrees.rs
index fb27ccfbd9429..d5977ca3c7d2f 100644
--- a/compiler/rustc_parse/src/lexer/tokentrees.rs
+++ b/compiler/rustc_parse/src/lexer/tokentrees.rs
@@ -262,7 +262,10 @@ impl<'a> TokenTreesReader<'a> {
             }
             _ => {
                 let tt = TokenTree::Token(self.token.take());
-                let is_joint = self.bump();
+                let mut is_joint = self.bump();
+                if !self.token.is_op() {
+                    is_joint = NonJoint;
+                }
                 Ok((tt, is_joint))
             }
         }
diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs
index 1b2067f8f256b..84edfecad192f 100644
--- a/compiler/rustc_parse/src/parser/mod.rs
+++ b/compiler/rustc_parse/src/parser/mod.rs
@@ -822,15 +822,15 @@ impl<'a> Parser<'a> {
         }
 
         let frame = &self.token_cursor.frame;
-        match frame.tree_cursor.look_ahead(dist - 1) {
+        looker(&match frame.tree_cursor.look_ahead(dist - 1) {
             Some(tree) => match tree {
-                TokenTree::Token(token) => looker(token),
+                TokenTree::Token(token) => token,
                 TokenTree::Delimited(dspan, delim, _) => {
-                    looker(&Token::new(token::OpenDelim(delim.clone()), dspan.open))
+                    Token::new(token::OpenDelim(delim), dspan.open)
                 }
             },
-            None => looker(&Token::new(token::CloseDelim(frame.delim), frame.span.close)),
-        }
+            None => Token::new(token::CloseDelim(frame.delim), frame.span.close),
+        })
     }
 
     /// Returns whether any of the given keywords are `dist` tokens ahead of the current one.