Support inner doc comments in proc_macro

[dtolnay]

An inner doc comment //! doc currently incorrectly tokenizes as an outer attribute, #[doc = "//! doc"]. It should be an inner attribute, #![doc = " doc"]. You can see the correct behavior in macro_rules:

macro_rules! tokens {
    (#![doc = $tt:tt]) => {
        println!("{:?}", $tt);
    }
}

fn main() {
    tokens! {
        //! doc
    }
}

Repro script

#!/bin/sh

cargo new --lib repro_macro
cargo new --lib repro

echo >repro_macro/src/lib.rs '
#![feature(proc_macro)]

extern crate proc_macro;
use proc_macro::TokenStream;

#[proc_macro]
pub fn repro(_input: TokenStream) -> TokenStream {
    let mut tts = "//! doc\n".parse::<TokenStream>().unwrap().into_iter();
    println!("{}", tts.next().unwrap());
    println!("{}", tts.next().unwrap());
    TokenStream::empty()
}
'

echo >>repro_macro/Cargo.toml '
[lib]
proc-macro = true
'

echo >repro/src/lib.rs '
#![feature(proc_macro)]

extern crate repro_macro;
repro_macro::repro!();
'

echo >>repro/Cargo.toml '
repro_macro = { path = "../repro_macro" }
'

cargo build --manifest-path repro/Cargo.toml

#
[ doc = "//! doc" ]

@alexcrichton

[alexcrichton]

I've added a fix for this in #49597