Auto merge of #44781 - QuietMisdreavus:doc-include, r=GuillaumeGomez

rustdoc: include external files in documentation (RFC 1990)

Part of rust-lang/rfcs#1990 (needs work on the error reporting, which i'm deferring to after this initial PR)

cc #44732

Also fixes #42760, because the prep work for the error reporting made it easy to fix that at the same time.

src/doc/unstable-book/src/language-features/external-doc.md

@@ -0,0 +1,40 @@
+# `external_doc`
+
+The tracking issue for this feature is: [#44732]
+
+The `external_doc` feature allows the use of the `include` parameter to the `#[doc]` attribute, to
+include external files in documentation. Use the attribute in place of, or in addition to, regular
+doc comments and `#[doc]` attributes, and `rustdoc` will load the given file when it renders
+documentation for your crate.
+
+With the following files in the same directory:
+
+`external-doc.md`:
+
+```markdown
+# My Awesome Type
+
+This is the documentation for this spectacular type.
+```
+
+`lib.rs`:
+
+```no_run (needs-external-files)
+#![feature(external_doc)]
+
+#[doc(include = "external-doc.md")]
+pub struct MyAwesomeType;
+```
+
+`rustdoc` will load the file `external-doc.md` and use it as the documentation for the `MyAwesomeType`
+struct.
+
+When locating files, `rustdoc` will base paths in the `src/` directory, as if they were alongside the
+`lib.rs` for your crate. So if you want a `docs/` folder to live alongside the `src/` directory,
+start your paths with `../docs/` for `rustdoc` to properly find the file.
+
+This feature was proposed in [RFC #1990] and initially implemented in PR [#44781].
+
+[#44732]: https://github.com/rust-lang/rust/issues/44732
+[RFC #1990]: https://github.com/rust-lang/rfcs/pull/1990
+[#44781]: https://github.com/rust-lang/rust/pull/44781

src/librustdoc/clean/mod.rs

@@ -44,6 +44,7 @@ use rustc::hir;
 
 use rustc_const_math::ConstInt;
 use std::{mem, slice, vec};
+use std::iter::FromIterator;
 use std::path::PathBuf;
 use std::rc::Rc;
 use std::sync::Arc;
@@ -300,6 +301,11 @@ impl Item {
     pub fn doc_value<'a>(&'a self) -> Option<&'a str> {
         self.attrs.doc_value()
     }
+    /// Finds all `doc` attributes as NameValues and returns their corresponding values, joined
+    /// with newlines.
+    pub fn collapsed_doc_value(&self) -> Option<String> {
+        self.attrs.collapsed_doc_value()
+    }
     pub fn is_crate(&self) -> bool {
         match self.inner {
             StrippedItem(box ModuleItem(Module { is_crate: true, ..})) |
@@ -564,9 +570,69 @@ impl<I: IntoIterator<Item=ast::NestedMetaItem>> NestedAttributesExt for I {
     }
 }
 
+/// A portion of documentation, extracted from a `#[doc]` attribute.
+///
+/// Each variant contains the line number within the complete doc-comment where the fragment
+/// starts, as well as the Span where the corresponding doc comment or attribute is located.
+///
+/// Included files are kept separate from inline doc comments so that proper line-number
+/// information can be given when a doctest fails. Sugared doc comments and "raw" doc comments are
+/// kept separate because of issue #42760.
+#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Debug)]
+pub enum DocFragment {
+    // FIXME #44229 (misdreavus): sugared and raw doc comments can be brought back together once
+    // hoedown is completely removed from rustdoc.
+    /// A doc fragment created from a `///` or `//!` doc comment.
+    SugaredDoc(usize, syntax_pos::Span, String),
+    /// A doc fragment created from a "raw" `#[doc=""]` attribute.
+    RawDoc(usize, syntax_pos::Span, String),
+    /// A doc fragment created from a `#[doc(include="filename")]` attribute. Contains both the
+    /// given filename and the file contents.
+    Include(usize, syntax_pos::Span, String, String),
+}
+
+impl DocFragment {
+    pub fn as_str(&self) -> &str {
+        match *self {
+            DocFragment::SugaredDoc(_, _, ref s) => &s[..],
+            DocFragment::RawDoc(_, _, ref s) => &s[..],
+            DocFragment::Include(_, _, _, ref s) => &s[..],
+        }
+    }
+
+    pub fn span(&self) -> syntax_pos::Span {
+        match *self {
+            DocFragment::SugaredDoc(_, span, _) |
+                DocFragment::RawDoc(_, span, _) |
+                DocFragment::Include(_, span, _, _) => span,
+        }
+    }
+}
+
+impl<'a> FromIterator<&'a DocFragment> for String {
+    fn from_iter<T>(iter: T) -> Self
+    where
+        T: IntoIterator<Item = &'a DocFragment>
+    {
+        iter.into_iter().fold(String::new(), |mut acc, frag| {
+            if !acc.is_empty() {
+                acc.push('\n');
+            }
+            match *frag {
+                DocFragment::SugaredDoc(_, _, ref docs)
+                    | DocFragment::RawDoc(_, _, ref docs)
+                    | DocFragment::Include(_, _, _, ref docs) =>
+                    acc.push_str(docs),
+            }
+
+            acc
+        })
+    }
+}
+
 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Debug, Default)]
 pub struct Attributes {
-    pub doc_strings: Vec<String>,
+    pub doc_strings: Vec<DocFragment>,
     pub other_attrs: Vec<ast::Attribute>,
     pub cfg: Option<Rc<Cfg>>,
     pub span: Option<syntax_pos::Span>,
@@ -596,6 +662,47 @@ impl Attributes {
         None
     }
 
+    /// Reads a `MetaItem` from within an attribute, looks for whether it is a
+    /// `#[doc(include="file")]`, and returns the filename and contents of the file as loaded from
+    /// its expansion.
+    fn extract_include(mi: &ast::MetaItem)
+        -> Option<(String, String)>
+    {
+        mi.meta_item_list().and_then(|list| {
+            for meta in list {
+                if meta.check_name("include") {
+                    // the actual compiled `#[doc(include="filename")]` gets expanded to
+                    // `#[doc(include(file="filename", contents="file contents")]` so we need to
+                    // look for that instead
+                    return meta.meta_item_list().and_then(|list| {
+                        let mut filename: Option<String> = None;
+                        let mut contents: Option<String> = None;
+
+                        for it in list {
+                            if it.check_name("file") {
+                                if let Some(name) = it.value_str() {
+                                    filename = Some(name.to_string());
+                                }
+                            } else if it.check_name("contents") {
+                                if let Some(docs) = it.value_str() {
+                                    contents = Some(docs.to_string());
+                                }
+                            }
+                        }
+
+                        if let (Some(filename), Some(contents)) = (filename, contents) {
+                            Some((filename, contents))
+                        } else {
+                            None
+                        }
+                    });
+                }
+            }
+
+            None
+        })
+    }
+
     pub fn has_doc_flag(&self, flag: &str) -> bool {
         for attr in &self.other_attrs {
             if !attr.check_name("doc") { continue; }
@@ -610,18 +717,29 @@ impl Attributes {
         false
     }
 
-    pub fn from_ast(diagnostic: &::errors::Handler, attrs: &[ast::Attribute]) -> Attributes {
+    pub fn from_ast(diagnostic: &::errors::Handler,
+                    attrs: &[ast::Attribute]) -> Attributes {
         let mut doc_strings = vec![];
         let mut sp = None;
         let mut cfg = Cfg::True;
+        let mut doc_line = 0;
 
         let other_attrs = attrs.iter().filter_map(|attr| {
             attr.with_desugared_doc(|attr| {
                 if attr.check_name("doc") {
                     if let Some(mi) = attr.meta() {
                         if let Some(value) = mi.value_str() {
                             // Extracted #[doc = "..."]
-                            doc_strings.push(value.to_string());
+                            let value = value.to_string();
+                            let line = doc_line;
+                            doc_line += value.lines().count();
+
+                            if attr.is_sugared_doc {
+                                doc_strings.push(DocFragment::SugaredDoc(line, attr.span, value));
+                            } else {
+                                doc_strings.push(DocFragment::RawDoc(line, attr.span, value));
+                            }
+
                             if sp.is_none() {
                                 sp = Some(attr.span);
                             }
@@ -633,6 +751,14 @@ impl Attributes {
                                 Err(e) => diagnostic.span_err(e.span, e.msg),
                             }
                             return None;
+                        } else if let Some((filename, contents)) = Attributes::extract_include(&mi)
+                        {
+                            let line = doc_line;
+                            doc_line += contents.lines().count();
+                            doc_strings.push(DocFragment::Include(line,
+                                                                  attr.span,
+                                                                  filename,
+                                                                  contents));
                         }
                     }
                 }
@@ -650,7 +776,17 @@ impl Attributes {
     /// Finds the `doc` attribute as a NameValue and returns the corresponding
     /// value found.
     pub fn doc_value<'a>(&'a self) -> Option<&'a str> {
-        self.doc_strings.first().map(|s| &s[..])
+        self.doc_strings.first().map(|s| s.as_str())
+    }
+
+    /// Finds all `doc` attributes as NameValues and returns their corresponding values, joined
+    /// with newlines.
+    pub fn collapsed_doc_value(&self) -> Option<String> {
+        if !self.doc_strings.is_empty() {
+            Some(self.doc_strings.iter().collect())
+        } else {
+            None
+        }
     }
 }
 

src/librustdoc/html/render.rs

@@ -36,6 +36,7 @@ pub use self::ExternalLocation::*;
 
 #[cfg(stage0)]
 use std::ascii::AsciiExt;
+use std::borrow::Cow;
 use std::cell::RefCell;
 use std::cmp::Ordering;
 use std::collections::{BTreeMap, HashSet};
@@ -143,6 +144,23 @@ impl SharedContext {
     }
 }
 
+impl SharedContext {
+    /// Returns whether the `collapse-docs` pass was run on this crate.
+    pub fn was_collapsed(&self) -> bool {
+        self.passes.contains("collapse-docs")
+    }
+
+    /// Based on whether the `collapse-docs` pass was run, return either the `doc_value` or the
+    /// `collapsed_doc_value` of the given item.
+    pub fn maybe_collapsed_doc_value<'a>(&self, item: &'a clean::Item) -> Option<Cow<'a, str>> {
+        if self.was_collapsed() {
+            item.collapsed_doc_value().map(|s| s.into())
+        } else {
+            item.doc_value().map(|s| s.into())
+        }
+    }
+}
+
 /// Indicates where an external crate can be found.
 pub enum ExternalLocation {
     /// Remote URL root of the external crate
@@ -1817,6 +1835,9 @@ fn plain_summary_line(s: Option<&str>) -> String {
 }
 
 fn document(w: &mut fmt::Formatter, cx: &Context, item: &clean::Item) -> fmt::Result {
+    if let Some(ref name) = item.name {
+        info!("Documenting {}", name);
+    }
     document_stability(w, cx, item)?;
     let prefix = render_assoc_const_value(item);
     document_full(w, item, cx, &prefix)?;
@@ -1893,8 +1914,9 @@ fn render_assoc_const_value(item: &clean::Item) -> String {
 
 fn document_full(w: &mut fmt::Formatter, item: &clean::Item,
                  cx: &Context, prefix: &str) -> fmt::Result {
-    if let Some(s) = item.doc_value() {
-        render_markdown(w, s, item.source.clone(), cx.render_type, prefix, &cx.shared)?;
+    if let Some(s) = cx.shared.maybe_collapsed_doc_value(item) {
+        debug!("Doc block: =====\n{}\n=====", s);
+        render_markdown(w, &*s, item.source.clone(), cx.render_type, prefix, &cx.shared)?;
     } else if !prefix.is_empty() {
         write!(w, "<div class='docblock'>{}</div>", prefix)?;
     }
@@ -3326,8 +3348,8 @@ fn render_impl(w: &mut fmt::Formatter, cx: &Context, i: &Impl, link: AssocItemLi
         }
         write!(w, "</span>")?;
         write!(w, "</h3>\n")?;
-        if let Some(ref dox) = i.impl_item.doc_value() {
-            write!(w, "<div class='docblock'>{}</div>", Markdown(dox, cx.render_type))?;
+        if let Some(ref dox) = cx.shared.maybe_collapsed_doc_value(&i.impl_item) {
+            write!(w, "<div class='docblock'>{}</div>", Markdown(&*dox, cx.render_type))?;
         }
     }