Add unstable --output-format option to "rustdoc"

We achieved this by:
   * Handle `--output-format` argument, accepting `html` or `json`
   * If `json` is passed, we append the following in
   `prepare_rustdoc`:
     	1. `-Zunstable-options`
	2. `--output-format`

Fixes rust-lang#12103

Signed-off-by: Charalampos Mitrodimas <charmitro@posteo.net>

src/bin/cargo/commands/doc.rs

@@ -49,13 +49,15 @@ pub fn exec(config: &mut Config, args: &ArgMatches) -> CliResult {
     let ws = args.workspace(config)?;
     let mode = CompileMode::Doc {
         deps: !args.flag("no-deps"),
+        json: false,
     };
     let mut compile_opts =
         args.compile_options(config, mode, Some(&ws), ProfileChecking::Custom)?;
     compile_opts.rustdoc_document_private_items = args.flag("document-private-items");
 
     let doc_opts = DocOptions {
         open_result: args.flag("open"),
+        output_format: ops::OutputFormat::Html,
         compile_opts,
     };
     ops::doc(&ws, &doc_opts)?;

src/bin/cargo/commands/rustdoc.rs

@@ -1,4 +1,6 @@
-use cargo::ops::{self, DocOptions};
+use std::str::FromStr;
+
+use cargo::ops::{self, DocOptions, OutputFormat};
 
 use crate::command_prelude::*;
 
@@ -38,6 +40,10 @@ pub fn cli() -> Command {
         .arg_profile("Build artifacts with the specified profile")
         .arg_target_triple("Build for the target triple")
         .arg_target_dir()
+        .arg(
+            opt("output-format", "the output type to write (unstable)")
+                .value_parser(["html", "json"]),
+        )
         .arg_unit_graph()
         .arg_timings()
         .arg_manifest_path()
@@ -48,20 +54,45 @@ pub fn cli() -> Command {
 
 pub fn exec(config: &mut Config, args: &ArgMatches) -> CliResult {
     let ws = args.workspace(config)?;
+    let output_format = if let Some(output_format) = args._value_of("output-format") {
+        if matches!(output_format, "json") {
+            config
+                .cli_unstable()
+                .fail_if_stable_opt("--output-format", 12103)?;
+        }
+        OutputFormat::from_str(output_format).unwrap()
+    } else {
+        OutputFormat::Html
+    };
+    let open_result = args.flag("open");
+
+    if open_result && matches!(output_format, OutputFormat::Json) {
+        return Err(CliError::new(
+            anyhow::format_err!("`--open` is not allowed with `json` output format."),
+            101,
+        ));
+    }
+
     let mut compile_opts = args.compile_options_for_single_package(
         config,
-        CompileMode::Doc { deps: false },
+        CompileMode::Doc {
+            deps: false,
+            json: matches!(output_format, OutputFormat::Json),
+        },
         Some(&ws),
         ProfileChecking::Custom,
     )?;
     let target_args = values(args, "args");
+
     compile_opts.target_rustdoc_args = if target_args.is_empty() {
         None
     } else {
         Some(target_args)
     };
+
     let doc_opts = DocOptions {
-        open_result: args.flag("open"),
+        open_result,
+        output_format: OutputFormat::Html,
         compile_opts,
     };
     ops::doc(&ws, &doc_opts)?;

src/cargo/core/compiler/build_config.rs

@@ -176,8 +176,10 @@ pub enum CompileMode {
     /// allows some de-duping of Units to occur.
     Bench,
     /// A target that will be documented with `rustdoc`.
+
     /// If `deps` is true, then it will also document all dependencies.
-    Doc { deps: bool },
+    /// if `json` is true, the documentation output is in json format.
+    Doc { deps: bool, json: bool },
     /// A target that will be tested with `rustdoc`.
     Doctest,
     /// An example or library that will be scraped for function calls by `rustdoc`.

src/cargo/core/compiler/context/compilation_files.rs

@@ -435,11 +435,16 @@ impl<'a, 'cfg: 'a> CompilationFiles<'a, 'cfg> {
         bcx: &BuildContext<'a, 'cfg>,
     ) -> CargoResult<Arc<Vec<OutputFile>>> {
         let ret = match unit.mode {
-            CompileMode::Doc { .. } => {
-                let path = self
-                    .out_dir(unit)
-                    .join(unit.target.crate_name())
-                    .join("index.html");
+            CompileMode::Doc { json, .. } => {
+                let path = if json {
+                    self.out_dir(unit)
+                        .join(format!("{}.json", unit.target.crate_name()))
+                } else {
+                    self.out_dir(unit)
+                        .join(unit.target.crate_name())
+                        .join("index.html")
+                };
+
                 vec![OutputFile {
                     path,
                     hardlink: None,

src/cargo/core/compiler/mod.rs

@@ -762,6 +762,8 @@ fn prepare_rustdoc(cx: &Context<'_, '_>, unit: &Unit) -> CargoResult<ProcessBuil
     build_deps_args(&mut rustdoc, cx, unit)?;
     rustdoc::add_root_urls(cx, unit, &mut rustdoc)?;
 
+    rustdoc::add_output_format(cx, unit, &mut rustdoc)?;
+
     rustdoc.args(bcx.rustdocflags_args(unit));
 
     if !crate_version_flag_already_present(&rustdoc) {

src/cargo/core/compiler/rustdoc.rs

@@ -11,6 +11,8 @@ use std::fmt;
 use std::hash;
 use url::Url;
 
+use super::CompileMode;
+
 const DOCS_RS_URL: &'static str = "https://docs.rs/";
 
 /// Mode used for `std`. This is for unstable feature [`-Zrustdoc-map`][1].
@@ -204,6 +206,29 @@ pub fn add_root_urls(
     Ok(())
 }
 
+/// Adds unstable flag [`--output-format`][1] to the given `rustdoc`
+/// invocation. This is for unstable feature [`-Zunstable-features`].
+///
+/// [1]: https://doc.rust-lang.org/nightly/rustdoc/unstable-features.html?highlight=output-format#-w--output-format-output-format
+pub fn add_output_format(
+    cx: &Context<'_, '_>,
+    unit: &Unit,
+    rustdoc: &mut ProcessBuilder,
+) -> CargoResult<()> {
+    let config = cx.bcx.config;
+    if !config.cli_unstable().unstable_options {
+        tracing::debug!("`unstable-options` is ignored, required -Zunstable-options flag");
+        return Ok(());
+    }
+
+    if let CompileMode::Doc { json: true, .. } = unit.mode {
+        rustdoc.arg("-Zunstable-options");
+        rustdoc.arg("--output-format=json");
+    }
+
+    Ok(())
+}
+
 /// Indicates whether a target should have examples scraped from it by rustdoc.
 /// Configured within Cargo.toml and only for unstable feature
 /// [`-Zrustdoc-scrape-examples`][1].

src/cargo/core/compiler/unit_dependencies.rs

@@ -627,7 +627,7 @@ fn compute_deps_doc(
         )?;
         ret.push(lib_unit_dep);
         if dep_lib.documented() {
-            if let CompileMode::Doc { deps: true } = unit.mode {
+            if let CompileMode::Doc { deps: true, .. } = unit.mode {
                 // Document this lib as well.
                 let doc_unit_dep = new_unit_dep(
                     state,

src/cargo/ops/cargo_compile/mod.rs

@@ -420,7 +420,7 @@ pub fn create_bcx<'a, 'cfg>(
 
     // TODO: In theory, Cargo should also dedupe the roots, but I'm uncertain
     // what heuristics to use in that case.
-    if build_config.mode == (CompileMode::Doc { deps: true }) {
+    if matches!(build_config.mode, CompileMode::Doc { deps: true, .. }) {
         remove_duplicate_doc(build_config, &units, &mut unit_graph);
     }
 

src/cargo/ops/cargo_doc.rs

@@ -5,12 +5,37 @@ use crate::util::CargoResult;
 use std::path::Path;
 use std::path::PathBuf;
 use std::process::Command;
+use std::str::FromStr;
+
+/// Format of rustdoc [`--output-format`][1].
+///
+/// [1]: https://doc.rust-lang.org/nightly/rustdoc/unstable-features.html#-w--output-format-output-format
+#[derive(Debug, Default, Clone)]
+pub enum OutputFormat {
+    #[default]
+    Html,
+    Json,
+}
+
+impl FromStr for OutputFormat {
+    type Err = &'static str;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "json" => Ok(Self::Json),
+            "html" => Ok(Self::Html),
+            _ => Err("Allowed values are for `--output-format` are `html` or `json`"),
+        }
+    }
+}
 
 /// Strongly typed options for the `cargo doc` command.
 #[derive(Debug)]
 pub struct DocOptions {
     /// Whether to attempt to open the browser after compiling the docs
     pub open_result: bool,
+    /// Same as `rustdoc --output-format`
+    pub output_format: OutputFormat,
     /// Options to pass through to the compiler
     pub compile_opts: ops::CompileOptions,
 }
@@ -19,7 +44,7 @@ pub struct DocOptions {
 pub fn doc(ws: &Workspace<'_>, options: &DocOptions) -> CargoResult<()> {
     let compilation = ops::compile(ws, &options.compile_opts)?;
 
-    if options.open_result {
+    if options.open_result && matches!(options.output_format, OutputFormat::Html) {
         let name = &compilation
             .root_crate_names
             .get(0)

src/cargo/ops/mod.rs

@@ -5,7 +5,7 @@ pub use self::cargo_compile::{
     compile, compile_with_exec, compile_ws, create_bcx, print, resolve_all_features, CompileOptions,
 };
 pub use self::cargo_compile::{CompileFilter, FilterRule, LibRule, Packages};
-pub use self::cargo_doc::{doc, DocOptions};
+pub use self::cargo_doc::{doc, DocOptions, OutputFormat};
 pub use self::cargo_fetch::{fetch, FetchOptions};
 pub use self::cargo_generate_lockfile::generate_lockfile;
 pub use self::cargo_generate_lockfile::update_lockfile;

src/doc/man/cargo-rustdoc.md

@@ -102,6 +102,7 @@ if its name is the same as the lib target. Binaries are skipped if they have
 {{#options}}
 {{> options-jobs }}
 {{> options-keep-going }}
+{{> options-output-format }}
 {{/options}}
 
 {{> section-environment }}

src/doc/man/generated_txt/cargo-rustdoc.txt

@@ -327,6 +327,18 @@ OPTIONS
            --keep-going would definitely run both builds, even if the one run
            first fails.
 
+       --output-format
+           The output type for the documentation emitted. Valid values:
+
+           o  html (default): Emit the documentation in HTML format.
+
+           o  json: Emit the documentation in the experimental JSON format
+              <https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc_json_types>.
+
+           This option is only available on the nightly channel
+           <https://doc.rust-lang.org/book/appendix-07-nightly-rust.html> and
+           requires the -Z unstable-options flag to enable.
+
 ENVIRONMENT
        See the reference
        <https://doc.rust-lang.org/cargo/reference/environment-variables.html>

src/doc/man/includes/options-output-format.md

@@ -0,0 +1,9 @@
+{{#option "`--output-format`"}}
+The output type for the documentation emitted. Valid values:
+
+* `html` (default): Emit the documentation in HTML format.
+* `json`: Emit the documentation in the [experimental JSON format](https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc_json_types).
+
+This option is only available on the [nightly channel](https://doc.rust-lang.org/book/appendix-07-nightly-rust.html)
+and requires the `-Z unstable-options` flag to enable.
+{{/option}}

src/doc/src/commands/cargo-rustdoc.md

@@ -371,6 +371,15 @@ one that succeeds (depending on which one of the two builds Cargo picked to run
 first), whereas <code>cargo rustdoc -j1 --keep-going</code> would definitely run both
 builds, even if the one run first fails.</dd>
 
+<dt class="option-term" id="option-cargo-rustdoc---output-format"><a class="option-anchor" href="#option-cargo-rustdoc---output-format"></a><code>--output-format</code></dt>
+<dd class="option-desc">The output type for the documentation emitted. Valid values:</p>
+<ul>
+<li><code>html</code> (default): Emit the documentation in HTML format.</li>
+<li><code>json</code>: Emit the documentation in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc_json_types">experimental JSON format</a>.</li>
+</ul>
+<p>This option is only available on the <a href="https://doc.rust-lang.org/book/appendix-07-nightly-rust.html">nightly channel</a>
+and requires the <code>-Z unstable-options</code> flag to enable.</dd>
+
 </dl>
 
 ## ENVIRONMENT

src/etc/man/cargo-rustdoc.1

@@ -395,6 +395,22 @@ one that succeeds (depending on which one of the two builds Cargo picked to run
 first), whereas \fBcargo rustdoc \-j1 \-\-keep\-going\fR would definitely run both
 builds, even if the one run first fails.
 .RE
+.sp
+\fB\-\-output\-format\fR
+.RS 4
+The output type for the documentation emitted. Valid values:
+.sp
+.RS 4
+\h'-04'\(bu\h'+02'\fBhtml\fR (default): Emit the documentation in HTML format.
+.RE
+.sp
+.RS 4
+\h'-04'\(bu\h'+02'\fBjson\fR: Emit the documentation in the \fIexperimental JSON format\fR <https://doc.rust\-lang.org/nightly/nightly\-rustc/rustdoc_json_types>\&.
+.RE
+.sp
+This option is only available on the \fInightly channel\fR <https://doc.rust\-lang.org/book/appendix\-07\-nightly\-rust.html>
+and requires the \fB\-Z unstable\-options\fR flag to enable.
+.RE
 .SH "ENVIRONMENT"
 See \fIthe reference\fR <https://doc.rust\-lang.org/cargo/reference/environment\-variables.html> for
 details on environment variables that Cargo reads.

tests/testsuite/cargo_rustdoc/help/stdout.log

@@ -6,16 +6,18 @@ Arguments:
   [ARGS]...  Extra rustdoc flags
 
 Options:
-      --open                  Opens the docs in a browser after the operation
-      --ignore-rust-version   Ignore `rust-version` specification in packages
-      --message-format <FMT>  Error format
-  -v, --verbose...            Use verbose output (-vv very verbose/build.rs output)
-  -q, --quiet                 Do not print cargo log messages
-      --color <WHEN>          Coloring: auto, always, never
-      --config <KEY=VALUE>    Override a configuration value
-  -Z <FLAG>                   Unstable (nightly-only) flags to Cargo, see 'cargo -Z help' for
-                              details
-  -h, --help                  Print help
+      --open                           Opens the docs in a browser after the operation
+      --ignore-rust-version            Ignore `rust-version` specification in packages
+      --message-format <FMT>           Error format
+      --output-format <output-format>  the output type to write (unstable) [possible values: html,
+                                       json]
+  -v, --verbose...                     Use verbose output (-vv very verbose/build.rs output)
+  -q, --quiet                          Do not print cargo log messages
+      --color <WHEN>                   Coloring: auto, always, never
+      --config <KEY=VALUE>             Override a configuration value
+  -Z <FLAG>                            Unstable (nightly-only) flags to Cargo, see 'cargo -Z help'
+                                       for details
+  -h, --help                           Print help
 
 Package Selection:
   -p, --package [<SPEC>]  Package to document