From f42407f11401eecccda3a641a59c0a60f46b957e Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Sun, 16 Dec 2018 02:35:39 +0100 Subject: [PATCH 1/3] Reduce search-index.js size --- Cargo.lock | 6 +++--- src/librustdoc/Cargo.toml | 2 +- src/librustdoc/html/render.rs | 5 ++++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d659e0c19a8f0..92186d1700e53 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1332,7 +1332,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "minifier" -version = "0.0.20" +version = "0.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "macro-utils 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2692,7 +2692,7 @@ dependencies = [ name = "rustdoc" version = "0.0.0" dependencies = [ - "minifier 0.0.20 (registry+https://github.com/rust-lang/crates.io-index)", + "minifier 0.0.21 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "pulldown-cmark 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "tempfile 3.0.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3510,7 +3510,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum memchr 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0a3eb002f0535929f1199681417029ebea04aadc0c7a4224b46be99c7f5d6a16" "checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff" "checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3" -"checksum minifier 0.0.20 (registry+https://github.com/rust-lang/crates.io-index)" = "96c269bb45c39b333392b2b18ad71760b34ac65666591386b0e959ed58b3f474" +"checksum minifier 0.0.21 (registry+https://github.com/rust-lang/crates.io-index)" = "9707d0ff1b828cba09c4bb27d5c3dceb6f49bd46f700042343bea350a131bf4f" "checksum miniz-sys 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "0300eafb20369952951699b68243ab4334f4b10a88f411c221d444b36c40e649" "checksum miniz_oxide 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5ad30a47319c16cde58d0314f5d98202a80c9083b5f61178457403dfb14e509c" "checksum miniz_oxide_c_api 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "28edaef377517fd9fe3e085c37d892ce7acd1fbeab9239c5a36eec352d8a8b7e" diff --git a/src/librustdoc/Cargo.toml b/src/librustdoc/Cargo.toml index 8bac007b748ac..bc6447d74f1b2 100644 --- a/src/librustdoc/Cargo.toml +++ b/src/librustdoc/Cargo.toml @@ -9,6 +9,6 @@ path = "lib.rs" [dependencies] pulldown-cmark = { version = "0.1.2", default-features = false } -minifier = "0.0.20" +minifier = "0.0.21" tempfile = "3" parking_lot = "0.6.4" diff --git a/src/librustdoc/html/render.rs b/src/librustdoc/html/render.rs index 31e06cb1a045f..2552c97179401 100644 --- a/src/librustdoc/html/render.rs +++ b/src/librustdoc/html/render.rs @@ -1223,7 +1223,10 @@ fn write_minify_replacer(dst: &mut W, -> io::Result<()> { if enable_minification { writeln!(dst, "{}", - minifier::js::minify_and_replace_keywords(contents, keywords_to_replace)) + minifier::js::minify_and_replace_keywords(contents, keywords_to_replace) + .apply(minifier::js::clean_tokens) + .apply(minifier::js::aggregate_strings) + .to_string()) } else { writeln!(dst, "{}", contents) } From 2e8fd44598f730d6a79e5fdabefcb26034fc093a Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 10 Jan 2019 21:26:41 +0100 Subject: [PATCH 2/3] Minify search-index in one pass --- src/librustdoc/html/render.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/librustdoc/html/render.rs b/src/librustdoc/html/render.rs index 2552c97179401..26512d93655f4 100644 --- a/src/librustdoc/html/render.rs +++ b/src/librustdoc/html/render.rs @@ -1057,11 +1057,11 @@ themePicker.onblur = handleThemeButtonsBlur; all_indexes.sort(); let mut w = try_err!(File::create(&dst), &dst); try_err!(writeln!(&mut w, "var N = null;var searchIndex = {{}};"), &dst); - for index in &all_indexes { - try_err!(write_minify_replacer(&mut w, &*index, options.enable_minification, - &[(minifier::js::Keyword::Null, "N")]), - &dst); - } + try_err!(write_minify_replacer(&mut w, + &all_indexes.join("\n"), + options.enable_minification, + &[(minifier::js::Keyword::Null, "N")]), + &dst); try_err!(writeln!(&mut w, "initSearch(searchIndex);addSearchOptions(searchIndex);"), &dst); if options.enable_index_page { From d405606c3bc499d215ff0bcb6a56080ff4bf07d4 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Mon, 14 Jan 2019 00:46:11 +0100 Subject: [PATCH 3/3] End fixing search index minification --- Cargo.lock | 6 +- src/librustdoc/Cargo.toml | 2 +- src/librustdoc/html/render.rs | 114 +++++++++++++++++++++++------ src/librustdoc/html/static/main.js | 4 +- 4 files changed, 98 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 92186d1700e53..d4593c1ad234b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1332,7 +1332,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "minifier" -version = "0.0.21" +version = "0.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "macro-utils 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2692,7 +2692,7 @@ dependencies = [ name = "rustdoc" version = "0.0.0" dependencies = [ - "minifier 0.0.21 (registry+https://github.com/rust-lang/crates.io-index)", + "minifier 0.0.26 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "pulldown-cmark 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "tempfile 3.0.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3510,7 +3510,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum memchr 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0a3eb002f0535929f1199681417029ebea04aadc0c7a4224b46be99c7f5d6a16" "checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff" "checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3" -"checksum minifier 0.0.21 (registry+https://github.com/rust-lang/crates.io-index)" = "9707d0ff1b828cba09c4bb27d5c3dceb6f49bd46f700042343bea350a131bf4f" +"checksum minifier 0.0.26 (registry+https://github.com/rust-lang/crates.io-index)" = "f299df45afd73332044ea9f717c816a84fc90c8b631409abf339ba93642a7985" "checksum miniz-sys 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "0300eafb20369952951699b68243ab4334f4b10a88f411c221d444b36c40e649" "checksum miniz_oxide 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5ad30a47319c16cde58d0314f5d98202a80c9083b5f61178457403dfb14e509c" "checksum miniz_oxide_c_api 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "28edaef377517fd9fe3e085c37d892ce7acd1fbeab9239c5a36eec352d8a8b7e" diff --git a/src/librustdoc/Cargo.toml b/src/librustdoc/Cargo.toml index bc6447d74f1b2..4b421881db48d 100644 --- a/src/librustdoc/Cargo.toml +++ b/src/librustdoc/Cargo.toml @@ -9,6 +9,6 @@ path = "lib.rs" [dependencies] pulldown-cmark = { version = "0.1.2", default-features = false } -minifier = "0.0.21" +minifier = "0.0.26" tempfile = "3" parking_lot = "0.6.4" diff --git a/src/librustdoc/html/render.rs b/src/librustdoc/html/render.rs index 26512d93655f4..3a64c56fc8eeb 100644 --- a/src/librustdoc/html/render.rs +++ b/src/librustdoc/html/render.rs @@ -741,8 +741,8 @@ fn build_index(krate: &clean::Crate, cache: &mut Cache) -> String { let mut crate_data = BTreeMap::new(); crate_data.insert("doc".to_owned(), Json::String(crate_doc)); - crate_data.insert("items".to_owned(), Json::Array(crate_items)); - crate_data.insert("paths".to_owned(), Json::Array(crate_paths)); + crate_data.insert("i".to_owned(), Json::Array(crate_items)); + crate_data.insert("p".to_owned(), Json::Array(crate_paths)); // Collect the index into a string format!("searchIndex[{}] = {};", @@ -914,12 +914,44 @@ themePicker.onblur = handleThemeButtonsBlur; write(cx.dst.join("COPYRIGHT.txt"), static_files::COPYRIGHT)?; - fn collect(path: &Path, krate: &str, key: &str) -> io::Result<(Vec, Vec)> { + fn collect( + path: &Path, + krate: &str, + key: &str, + for_search_index: bool, + ) -> io::Result<(Vec, Vec, Vec)> { + use minifier::js; + let mut ret = Vec::new(); let mut krates = Vec::new(); + let mut variables = Vec::new(); + + let mut krate = krate.to_owned(); + if path.exists() { for line in BufReader::new(File::open(path)?).lines() { let line = line?; + if for_search_index && line.starts_with("var r_") { + variables.push(line.clone()); + // We need to check if the crate name has been put into a variable as well. + let tokens = js::simple_minify(&line).apply(js::clean_tokens); + let mut pos = 0; + while pos < tokens.len() { + if let Some((var_pos, Some(value_pos))) = + js::get_variable_name_and_value_positions(&tokens, pos) { + if let Some(s) = tokens.0[value_pos].get_string() { + if &s[1..s.len() - 1] == krate { + if let Some(var) = tokens[var_pos].get_other() { + krate = var.to_owned(); + break + } + } + } + } + pos += 1; + } + continue; + } if !line.starts_with(key) { continue; } @@ -933,7 +965,7 @@ themePicker.onblur = handleThemeButtonsBlur; .unwrap_or_else(|| String::new())); } } - Ok((ret, krates)) + Ok((ret, krates, variables)) } fn show_item(item: &IndexItem, krate: &str) -> String { @@ -948,7 +980,7 @@ themePicker.onblur = handleThemeButtonsBlur; let dst = cx.dst.join("aliases.js"); { - let (mut all_aliases, _) = try_err!(collect(&dst, &krate.name, "ALIASES"), &dst); + let (mut all_aliases, _, _) = try_err!(collect(&dst, &krate.name, "ALIASES", false), &dst); let mut w = try_err!(File::create(&dst), &dst); let mut output = String::with_capacity(100); for (alias, items) in &cache.aliases { @@ -1035,7 +1067,9 @@ themePicker.onblur = handleThemeButtonsBlur; } let dst = cx.dst.join("source-files.js"); - let (mut all_sources, _krates) = try_err!(collect(&dst, &krate.name, "sourcesIndex"), &dst); + let (mut all_sources, _krates, _) = try_err!(collect(&dst, &krate.name, "sourcesIndex", + false), + &dst); all_sources.push(format!("sourcesIndex[\"{}\"] = {};", &krate.name, hierarchy.to_json_string())); @@ -1049,20 +1083,22 @@ themePicker.onblur = handleThemeButtonsBlur; // Update the search index let dst = cx.dst.join("search-index.js"); - let (mut all_indexes, mut krates) = try_err!(collect(&dst, &krate.name, "searchIndex"), &dst); + let (mut all_indexes, mut krates, variables) = try_err!(collect(&dst, + &krate.name, + "searchIndex", + true), &dst); all_indexes.push(search_index); // Sort the indexes by crate so the file will be generated identically even // with rustdoc running in parallel. all_indexes.sort(); let mut w = try_err!(File::create(&dst), &dst); - try_err!(writeln!(&mut w, "var N = null;var searchIndex = {{}};"), &dst); + try_err!(writeln!(&mut w, "var N=null,E=\"\",T=\"t\",U=\"u\",searchIndex={{}};"), &dst); try_err!(write_minify_replacer(&mut w, - &all_indexes.join("\n"), - options.enable_minification, - &[(minifier::js::Keyword::Null, "N")]), + &format!("{}\n{}", variables.join(""), all_indexes.join("\n")), + options.enable_minification), &dst); - try_err!(writeln!(&mut w, "initSearch(searchIndex);addSearchOptions(searchIndex);"), &dst); + try_err!(write!(&mut w, "initSearch(searchIndex);addSearchOptions(searchIndex);"), &dst); if options.enable_index_page { if let Some(index_page) = options.index_page.clone() { @@ -1161,8 +1197,9 @@ themePicker.onblur = handleThemeButtonsBlur; remote_item_type.css_class(), remote_path[remote_path.len() - 1])); - let (mut all_implementors, _) = try_err!(collect(&mydst, &krate.name, "implementors"), - &mydst); + let (mut all_implementors, _, _) = try_err!(collect(&mydst, &krate.name, "implementors", + false), + &mydst); all_implementors.push(implementors); // Sort the implementors by crate so the file will be generated // identically even with rustdoc running in parallel. @@ -1216,16 +1253,49 @@ fn write_minify(dst: PathBuf, contents: &str, enable_minification: bool) -> Resu } } -fn write_minify_replacer(dst: &mut W, - contents: &str, - enable_minification: bool, - keywords_to_replace: &[(minifier::js::Keyword, &str)]) - -> io::Result<()> { +fn write_minify_replacer( + dst: &mut W, + contents: &str, + enable_minification: bool, +) -> io::Result<()> { + use minifier::js::{Keyword, ReservedChar, Token}; + if enable_minification { writeln!(dst, "{}", - minifier::js::minify_and_replace_keywords(contents, keywords_to_replace) - .apply(minifier::js::clean_tokens) - .apply(minifier::js::aggregate_strings) + minifier::js::simple_minify(contents) + .apply(|f| { + // We keep backlines. + minifier::js::clean_tokens_except(f, |c| { + c.get_char() != Some(ReservedChar::Backline) + }) + }) + .apply(|f| { + minifier::js::replace_token_with(f, |t| { + match *t { + Token::Keyword(Keyword::Null) => Some(Token::Other("N")), + Token::String(s) => { + let s = &s[1..s.len() -1]; // The quotes are included + if s.is_empty() { + Some(Token::Other("E")) + } else if s == "t" { + Some(Token::Other("T")) + } else if s == "u" { + Some(Token::Other("U")) + } else { + None + } + } + _ => None, + } + }) + }) + .apply(|f| { + // We add a backline after the newly created variables. + minifier::js::aggregate_strings_with_separation( + f, + Token::Char(ReservedChar::Backline), + ) + }) .to_string()) } else { writeln!(dst, "{}", contents) diff --git a/src/librustdoc/html/static/main.js b/src/librustdoc/html/static/main.js index ceadbd5afac5f..82604cc7ad8bb 100644 --- a/src/librustdoc/html/static/main.js +++ b/src/librustdoc/html/static/main.js @@ -1555,10 +1555,10 @@ if (!DOMTokenList.prototype.remove) { // (String) description, // (Number | null) the parent path index to `paths`] // (Object | null) the type of the function (if any) - var items = rawSearchIndex[crate].items; + var items = rawSearchIndex[crate].i; // an array of [(Number) item type, // (String) name] - var paths = rawSearchIndex[crate].paths; + var paths = rawSearchIndex[crate].p; // convert `paths` into an object form var len = paths.length;