Skip to content

Commit

Permalink
Auto merge of #66828 - GuillaumeGomez:less-minification, r=kinnison
Browse files Browse the repository at this point in the history 
Less minification

The goal of this PR is to remove the minification process on the `search-index.js` file. It provides great result in term of space reduction but the computation time is far too long. I'll work on this issue and will put it back once it's fast enough.

cc @nox @lqd
r? @kinnison
  • Loading branch information
@bors
bors committed Dec 5, 2019
2 parents a0312c1 + 2d0f0ca commit 710a362
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 91 deletions.
103 changes: 13 additions & 90 deletions src/librustdoc/html/render.rs
View file
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -644,10 +644,9 @@ themePicker.onblur = handleThemeButtonsBlur;
themes.appendChild(but); themes.appendChild(but);
}});"#, }});"#,
as_json(&themes)); as_json(&themes));
write(cx.dst.join(&format!("theme{}.js", cx.shared.resource_suffix)), write_minify(&cx.shared.fs, cx.path("theme.js"),
theme_js.as_bytes() &theme_js,
)?; options.enable_minification)?;

write_minify(&cx.shared.fs, cx.path("main.js"), write_minify(&cx.shared.fs, cx.path("main.js"),
static_files::MAIN_JS, static_files::MAIN_JS,
options.enable_minification)?; options.enable_minification)?;
Expand Down Expand Up @@ -715,19 +714,13 @@ themePicker.onblur = handleThemeButtonsBlur;
path: &Path, path: &Path,
krate: &str, krate: &str,
key: &str, key: &str,
for_search_index: bool, ) -> io::Result<(Vec<String>, Vec<String>)> {
) -> io::Result<(Vec<String>, Vec<String>, Vec<String>)> {
let mut ret = Vec::new(); let mut ret = Vec::new();
let mut krates = Vec::new(); let mut krates = Vec::new();
let mut variables = Vec::new();


if path.exists() { if path.exists() {
for line in BufReader::new(File::open(path)?).lines() { for line in BufReader::new(File::open(path)?).lines() {
let line = line?; let line = line?;
if for_search_index && line.starts_with("var R") {
variables.push(line.clone());
continue;
}
if !line.starts_with(key) { if !line.starts_with(key) {
continue; continue;
} }
Expand All @@ -741,7 +734,7 @@ themePicker.onblur = handleThemeButtonsBlur;
.unwrap_or_else(|| String::new())); .unwrap_or_else(|| String::new()));
} }
} }
Ok((ret, krates, variables)) Ok((ret, krates))
} }


fn show_item(item: &IndexItem, krate: &str) -> String { fn show_item(item: &IndexItem, krate: &str) -> String {
Expand All @@ -756,7 +749,7 @@ themePicker.onblur = handleThemeButtonsBlur;


let dst = cx.dst.join(&format!("aliases{}.js", cx.shared.resource_suffix)); let dst = cx.dst.join(&format!("aliases{}.js", cx.shared.resource_suffix));
{ {
let (mut all_aliases, _, _) = try_err!(collect(&dst, &krate.name, "ALIASES", false), &dst); let (mut all_aliases, _) = try_err!(collect(&dst, &krate.name, "ALIASES"), &dst);
let mut output = String::with_capacity(100); let mut output = String::with_capacity(100);
for (alias, items) in &cx.cache.aliases { for (alias, items) in &cx.cache.aliases {
if items.is_empty() { if items.is_empty() {
Expand Down Expand Up @@ -853,9 +846,7 @@ themePicker.onblur = handleThemeButtonsBlur;
} }


let dst = cx.dst.join(&format!("source-files{}.js", cx.shared.resource_suffix)); let dst = cx.dst.join(&format!("source-files{}.js", cx.shared.resource_suffix));
let (mut all_sources, _krates, _) = try_err!(collect(&dst, &krate.name, "sourcesIndex", let (mut all_sources, _krates) = try_err!(collect(&dst, &krate.name, "sourcesIndex"), &dst);
false),
&dst);
all_sources.push(format!("sourcesIndex[\"{}\"] = {};", all_sources.push(format!("sourcesIndex[\"{}\"] = {};",
&krate.name, &krate.name,
hierarchy.to_json_string())); hierarchy.to_json_string()));
Expand All @@ -867,23 +858,18 @@ themePicker.onblur = handleThemeButtonsBlur;


// Update the search index // Update the search index
let dst = cx.dst.join(&format!("search-index{}.js", cx.shared.resource_suffix)); let dst = cx.dst.join(&format!("search-index{}.js", cx.shared.resource_suffix));
let (mut all_indexes, mut krates, variables) = try_err!(collect(&dst, let (mut all_indexes, mut krates) = try_err!(collect(&dst, &krate.name, "searchIndex"), &dst);
&krate.name,
"searchIndex",
true), &dst);
all_indexes.push(search_index); all_indexes.push(search_index);


// Sort the indexes by crate so the file will be generated identically even // Sort the indexes by crate so the file will be generated identically even
// with rustdoc running in parallel. // with rustdoc running in parallel.
all_indexes.sort(); all_indexes.sort();
{ {
let mut v = String::from("var N=null,E=\"\",T=\"t\",U=\"u\",searchIndex={};\n"); let mut v = String::from("var searchIndex={};\n");
v.push_str(&minify_replacer( v.push_str(&all_indexes.join("\n"));
&format!("{}\n{}", variables.join(""), all_indexes.join("\n")),
options.enable_minification));
// "addSearchOptions" has to be called first so the crate filtering can be set before the // "addSearchOptions" has to be called first so the crate filtering can be set before the
// search might start (if it's set into the URL for example). // search might start (if it's set into the URL for example).
v.push_str("addSearchOptions(searchIndex);initSearch(searchIndex);"); v.push_str("\naddSearchOptions(searchIndex);initSearch(searchIndex);");
cx.shared.fs.write(&dst, &v)?; cx.shared.fs.write(&dst, &v)?;
} }
if options.enable_index_page { if options.enable_index_page {
Expand Down Expand Up @@ -981,9 +967,8 @@ themePicker.onblur = handleThemeButtonsBlur;
remote_item_type, remote_item_type,
remote_path[remote_path.len() - 1])); remote_path[remote_path.len() - 1]));


let (mut all_implementors, _, _) = try_err!(collect(&mydst, &krate.name, "implementors", let (mut all_implementors, _) = try_err!(collect(&mydst, &krate.name, "implementors"),
false), &mydst);
&mydst);
all_implementors.push(implementors); all_implementors.push(implementors);
// Sort the implementors by crate so the file will be generated // Sort the implementors by crate so the file will be generated
// identically even with rustdoc running in parallel. // identically even with rustdoc running in parallel.
Expand Down Expand Up @@ -1020,68 +1005,6 @@ fn write_minify(fs:&DocFS, dst: PathBuf, contents: &str, enable_minification: bo
} }
} }


fn minify_replacer(
contents: &str,
enable_minification: bool,
) -> String {
use minifier::js::{simple_minify, Keyword, ReservedChar, Token, Tokens};

if enable_minification {
let tokens: Tokens<'_> = simple_minify(contents)
.into_iter()
.filter(|(f, next)| {
// We keep backlines.
minifier::js::clean_token_except(f, next, &|c: &Token<'_>| {
c.get_char() != Some(ReservedChar::Backline)
})
})
.map(|(f, _)| {
minifier::js::replace_token_with(f, &|t: &Token<'_>| {
match *t {
Token::Keyword(Keyword::Null) => Some(Token::Other("N")),
Token::String(s) => {
let s = &s[1..s.len() -1]; // The quotes are included
if s.is_empty() {
Some(Token::Other("E"))
} else if s == "t" {
Some(Token::Other("T"))
} else if s == "u" {
Some(Token::Other("U"))
} else {
None
}
}
_ => None,
}
})
})
.collect::<Vec<_>>()
.into();
let o = tokens.apply(|f| {
// We add a backline after the newly created variables.
minifier::js::aggregate_strings_into_array_with_separation_filter(
f,
"R",
Token::Char(ReservedChar::Backline),
// This closure prevents crates' names from being aggregated.
//
// The point here is to check if the string is preceded by '[' and
// "searchIndex". If so, it means this is a crate name and that it
// shouldn't be aggregated.
|tokens, pos| {
pos < 2 ||
!tokens[pos - 1].eq_char(ReservedChar::OpenBracket) ||
tokens[pos - 2].get_other() != Some("searchIndex")
}
)
})
.to_string();
format!("{}\n", o)
} else {
format!("{}\n", contents)
}
}

#[derive(Debug, Eq, PartialEq, Hash)] #[derive(Debug, Eq, PartialEq, Hash)]
struct ItemEntry { struct ItemEntry {
url: String, url: String,
Expand Down
2 changes: 1 addition & 1 deletion src/librustdoc/html/static/main.js
View file
Original file line number Original file line Diff line number Diff line change
Expand Up @@ -1221,7 +1221,7 @@ function getSearchElement() {
// then an exact path match // then an exact path match
path.indexOf(keys[i]) > -1 || path.indexOf(keys[i]) > -1 ||
// next if there is a parent, check for exact parent match // next if there is a parent, check for exact parent match
(parent !== undefined && (parent !== undefined && parent.name !== undefined &&
parent.name.toLowerCase().indexOf(keys[i]) > -1) || parent.name.toLowerCase().indexOf(keys[i]) > -1) ||
// lastly check to see if the name was a levenshtein match // lastly check to see if the name was a levenshtein match
levenshtein(name, keys[i]) <= MAX_LEV_DISTANCE)) { levenshtein(name, keys[i]) <= MAX_LEV_DISTANCE)) {
Expand Down

0 comments on commit 710a362

Please sign in to comment.