Skip to content

Commit

Permalink
End fixing search index minification
Browse files Browse the repository at this point in the history
  • Loading branch information
@GuillaumeGomez
GuillaumeGomez committed Jan 17, 2019
1 parent 2e8fd44 commit d405606
Show file tree
Hide file tree
Showing 4 changed files with 98 additions and 28 deletions.
6 changes: 3 additions & 3 deletions Cargo.lock
View file
Expand Up @@ -1332,7 +1332,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "minifier"
version = "0.0.21"
version = "0.0.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"macro-utils 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
Expand Down Expand Up @@ -2692,7 +2692,7 @@ dependencies = [
name = "rustdoc"
version = "0.0.0"
dependencies = [
"minifier 0.0.21 (registry+https://github.com/rust-lang/crates.io-index)",
"minifier 0.0.26 (registry+https://github.com/rust-lang/crates.io-index)",
"parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)",
"pulldown-cmark 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"tempfile 3.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
Expand Down Expand Up @@ -3510,7 +3510,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum memchr 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0a3eb002f0535929f1199681417029ebea04aadc0c7a4224b46be99c7f5d6a16"
"checksum memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff"
"checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3"
"checksum minifier 0.0.21 (registry+https://github.com/rust-lang/crates.io-index)" = "9707d0ff1b828cba09c4bb27d5c3dceb6f49bd46f700042343bea350a131bf4f"
"checksum minifier 0.0.26 (registry+https://github.com/rust-lang/crates.io-index)" = "f299df45afd73332044ea9f717c816a84fc90c8b631409abf339ba93642a7985"
"checksum miniz-sys 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "0300eafb20369952951699b68243ab4334f4b10a88f411c221d444b36c40e649"
"checksum miniz_oxide 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5ad30a47319c16cde58d0314f5d98202a80c9083b5f61178457403dfb14e509c"
"checksum miniz_oxide_c_api 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "28edaef377517fd9fe3e085c37d892ce7acd1fbeab9239c5a36eec352d8a8b7e"
Expand Down
2 changes: 1 addition & 1 deletion src/librustdoc/Cargo.toml
View file
Expand Up @@ -9,6 +9,6 @@ path = "lib.rs"

[dependencies]
pulldown-cmark = { version = "0.1.2", default-features = false }
minifier = "0.0.21"
minifier = "0.0.26"
tempfile = "3"
parking_lot = "0.6.4"
114 changes: 92 additions & 22 deletions src/librustdoc/html/render.rs
View file
Expand Up @@ -741,8 +741,8 @@ fn build_index(krate: &clean::Crate, cache: &mut Cache) -> String {

let mut crate_data = BTreeMap::new();
crate_data.insert("doc".to_owned(), Json::String(crate_doc));
crate_data.insert("items".to_owned(), Json::Array(crate_items));
crate_data.insert("paths".to_owned(), Json::Array(crate_paths));
crate_data.insert("i".to_owned(), Json::Array(crate_items));
crate_data.insert("p".to_owned(), Json::Array(crate_paths));

// Collect the index into a string
format!("searchIndex[{}] = {};",
Expand Down Expand Up @@ -914,12 +914,44 @@ themePicker.onblur = handleThemeButtonsBlur;
write(cx.dst.join("COPYRIGHT.txt"),
static_files::COPYRIGHT)?;

fn collect(path: &Path, krate: &str, key: &str) -> io::Result<(Vec<String>, Vec<String>)> {
fn collect(
path: &Path,
krate: &str,
key: &str,
for_search_index: bool,
) -> io::Result<(Vec<String>, Vec<String>, Vec<String>)> {
use minifier::js;

let mut ret = Vec::new();
let mut krates = Vec::new();
let mut variables = Vec::new();

let mut krate = krate.to_owned();

if path.exists() {
for line in BufReader::new(File::open(path)?).lines() {
let line = line?;
if for_search_index && line.starts_with("var r_") {
variables.push(line.clone());
// We need to check if the crate name has been put into a variable as well.
let tokens = js::simple_minify(&line).apply(js::clean_tokens);
let mut pos = 0;
while pos < tokens.len() {
if let Some((var_pos, Some(value_pos))) =
js::get_variable_name_and_value_positions(&tokens, pos) {
if let Some(s) = tokens.0[value_pos].get_string() {
if &s[1..s.len() - 1] == krate {
if let Some(var) = tokens[var_pos].get_other() {
krate = var.to_owned();
break
}
}
}
}
pos += 1;
}
continue;
}
if !line.starts_with(key) {
continue;
}
Expand All @@ -933,7 +965,7 @@ themePicker.onblur = handleThemeButtonsBlur;
.unwrap_or_else(|| String::new()));
}
}
Ok((ret, krates))
Ok((ret, krates, variables))
}

fn show_item(item: &IndexItem, krate: &str) -> String {
Expand All @@ -948,7 +980,7 @@ themePicker.onblur = handleThemeButtonsBlur;

let dst = cx.dst.join("aliases.js");
{
let (mut all_aliases, _) = try_err!(collect(&dst, &krate.name, "ALIASES"), &dst);
let (mut all_aliases, _, _) = try_err!(collect(&dst, &krate.name, "ALIASES", false), &dst);
let mut w = try_err!(File::create(&dst), &dst);
let mut output = String::with_capacity(100);
for (alias, items) in &cache.aliases {
Expand Down Expand Up @@ -1035,7 +1067,9 @@ themePicker.onblur = handleThemeButtonsBlur;
}

let dst = cx.dst.join("source-files.js");
let (mut all_sources, _krates) = try_err!(collect(&dst, &krate.name, "sourcesIndex"), &dst);
let (mut all_sources, _krates, _) = try_err!(collect(&dst, &krate.name, "sourcesIndex",
false),
&dst);
all_sources.push(format!("sourcesIndex[\"{}\"] = {};",
&krate.name,
hierarchy.to_json_string()));
Expand All @@ -1049,20 +1083,22 @@ themePicker.onblur = handleThemeButtonsBlur;

// Update the search index
let dst = cx.dst.join("search-index.js");
let (mut all_indexes, mut krates) = try_err!(collect(&dst, &krate.name, "searchIndex"), &dst);
let (mut all_indexes, mut krates, variables) = try_err!(collect(&dst,
&krate.name,
"searchIndex",
true), &dst);
all_indexes.push(search_index);

// Sort the indexes by crate so the file will be generated identically even
// with rustdoc running in parallel.
all_indexes.sort();
let mut w = try_err!(File::create(&dst), &dst);
try_err!(writeln!(&mut w, "var N = null;var searchIndex = {{}};"), &dst);
try_err!(writeln!(&mut w, "var N=null,E=\"\",T=\"t\",U=\"u\",searchIndex={{}};"), &dst);
try_err!(write_minify_replacer(&mut w,
&all_indexes.join("\n"),
options.enable_minification,
&[(minifier::js::Keyword::Null, "N")]),
&format!("{}\n{}", variables.join(""), all_indexes.join("\n")),
options.enable_minification),
&dst);
try_err!(writeln!(&mut w, "initSearch(searchIndex);addSearchOptions(searchIndex);"), &dst);
try_err!(write!(&mut w, "initSearch(searchIndex);addSearchOptions(searchIndex);"), &dst);

if options.enable_index_page {
if let Some(index_page) = options.index_page.clone() {
Expand Down Expand Up @@ -1161,8 +1197,9 @@ themePicker.onblur = handleThemeButtonsBlur;
remote_item_type.css_class(),
remote_path[remote_path.len() - 1]));

let (mut all_implementors, _) = try_err!(collect(&mydst, &krate.name, "implementors"),
&mydst);
let (mut all_implementors, _, _) = try_err!(collect(&mydst, &krate.name, "implementors",
false),
&mydst);
all_implementors.push(implementors);
// Sort the implementors by crate so the file will be generated
// identically even with rustdoc running in parallel.
Expand Down Expand Up @@ -1216,16 +1253,49 @@ fn write_minify(dst: PathBuf, contents: &str, enable_minification: bool) -> Resu
}
}

fn write_minify_replacer<W: Write>(dst: &mut W,
contents: &str,
enable_minification: bool,
keywords_to_replace: &[(minifier::js::Keyword, &str)])
-> io::Result<()> {
fn write_minify_replacer<W: Write>(
dst: &mut W,
contents: &str,
enable_minification: bool,
) -> io::Result<()> {
use minifier::js::{Keyword, ReservedChar, Token};

if enable_minification {
writeln!(dst, "{}",
minifier::js::minify_and_replace_keywords(contents, keywords_to_replace)
.apply(minifier::js::clean_tokens)
.apply(minifier::js::aggregate_strings)
minifier::js::simple_minify(contents)
.apply(|f| {
// We keep backlines.
minifier::js::clean_tokens_except(f, |c| {
c.get_char() != Some(ReservedChar::Backline)
})
})
.apply(|f| {
minifier::js::replace_token_with(f, |t| {
match *t {
Token::Keyword(Keyword::Null) => Some(Token::Other("N")),
Token::String(s) => {
let s = &s[1..s.len() -1]; // The quotes are included
if s.is_empty() {
Some(Token::Other("E"))
} else if s == "t" {
Some(Token::Other("T"))
} else if s == "u" {
Some(Token::Other("U"))
} else {
None
}
}
_ => None,
}
})
})
.apply(|f| {
// We add a backline after the newly created variables.
minifier::js::aggregate_strings_with_separation(
f,
Token::Char(ReservedChar::Backline),
)
})
.to_string())
} else {
writeln!(dst, "{}", contents)
Expand Down
4 changes: 2 additions & 2 deletions src/librustdoc/html/static/main.js
View file
Expand Up @@ -1555,10 +1555,10 @@ if (!DOMTokenList.prototype.remove) {
// (String) description,
// (Number | null) the parent path index to `paths`]
// (Object | null) the type of the function (if any)
var items = rawSearchIndex[crate].items;
var items = rawSearchIndex[crate].i;
// an array of [(Number) item type,
// (String) name]
var paths = rawSearchIndex[crate].paths;
var paths = rawSearchIndex[crate].p;

// convert `paths` into an object form
var len = paths.length;
Expand Down

0 comments on commit d405606

Please sign in to comment.