Skip to content
Permalink
Browse files

WebIDL codegen: Replace cmake with a single Python script

When playing around with Cargo’s new timing visualization:
https://internals.rust-lang.org/t/exploring-crate-graph-build-times-with-cargo-build-ztimings/10975/21

… I was surprised to see the `script` crate’s build script take 76 seconds.
I did not expect WebIDL bindings generation to be *that* computationally
intensive.

It turns out almost all of this time is overhead. The build script uses CMake
to generate bindings for each WebIDL file in parallel, but that causes a lot
of work to be repeated 366 times:

* Starting up a Python VM
* Importing (parts of) the Python standard library
* Importing ~16k lines of our Python code
* Recompiling the latter to bytecode, since we used `python -B` to disable
  writing `.pyc` file
* Deserializing with `cPickle` and recreating in memory the results
  of parsing all WebIDL files

----

This commit remove the use of CMake and cPickle for the `script` crate.
Instead, all WebIDL bindings generation is done sequentially
in a single Python process. This takes 2 to 3 seconds.
  • Loading branch information...
SimonSapin committed Sep 27, 2019
1 parent 0495278 commit 5c60023cb8ad6f07927bd53f30c90873d07b300f

This file was deleted.

@@ -9,38 +9,30 @@ use std::fmt;
use std::fs::File;
use std::io::Write;
use std::path::PathBuf;
use std::str;
use std::process::Command;
use std::time::Instant;

fn main() {
let start = Instant::now();

let style_out_dir = PathBuf::from(env::var_os("DEP_SERVO_STYLE_CRATE_OUT_DIR").unwrap());
let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap());
let json = "css-properties.json";
std::fs::copy(style_out_dir.join(json), out_dir.join(json)).unwrap();
let target_dir = PathBuf::from(env::var_os("CARGO_TARGET_DIR").unwrap());

// This must use the Ninja generator -- it's the only one that
// parallelizes cmake's output properly. (Cmake generates
// separate makefiles, each of which try to build
// ParserResults.pkl, and then stomp on eachother.)
let mut build = cmake::Config::new(".");

let target = env::var("TARGET").unwrap();
if target.contains("windows-msvc") {
// We must use Ninja on Windows for this -- msbuild is painfully slow,
// and ninja is easier to install than make.
build.generator("Ninja");
let status = Command::new(find_python())
.arg("dom/bindings/codegen/run.py")
.arg(style_out_dir.join("css-properties.json"))
.arg(&out_dir)
.arg(target_dir.join("doc").join("servo"))
.status()
.unwrap();
if !status.success() {
std::process::exit(1)
}

build.build();

println!(
"Binding generation completed in {}s",
start.elapsed().as_secs()
);
println!("Binding generation completed in {:?}", start.elapsed());

let json = out_dir.join("build").join("InterfaceObjectMapData.json");
let json = out_dir.join("InterfaceObjectMapData.json");
let json: Value = serde_json::from_reader(File::open(&json).unwrap()).unwrap();
let mut map = phf_codegen::Map::new();
for (key, value) in json.as_object().unwrap() {
@@ -74,3 +66,27 @@ impl<'a> phf_shared::PhfHash for Bytes<'a> {
self.0.as_bytes().phf_hash(hasher)
}
}

fn find_python() -> String {
env::var("PYTHON").ok().unwrap_or_else(|| {
let candidates = if cfg!(windows) {
["python2.7.exe", "python27.exe", "python.exe"]
} else {
["python2.7", "python2", "python"]
};
for &name in &candidates {
if Command::new(name)
.arg("--version")
.output()
.ok()
.map_or(false, |out| out.status.success())
{
return name.to_owned();
}
}
panic!(
"Can't find python (tried {})! Try fixing PATH or setting the PYTHON env var",
candidates.join(", ")
)
})
}

This file was deleted.

@@ -54,32 +54,6 @@
"use", "virtual", "where", "while", "yield"}


def replaceFileIfChanged(filename, newContents):
"""
Read a copy of the old file, so that we don't touch it if it hasn't changed.
Returns True if the file was updated, false otherwise.
"""
# XXXjdm This doesn't play well with make right now.
# Force the file to always be updated, or else changing CodegenRust.py
# will cause many autogenerated bindings to be regenerated perpetually
# until the result is actually different.

# oldFileContents = ""
# try:
# with open(filename, 'rb') as oldFile:
# oldFileContents = ''.join(oldFile.readlines())
# except:
# pass

# if newContents == oldFileContents:
# return False

with open(filename, 'wb') as f:
f.write(newContents)

return True


def toStringBool(arg):
return str(not not arg).lower()

0 comments on commit 5c60023

Please sign in to comment.
You can’t perform that action at this time.