Skip to content

Commit

Permalink
Implement the local JS snippets RFC
Browse files Browse the repository at this point in the history
This commit is an implementation of [RFC 6] which enables crates to
inline local JS snippets into the final output artifact of
`wasm-bindgen`. This is accompanied with a few minor breaking changes
which are intended to be relatively minor in practice:

* The `module` attribute disallows paths starting with `./` and `../`.
  It requires paths starting with `/` to actually exist on the filesystem.
* The `--browser` flag no longer emits bundler-compatible code, but
  rather emits an ES module that can be natively loaded into a browser.

Otherwise be sure to check out [the RFC][RFC 6] for more details, and
otherwise this should implement at least the MVP version of the RFC!
Notably at this time JS snippets with `--nodejs` or `--no-modules` are
not supported and will unconditionally generate an error.

[RFC 6]: rustwasm/rfcs#6

Closes #1311
  • Loading branch information
alexcrichton committed Mar 5, 2019
1 parent f161717 commit b762948
Show file tree
Hide file tree
Showing 32 changed files with 984 additions and 379 deletions.
1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ members = [
"examples/webaudio",
"examples/webgl",
"examples/without-a-bundler",
"examples/without-a-bundler-no-modules",
"tests/no-std",
]
exclude = ['crates/typescript']
Expand Down
31 changes: 30 additions & 1 deletion crates/backend/src/ast.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ use proc_macro2::{Ident, Span};
use shared;
use syn;
use Diagnostic;
use std::hash::{Hash, Hasher};

/// An abstract syntax tree representing a rust program. Contains
/// extra information for joining up this rust code with javascript.
Expand All @@ -24,6 +25,8 @@ pub struct Program {
pub dictionaries: Vec<Dictionary>,
/// custom typescript sections to be included in the definition file
pub typescript_custom_sections: Vec<String>,
/// Inline JS snippets
pub inline_js: Vec<String>,
}

/// A rust to js interface. Allows interaction with rust objects/functions
Expand Down Expand Up @@ -66,11 +69,37 @@ pub enum MethodSelf {
#[cfg_attr(feature = "extra-traits", derive(Debug, PartialEq, Eq))]
#[derive(Clone)]
pub struct Import {
pub module: Option<String>,
pub module: ImportModule,
pub js_namespace: Option<Ident>,
pub kind: ImportKind,
}

#[cfg_attr(feature = "extra-traits", derive(Debug, PartialEq, Eq))]
#[derive(Clone)]
pub enum ImportModule {
None,
Named(String, Span),
Inline(usize, Span),
}

impl Hash for ImportModule {
fn hash<H: Hasher>(&self, h: &mut H) {
match self {
ImportModule::None => {
0u8.hash(h);
}
ImportModule::Named(name, _) => {
1u8.hash(h);
name.hash(h);
}
ImportModule::Inline(idx, _) => {
2u8.hash(h);
idx.hash(h);
}
}
}
}

#[cfg_attr(feature = "extra-traits", derive(Debug, PartialEq, Eq))]
#[derive(Clone)]
pub enum ImportKind {
Expand Down
27 changes: 24 additions & 3 deletions crates/backend/src/codegen.rs
Original file line number Diff line number Diff line change
Expand Up @@ -94,25 +94,46 @@ impl TryToTokens for ast::Program {
shared::SCHEMA_VERSION,
shared::version()
);
let encoded = encode::encode(self)?;
let mut bytes = Vec::new();
bytes.push((prefix_json.len() >> 0) as u8);
bytes.push((prefix_json.len() >> 8) as u8);
bytes.push((prefix_json.len() >> 16) as u8);
bytes.push((prefix_json.len() >> 24) as u8);
bytes.extend_from_slice(prefix_json.as_bytes());
bytes.extend_from_slice(&encode::encode(self)?);
bytes.extend_from_slice(&encoded.custom_section);

let generated_static_length = bytes.len();
let generated_static_value = syn::LitByteStr::new(&bytes, Span::call_site());

// We already consumed the contents of included files when generating
// the custom section, but we want to make sure that updates to the
// generated files will cause this macro to rerun incrementally. To do
// that we use `include_str!` to force rustc to think it has a
// dependency on these files. That way when the file changes Cargo will
// automatically rerun rustc which will rerun this macro. Other than
// this we don't actually need the results of the `include_str!`, so
// it's just shoved into an anonymous static.
let file_dependencies = encoded.included_files
.iter()
.map(|file| {
let file = file.to_str().unwrap();
quote! { include_str!(#file) }
});

(quote! {
#[allow(non_upper_case_globals)]
#[cfg(target_arch = "wasm32")]
#[link_section = "__wasm_bindgen_unstable"]
#[doc(hidden)]
#[allow(clippy::all)]
pub static #generated_static_name: [u8; #generated_static_length] =
*#generated_static_value;
pub static #generated_static_name: [u8; #generated_static_length] = {
#[doc(hidden)]
static _INCLUDED_FILES: &[&str] = &[#(#file_dependencies),*];

*#generated_static_value
};

})
.to_tokens(tokens);

Expand Down
107 changes: 98 additions & 9 deletions crates/backend/src/encode.rs
Original file line number Diff line number Diff line change
@@ -1,26 +1,50 @@
use std::cell::RefCell;
use std::collections::HashMap;

use proc_macro2::{Ident, Span};
use std::cell::RefCell;
use std::collections::{HashMap, HashSet};
use std::env;
use std::fs;
use std::path::PathBuf;
use util::ShortHash;

use ast;
use Diagnostic;

pub fn encode(program: &ast::Program) -> Result<Vec<u8>, Diagnostic> {
pub struct EncodeResult {
pub custom_section: Vec<u8>,
pub included_files: Vec<PathBuf>,
}

pub fn encode(program: &ast::Program) -> Result<EncodeResult, Diagnostic> {
let mut e = Encoder::new();
let i = Interner::new();
shared_program(program, &i)?.encode(&mut e);
Ok(e.finish())
let custom_section = e.finish();
let included_files = i.files.borrow().values().map(|p| &p.path).cloned().collect();
Ok(EncodeResult { custom_section, included_files })
}

struct Interner {
map: RefCell<HashMap<Ident, String>>,
strings: RefCell<HashSet<String>>,
files: RefCell<HashMap<String, LocalFile>>,
root: PathBuf,
crate_name: String,
}

struct LocalFile {
path: PathBuf,
definition: Span,
new_identifier: String,
}

impl Interner {
fn new() -> Interner {
Interner {
map: RefCell::new(HashMap::new()),
strings: RefCell::new(HashSet::new()),
files: RefCell::new(HashMap::new()),
root: env::current_dir().unwrap(),
crate_name: env::var("CARGO_PKG_NAME").unwrap(),
}
}

Expand All @@ -34,7 +58,45 @@ impl Interner {
}

fn intern_str(&self, s: &str) -> &str {
self.intern(&Ident::new(s, Span::call_site()))
let mut strings = self.strings.borrow_mut();
if let Some(s) = strings.get(s) {
return unsafe { &*(&**s as *const str) };
}
strings.insert(s.to_string());
drop(strings);
self.intern_str(s)
}

/// Given an import to a local module `id` this generates a unique module id
/// to assign to the contents of `id`.
///
/// Note that repeated invocations of this function will be memoized, so the
/// same `id` will always return the same resulting unique `id`.
fn resolve_import_module(&self, id: &str, span: Span) -> Result<&str, Diagnostic> {
let mut files = self.files.borrow_mut();
if let Some(file) = files.get(id) {
return Ok(self.intern_str(&file.new_identifier))
}
let path = if id.starts_with("/") {
self.root.join(&id[1..])
} else if id.starts_with("./") || id.starts_with("../") {
let msg = "relative module paths aren't supported yet";
return Err(Diagnostic::span_error(span, msg))
} else {
return Ok(self.intern_str(&id))
};

// Generate a unique ID which is somewhat readable as well, so mix in
// the crate name, hash to make it unique, and then the original path.
let new_identifier = format!("{}-{}{}", self.crate_name, ShortHash(0), id);
let file = LocalFile {
path,
definition: span,
new_identifier,
};
files.insert(id.to_string(), file);
drop(files);
self.resolve_import_module(id, span)
}
}

Expand Down Expand Up @@ -64,8 +126,29 @@ fn shared_program<'a>(
.iter()
.map(|x| -> &'a str { &x })
.collect(),
// version: shared::version(),
// schema_version: shared::SCHEMA_VERSION.to_string(),
local_modules: intern
.files
.borrow()
.values()
.map(|file| {
fs::read_to_string(&file.path)
.map(|s| {
LocalModule {
identifier: intern.intern_str(&file.new_identifier),
contents: intern.intern_str(&s),
}
})
.map_err(|e| {
let msg = format!("failed to read file `{}`: {}", file.path.display(), e);
Diagnostic::span_error(file.definition, msg)
})
})
.collect::<Result<Vec<_>, _>>()?,
inline_js: prog
.inline_js
.iter()
.map(|js| intern.intern_str(js))
.collect(),
})
}

Expand Down Expand Up @@ -111,7 +194,13 @@ fn shared_variant<'a>(v: &'a ast::Variant, intern: &'a Interner) -> EnumVariant<

fn shared_import<'a>(i: &'a ast::Import, intern: &'a Interner) -> Result<Import<'a>, Diagnostic> {
Ok(Import {
module: i.module.as_ref().map(|s| &**s),
module: match &i.module {
ast::ImportModule::Named(m, span) => {
ImportModule::Named(intern.resolve_import_module(m, *span)?)
}
ast::ImportModule::Inline(idx, _) => ImportModule::Inline(*idx as u32),
ast::ImportModule::None => ImportModule::None,
},
js_namespace: i.js_namespace.as_ref().map(|s| intern.intern(s)),
kind: shared_import_kind(&i.kind, intern)?,
})
Expand Down
2 changes: 1 addition & 1 deletion crates/backend/src/util.rs
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ pub fn ident_ty(ident: Ident) -> syn::Type {

pub fn wrap_import_function(function: ast::ImportFunction) -> ast::Import {
ast::Import {
module: None,
module: ast::ImportModule::None,
js_namespace: None,
kind: ast::ImportKind::Function(function),
}
Expand Down
Loading

0 comments on commit b762948

Please sign in to comment.