WIP: upload module-based scripts

src/upload/form/mod.rs

@@ -1,3 +1,4 @@
+mod modules_worker;
 mod plain_text;
 mod project_assets;
 mod service_worker;
@@ -9,19 +10,23 @@ use std::fs;
 use std::path::Path;
 use std::path::PathBuf;
 
+use ignore::WalkBuilder;
+
 use crate::settings::binding;
-use crate::settings::toml::{Target, TargetType};
+use crate::settings::toml::{ScriptFormat, Target, TargetType};
 use crate::sites::AssetManifest;
 use crate::wranglerjs;
 
 use plain_text::PlainText;
-use project_assets::ServiceWorkerAssets;
+use project_assets::{ModulesAssets, ServiceWorkerAssets};
 use text_blob::TextBlob;
 use wasm_module::WasmModule;
 
 // TODO: https://github.com/cloudflare/wrangler/issues/1083
 use super::{krate, Package};
 
+use self::project_assets::Module;
+
 pub fn build(
     target: &Target,
     asset_manifest: Option<AssetManifest>,
@@ -63,23 +68,73 @@ pub fn build(
 
             service_worker::build_form(&assets, session_config)
         }
-        TargetType::JavaScript => {
-            log::info!("JavaScript project detected. Publishing...");
-            let package_dir = target.package_dir()?;
-            let package = Package::new(&package_dir)?;
-
-            let script_path = package.main(&package_dir)?;
-
-            let assets = ServiceWorkerAssets::new(
-                script_path,
-                wasm_modules,
-                kv_namespaces.to_vec(),
-                text_blobs,
-                plain_texts,
-            )?;
-
-            service_worker::build_form(&assets, session_config)
-        }
+        TargetType::JavaScript => match &target.builder_config {
+            Some(config) => match &config.upload_format {
+                ScriptFormat::ServiceWorker => {
+                    log::info!("Plain JavaScript project detected. Publishing...");
+                    let package_dir = target.package_dir()?;
+                    let package = Package::new(&package_dir)?;
+                    let script_path = package.main(&package_dir)?;
+
+                    let assets = ServiceWorkerAssets::new(
+                        script_path,
+                        wasm_modules,
+                        kv_namespaces.to_vec(),
+                        text_blobs,
+                        plain_texts,
+                    )?;
+
+                    service_worker::build_form(&assets, session_config)
+                }
+                ScriptFormat::Modules => {
+                    let package_dir = target.package_dir()?;
+                    let package = Package::new(&package_dir)?;
+                    let main_module = package.module(&package_dir)?;
+                    let main_module_name = filename_from_path(&main_module)
+                        .ok_or_else(|| failure::err_msg("filename required for main module"))?;
+
+                    let modules_iter = WalkBuilder::new(config.build_dir.clone())
+                        .standard_filters(false)
+                        .build();
+
+                    let mut modules: Vec<Module> = vec![];
+
+                    for entry in modules_iter {
+                        let entry = entry?;
+                        let path = entry.path();
+                        if path.is_file() {
+                            modules.push(Module::new(path.to_owned())?);
+                        }
+                    }
+
+                    let assets = ModulesAssets::new(
+                        main_module_name,
+                        modules,
+                        kv_namespaces.to_vec(),
+                        plain_texts,
+                    )?;
+
+                    modules_worker::build_form(&assets, session_config)
+                }
+            },
+            None => {
+                log::info!("Plain JavaScript project detected. Publishing...");
+                let package_dir = target.package_dir()?;
+                let package = Package::new(&package_dir)?;
+
+                let script_path = package.main(&package_dir)?;
+
+                let assets = ServiceWorkerAssets::new(
+                    script_path,
+                    wasm_modules,
+                    kv_namespaces.to_vec(),
+                    text_blobs,
+                    plain_texts,
+                )?;
+
+                service_worker::build_form(&assets, session_config)
+            }
+        },
         TargetType::Webpack => {
             log::info!("webpack project detected. Publishing...");
             // TODO: https://github.com/cloudflare/wrangler/issues/850
@@ -121,10 +176,15 @@ fn get_asset_manifest_blob(asset_manifest: AssetManifest) -> Result<String, fail
     Ok(asset_manifest)
 }
 
-fn filename_from_path(path: &PathBuf) -> Option<String> {
+fn filestem_from_path(path: &PathBuf) -> Option<String> {
     path.file_stem()?.to_str().map(|s| s.to_string())
 }
 
+fn filename_from_path(path: &PathBuf) -> Option<String> {
+    path.file_name()
+        .map(|filename| filename.to_string_lossy().into_owned())
+}
+
 fn build_generated_dir() -> Result<(), failure::Error> {
     let dir = "./worker/generated";
     if !Path::new(dir).is_dir() {

src/upload/form/modules_worker.rs

@@ -0,0 +1,75 @@
+use std::fs::File;
+
+use reqwest::blocking::multipart::{Form, Part};
+use serde::Serialize;
+
+use crate::settings::binding::Binding;
+
+use super::ModulesAssets;
+
+#[derive(Serialize, Debug)]
+struct Metadata {
+    pub main_module: String,
+    pub bindings: Vec<Binding>,
+}
+
+pub fn build_form(
+    assets: &ModulesAssets,
+    session_config: Option<serde_json::Value>,
+) -> Result<Form, failure::Error> {
+    let mut form = Form::new();
+
+    // The preview service in particular streams the request form, and requires that the
+    // "metadata" part be set first, so this order is important.
+    form = add_metadata(form, assets)?;
+    form = add_files(form, assets)?;
+    if let Some(session_config) = session_config {
+        form = add_session_config(form, session_config)?
+    }
+
+    log::info!("building form");
+    log::info!("{:?}", &form);
+
+    Ok(form)
+}
+
+fn add_files(mut form: Form, assets: &ModulesAssets) -> Result<Form, failure::Error> {
+    for module in &assets.modules {
+        let file_name = module
+            .filename()
+            .ok_or_else(|| failure::err_msg("a filename is required for each module"))?;
+        let part = Part::reader(File::open(module.path.clone())?)
+            .mime_str(module.module_type.content_type())?
+            .file_name(file_name.clone());
+        form = form.part(file_name.clone(), part);
+    }
+    Ok(form)
+}
+
+fn add_metadata(mut form: Form, assets: &ModulesAssets) -> Result<Form, failure::Error> {
+    let metadata_json = serde_json::json!(&Metadata {
+        main_module: assets.main_module.clone(),
+        bindings: assets.bindings(),
+    });
+
+    let metadata = Part::text((metadata_json).to_string())
+        .file_name("metadata.json")
+        .mime_str("application/json")?;
+
+    form = form.part("metadata", metadata);
+
+    Ok(form)
+}
+
+fn add_session_config(
+    mut form: Form,
+    session_config: serde_json::Value,
+) -> Result<Form, failure::Error> {
+    let wrangler_session_config = Part::text((session_config).to_string())
+        .file_name("")
+        .mime_str("application/json")?;
+
+    form = form.part("wrangler-session-config", wrangler_session_config);
+
+    Ok(form)
+}

src/upload/form/project_assets.rs

@@ -3,10 +3,10 @@ use std::path::PathBuf;
 use failure::format_err;
 
 use super::binding::Binding;
-use super::filename_from_path;
 use super::plain_text::PlainText;
 use super::text_blob::TextBlob;
 use super::wasm_module::WasmModule;
+use super::{filename_from_path, filestem_from_path};
 
 use crate::settings::toml::KvNamespace;
 
@@ -28,7 +28,7 @@ impl ServiceWorkerAssets {
         text_blobs: Vec<TextBlob>,
         plain_texts: Vec<PlainText>,
     ) -> Result<Self, failure::Error> {
-        let script_name = filename_from_path(&script_path).ok_or_else(|| {
+        let script_name = filestem_from_path(&script_path).ok_or_else(|| {
             format_err!("filename should not be empty: {}", script_path.display())
         })?;
 
@@ -73,3 +73,94 @@ impl ServiceWorkerAssets {
         self.script_path.clone()
     }
 }
+
+pub struct Module {
+    pub path: PathBuf,
+    pub module_type: ModuleType,
+}
+
+impl Module {
+    pub fn new(path: PathBuf) -> Result<Module, failure::Error> {
+        let extension = path
+            .extension()
+            .ok_or(failure::err_msg(
+                "extension required to determine module type",
+            ))?
+            .to_string_lossy();
+
+        let module_type = match extension.as_ref() {
+            "mjs" => ModuleType::ES6,
+            "js" => ModuleType::CommonJS,
+            "wasm" => ModuleType::Wasm,
+            "txt" => ModuleType::Text,
+            _ => ModuleType::Data,
+        };
+
+        Ok(Module { path, module_type })
+    }
+
+    pub fn filename(&self) -> Option<String> {
+        filename_from_path(&self.path)
+    }
+}
+
+pub enum ModuleType {
+    ES6,
+    CommonJS,
+    Wasm,
+    Text,
+    Data,
+}
+
+impl ModuleType {
+    pub fn content_type(&self) -> &str {
+        match &self {
+            Self::ES6 => "application/javascript+module",
+            Self::CommonJS => "application/javascript",
+            Self::Wasm => "application/wasm",
+            Self::Text => "text/plain",
+            Self::Data => "application/octet-stream",
+        }
+    }
+}
+
+pub struct ModulesAssets {
+    pub main_module: String,
+    pub modules: Vec<Module>,
+    pub kv_namespaces: Vec<KvNamespace>,
+    pub plain_texts: Vec<PlainText>,
+}
+
+impl ModulesAssets {
+    pub fn new(
+        main_module: String,
+        modules: Vec<Module>,
+        kv_namespaces: Vec<KvNamespace>,
+        plain_texts: Vec<PlainText>,
+    ) -> Result<Self, failure::Error> {
+        Ok(Self {
+            main_module,
+            modules,
+            kv_namespaces,
+            plain_texts,
+        })
+    }
+
+    pub fn bindings(&self) -> Vec<Binding> {
+        let mut bindings = Vec::new();
+
+        // Bindings that refer to a `part` of the uploaded files
+        // in the service-worker format, are now modules.
+
+        for kv in &self.kv_namespaces {
+            let binding = kv.binding();
+            bindings.push(binding);
+        }
+        for plain_text in &self.plain_texts {
+            let binding = plain_text.binding();
+            bindings.push(binding);
+        }
+
+        bindings
+    }
+}

src/upload/form/text_blob.rs

@@ -1,6 +1,9 @@
 use super::binding::Binding;
 use serde::{Deserialize, Serialize};
 
+// Note: This is only used for service-worker scripts.
+// modules scripts use the universal Module class instead of this.
+
 #[derive(Debug, Deserialize, Serialize)]
 pub struct TextBlob {
     pub data: String,

src/upload/form/wasm_module.rs

@@ -3,7 +3,10 @@ use std::path::PathBuf;
 use failure::format_err;
 
 use super::binding::Binding;
-use super::filename_from_path;
+use super::filestem_from_path;
+
+// Note: This is only used for service-worker scripts.
+// modules scripts use the universal Module class instead of this.
 
 #[derive(Debug)]
 pub struct WasmModule {
@@ -14,7 +17,7 @@ pub struct WasmModule {
 
 impl WasmModule {
     pub fn new(path: PathBuf, binding: String) -> Result<Self, failure::Error> {
-        let filename = filename_from_path(&path)
+        let filename = filestem_from_path(&path)
             .ok_or_else(|| format_err!("filename should not be empty: {}", path.display()))?;
 
         Ok(Self {