diff --git a/Cargo.lock b/Cargo.lock index 371e962bbe3fc..59d3054854668 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2735,9 +2735,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82c2c1fdcd807d1098552c5b9a36e425e42e9fbd7c6a37a8425f390f781f7fa7" +checksum = "38dd04e3c8279e75b31ef29dbdceebfe5ad89f4d0937213c53f7d49d01b3d5a7" dependencies = [ "itoa 1.0.2", "ryu", @@ -4432,6 +4432,7 @@ dependencies = [ "turbo-tasks", "turbo-tasks-build", "turbo-tasks-fs", + "turbopack-hash", "url", ] @@ -4464,10 +4465,10 @@ dependencies = [ "futures", "hyper", "hyper-tungstenite", - "json", "lazy_static", "mime_guess", "serde", + "serde_json", "tokio", "turbo-tasks", "turbo-tasks-build", @@ -4513,6 +4514,7 @@ dependencies = [ "turbo-tasks-memory", "turbo-tasks-testing", "turbopack-core", + "turbopack-hash", "url", ] @@ -4522,6 +4524,7 @@ version = "0.1.0" dependencies = [ "base16", "md4", + "twox-hash", ] [[package]] @@ -4541,6 +4544,17 @@ dependencies = [ "turbopack-hash", ] +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if 1.0.0", + "rand", + "static_assertions", +] + [[package]] name = "typed-arena" version = "2.0.1" diff --git a/crates/next-dev/src/turbo_tasks_viz.rs b/crates/next-dev/src/turbo_tasks_viz.rs index c093249353f77..f8fb8fe000e8a 100644 --- a/crates/next-dev/src/turbo_tasks_viz.rs +++ b/crates/next-dev/src/turbo_tasks_viz.rs @@ -3,8 +3,9 @@ use std::{str::FromStr, sync::Arc}; use anyhow::Result; use mime::Mime; use turbo_tasks::TurboTasks; -use turbo_tasks_fs::{File, FileContent, FileContentVc}; +use turbo_tasks_fs::{File, FileContent}; use turbo_tasks_memory::{stats::Stats, viz, MemoryBackend}; +use turbopack_core::version::VersionedContentVc; use turbopack_dev_server::source::{ContentSource, ContentSourceVc}; #[turbo_tasks::value(serialization = "none", eq = "manual", cell = "new", into = "new")] @@ -22,7 +23,7 @@ impl TurboTasksSourceVc { #[turbo_tasks::value_impl] impl ContentSource for TurboTasksSource { #[turbo_tasks::function] - fn get(&self, path: &str) -> Result { + fn get(&self, path: &str) -> Result { let tt = &self.turbo_tasks; if path == "graph" { let mut stats = Stats::new(); @@ -56,7 +57,7 @@ impl ContentSource for TurboTasksSource { } #[turbo_tasks::function] - fn get_by_id(&self, _id: &str) -> FileContentVc { + fn get_by_id(&self, _id: &str) -> VersionedContentVc { FileContent::NotFound.into() } } diff --git a/crates/turbo-tasks-fs/src/lib.rs b/crates/turbo-tasks-fs/src/lib.rs index f6407dafa4856..dbd66a7926935 100644 --- a/crates/turbo-tasks-fs/src/lib.rs +++ b/crates/turbo-tasks-fs/src/lib.rs @@ -886,8 +886,9 @@ impl From for fs::Permissions { } #[turbo_tasks::value(shared)] +#[derive(Clone)] pub enum FileContent { - Content(#[turbo_tasks(debug_ignore)] File), + Content(File), NotFound, } @@ -902,6 +903,7 @@ pub enum LinkContent { #[derive(Clone)] pub struct File { meta: FileMeta, + #[turbo_tasks(debug_ignore)] content: Vec, } diff --git a/crates/turbopack-core/Cargo.toml b/crates/turbopack-core/Cargo.toml index 9ad41e624ff74..11b0c716855ed 100644 --- a/crates/turbopack-core/Cargo.toml +++ b/crates/turbopack-core/Cargo.toml @@ -23,6 +23,7 @@ serde_regex = "1.1.0" tokio = "1.11.0" turbo-tasks = { path = "../turbo-tasks" } turbo-tasks-fs = { path = "../turbo-tasks-fs" } +turbopack-hash = { path = "../turbopack-hash" } url = "2.2.2" [build-dependencies] diff --git a/crates/turbopack-core/src/asset.rs b/crates/turbopack-core/src/asset.rs index 0cfd645245aa4..bf3594bb8e6cc 100644 --- a/crates/turbopack-core/src/asset.rs +++ b/crates/turbopack-core/src/asset.rs @@ -1,7 +1,10 @@ use anyhow::Result; use turbo_tasks_fs::{FileContentVc, FileSystemPathVc}; -use crate::reference::AssetReferencesVc; +use crate::{ + reference::AssetReferencesVc, + version::{VersionedContentVc, VersionedFileContentVc}, +}; /// A list of [Asset]s #[turbo_tasks::value(shared, transparent)] @@ -31,4 +34,9 @@ pub trait Asset { /// Other things (most likely [Asset]s) referenced from this [Asset]. fn references(&self) -> AssetReferencesVc; + + /// The content of the [Asset] alongside its version. + async fn versioned_content(&self) -> Result { + Ok(VersionedFileContentVc::new(self.content()).await?.into()) + } } diff --git a/crates/turbopack-core/src/chunk/mod.rs b/crates/turbopack-core/src/chunk/mod.rs index 9b57852f69852..3bf3ad22eaf83 100644 --- a/crates/turbopack-core/src/chunk/mod.rs +++ b/crates/turbopack-core/src/chunk/mod.rs @@ -18,6 +18,7 @@ use crate::{ /// A module id, which can be a number or string #[turbo_tasks::value(shared)] #[derive(Debug, Clone, Hash)] +#[serde(untagged)] pub enum ModuleId { Number(u32), String(String), diff --git a/crates/turbopack-core/src/issue/mod.rs b/crates/turbopack-core/src/issue/mod.rs index f4e48f2d7bb20..f7f6a42b3a928 100644 --- a/crates/turbopack-core/src/issue/mod.rs +++ b/crates/turbopack-core/src/issue/mod.rs @@ -319,8 +319,8 @@ impl IssueSourceVc { } Ok(Self::cell( if let FileLinesContent::Lines(lines) = &*asset.content().lines().await? { - let start = find_line_and_column(lines, start); - let end = find_line_and_column(lines, end); + let start = find_line_and_column(lines.as_ref(), start); + let end = find_line_and_column(lines.as_ref(), end); IssueSource { asset, start, end } } else { IssueSource { diff --git a/crates/turbopack-core/src/lib.rs b/crates/turbopack-core/src/lib.rs index 1c8144a3804b6..29bed958f3cef 100644 --- a/crates/turbopack-core/src/lib.rs +++ b/crates/turbopack-core/src/lib.rs @@ -11,6 +11,7 @@ pub mod resolve; pub mod source_asset; pub mod target; mod utils; +pub mod version; pub fn register() { turbo_tasks::register(); diff --git a/crates/turbopack-core/src/source_asset.rs b/crates/turbopack-core/src/source_asset.rs index be612458731b9..bfc108a94ce22 100644 --- a/crates/turbopack-core/src/source_asset.rs +++ b/crates/turbopack-core/src/source_asset.rs @@ -1,3 +1,4 @@ +use anyhow::Result; use turbo_tasks_fs::{FileContentVc, FileSystemPathVc}; use crate::{ diff --git a/crates/turbopack-core/src/version.rs b/crates/turbopack-core/src/version.rs new file mode 100644 index 0000000000000..7ec3961743a8e --- /dev/null +++ b/crates/turbopack-core/src/version.rs @@ -0,0 +1,151 @@ +use anyhow::{anyhow, Result}; +use serde::{Deserialize, Serialize}; +use turbo_tasks::{debug::ValueDebugFormat, primitives::StringVc, trace::TraceRawVcs}; +use turbo_tasks_fs::{FileContent, FileContentVc}; +use turbopack_hash::{encode_hex, hash_xxh3_hash64}; + +/// The content of an [Asset] alongside its version. +#[turbo_tasks::value_trait] +pub trait VersionedContent { + /// The content of the [Asset]. + fn content(&self) -> FileContentVc; + + /// Get a unique identifier of the version as a string. There is no way + /// to convert a version identifier back to the original `VersionedContent`, + /// so the original object needs to be stored somewhere. + fn version(&self) -> VersionVc; + + /// Describes how to update the content from an earlier version to the + /// latest available one. + async fn update(self_vc: VersionedContentVc, from: VersionVc) -> Result { + // By default, since we can't make any assumptions about the versioning + // scheme of the content, we ask for a full invalidation, except in the + // case where versions are the same. And we can't compare `VersionVc`s + // directly since `.cell_local()` breaks referential equality checks. + let from_id = from.id(); + let to = self_vc.version(); + let to_id = to.id(); + Ok(if *from_id.await? == *to_id.await? { + Update::None.into() + } else { + Update::Total(TotalUpdate { to }).into() + }) + } +} + +/// A versioned file content. +#[turbo_tasks::value] +pub struct VersionedFileContent { + // We can't store a `FileContentVc` directly because we don't want + // `VersionedFileContentVc` to invalidate when the content changes. + // Otherwise, reading `content` and `version` at two different instants in + // time might return inconsistent values. + file_content: FileContent, +} + +#[turbo_tasks::value_impl] +impl VersionedContent for VersionedFileContent { + #[turbo_tasks::function] + fn content(&self) -> FileContentVc { + FileContentVc::cell(self.file_content.clone()) + } + + #[turbo_tasks::function] + async fn version(&self) -> Result { + Ok(FileHashVersionVc::compute(&self.file_content).await?.into()) + } +} + +impl VersionedFileContentVc { + /// Creates a new [VersionedFileContentVc] from a [FileContentVc]. + pub async fn new(file_content: FileContentVc) -> Result { + let file_content = file_content.strongly_consistent().await?.clone(); + Ok(Self::cell(VersionedFileContent { file_content })) + } +} + +impl From for VersionedFileContentVc { + fn from(file_content: FileContent) -> Self { + VersionedFileContent { file_content }.cell() + } +} + +impl From for VersionedContentVc { + fn from(file_content: FileContent) -> Self { + VersionedFileContent { file_content }.cell().into() + } +} + +/// Describes the current version of an object, and how to update them from an +/// earlier version. +#[turbo_tasks::value_trait] +pub trait Version { + /// Get a unique identifier of the version as a string. There is no way + /// to convert an id back to its original `Version`, so the original object + /// needs to be stored somewhere. + fn id(&self) -> StringVc; +} + +/// Describes an update to a versioned object. +#[turbo_tasks::value(shared)] +#[derive(Debug)] +pub enum Update { + /// The asset can't be meaningfully updated while the app is running, so the + /// whole thing needs to be replaced. + Total(TotalUpdate), + + /// The asset can (potentially) be updated to a new version by applying a + /// specific set of instructions. + Partial(PartialUpdate), + + /// No update required. + None, +} + +/// A total update to a versioned object. +#[derive(PartialEq, Eq, Debug, Clone, TraceRawVcs, ValueDebugFormat, Serialize, Deserialize)] +pub struct TotalUpdate { + /// The version this update will bring the object to. + pub to: VersionVc, +} + +/// A partial update to a versioned object. +#[derive(PartialEq, Eq, Debug, Clone, TraceRawVcs, ValueDebugFormat, Serialize, Deserialize)] +pub struct PartialUpdate { + /// The version this update will bring the object to. + pub to: VersionVc, + /// The instructions to be passed to a remote system in order to update the + /// versioned object. + // TODO(alexkirsz) Should this be a serde_json::Value? + pub instruction: StringVc, +} + +/// [`Version`] implementation that hashes a file at a given path and returns +/// the hex encoded hash as a version identifier. +#[turbo_tasks::value] +#[derive(Clone)] +pub struct FileHashVersion { + hash: String, +} + +impl FileHashVersionVc { + /// Computes a new [`FileHashVersionVc`] from a path. + pub async fn compute(file_content: &FileContent) -> Result { + match file_content { + FileContent::Content(file) => { + let hash = hash_xxh3_hash64(file.content()); + let hex_hash = encode_hex(hash); + Ok(Self::cell(FileHashVersion { hash: hex_hash })) + } + FileContent::NotFound => Err(anyhow!("file not found")), + } + } +} + +#[turbo_tasks::value_impl] +impl Version for FileHashVersion { + #[turbo_tasks::function] + async fn id(&self) -> Result { + Ok(StringVc::cell(self.hash.clone())) + } +} diff --git a/crates/turbopack-dev-server/Cargo.toml b/crates/turbopack-dev-server/Cargo.toml index addda0f287c80..26ff2ac351a08 100644 --- a/crates/turbopack-dev-server/Cargo.toml +++ b/crates/turbopack-dev-server/Cargo.toml @@ -15,10 +15,10 @@ event-listener = "2.5.2" futures = "0.3.21" hyper = { version = "0.14", features = ["full"] } hyper-tungstenite = "0.8.1" -json = "0.12.4" lazy_static = "1.4.0" mime_guess = "2.0.4" serde = "1.0.136" +serde_json = "1.0.83" tokio = "1.11.0" turbo-tasks = { path = "../turbo-tasks" } turbo-tasks-fs = { path = "../turbo-tasks-fs" } diff --git a/crates/turbopack-dev-server/src/lib.rs b/crates/turbopack-dev-server/src/lib.rs index 2b23287127d30..ff848ae9ad7e7 100644 --- a/crates/turbopack-dev-server/src/lib.rs +++ b/crates/turbopack-dev-server/src/lib.rs @@ -3,40 +3,22 @@ pub mod fs; pub mod html; pub mod source; +pub mod update; -use std::{ - future::Future, - net::SocketAddr, - pin::Pin, - sync::{Arc, Mutex}, - time::Instant, -}; +use std::{future::Future, net::SocketAddr, pin::Pin, time::Instant}; use anyhow::{anyhow, Result}; -use event_listener::Event; -use futures::{ - stream::{unfold, FuturesUnordered, StreamExt}, - SinkExt, Stream, -}; use hyper::{ service::{make_service_fn, service_fn}, Body, Request, Response, Server, }; -use hyper_tungstenite::tungstenite::Message; use mime_guess::mime; -use tokio::select; -use turbo_tasks::{trace::TraceRawVcs, util::FormatDuration, RawVcReadResult, TransientValue}; +use turbo_tasks::{trace::TraceRawVcs, util::FormatDuration, TransientValue}; use turbo_tasks_fs::FileContent; use turbopack_cli_utils::issue::issue_to_styled_string; -use turbopack_core::{asset::AssetVc, issue::IssueVc}; +use turbopack_core::issue::IssueVc; -use self::source::ContentSourceVc; - -#[turbo_tasks::value(shared)] -enum FindAssetResult { - NotFound, - Found(AssetVc), -} +use self::{source::ContentSourceVc, update::UpdateServer}; #[turbo_tasks::value(cell = "new", serialization = "none", eq = "manual")] pub struct DevServer { @@ -56,51 +38,6 @@ impl DevServerVc { } } -struct State { - event: Event, - prev_value: Option>>, -} - -impl State { - fn set_value(&mut self, value: Option>) { - if let Some(ref prev_value) = self.prev_value { - match (prev_value, &value) { - (None, None) => return, - (Some(a), Some(b)) if **a == **b => return, - _ => {} - } - } - self.prev_value = Some(value); - self.event.notify(usize::MAX); - } - fn take(&mut self) -> Option>> { - self.prev_value.take() - } -} - -#[turbo_tasks::value_impl] -impl DevServerVc { - #[turbo_tasks::function] - async fn content_with_state( - self, - id: &str, - state: TransientValue>>, - ) -> Result<()> { - let content = self - .await? - .source - .get_by_id(id) - .strongly_consistent() - .await?; - { - let state = state.into_value(); - let mut state = state.lock().unwrap(); - state.set_value(Some(content)); - } - Ok(()) - } -} - impl DevServerVc { pub async fn listen(self) -> Result { let tt = turbo_tasks::turbo_tasks(); @@ -115,39 +52,8 @@ impl DevServerVc { let future = async move { if hyper_tungstenite::is_upgrade_request(&request) { let (response, websocket) = hyper_tungstenite::upgrade(request, None)?; - - tt.run_once_process(Box::pin(async move { - if let Err(err) = (async move { - let mut websocket = websocket.await?; - let mut change_stream_futures = FuturesUnordered::new(); - loop { - select! { - Some(message) = websocket.next() => { - if let Message::Text(msg) = message? { - let data = json::parse(&msg)?; - if let Some(id) = data.as_str() { - let stream = self.change_stream(id).skip(1); - change_stream_futures.push(stream.into_future()); - } - } - } - Some((_change, stream)) = change_stream_futures.next() => { - websocket.send(Message::text("refresh")).await?; - change_stream_futures.push(stream.into_future()); - } - else => break - } - } - - Ok::<(), anyhow::Error>(()) - }) - .await - { - println!("[WS]: error {}", err); - } - Ok::<(), anyhow::Error>(()) - })); - + let update_server = UpdateServer::new(websocket, source); + update_server.run(&*tt); return Ok(response); } let (tx, rx) = tokio::sync::oneshot::channel(); @@ -160,7 +66,7 @@ impl DevServerVc { } let file_content = source.get(&asset_path); if let FileContent::Content(content) = - &*file_content.strongly_consistent().await? + &*file_content.content().strongly_consistent().await? { let content_type = content.content_type().map_or_else( || { @@ -253,33 +159,6 @@ impl DevServerVc { Ok(()) })) } - - fn change_stream( - self, - id: &str, - ) -> Pin>> + Send + Sync>> { - let state = State { - event: Event::new(), - prev_value: None, - }; - let listener = state.event.listen(); - let state = Arc::new(Mutex::new(state)); - self.content_with_state(id, TransientValue::new(state.clone())); - Box::pin(unfold( - (state, listener), - |(state, mut listener)| async move { - loop { - listener.await; - let mut s = state.lock().unwrap(); - listener = s.event.listen(); - if let Some(value) = s.take() { - drop(s); - return Some((value, (state, listener))); - } - } - }, - )) - } } #[derive(TraceRawVcs)] diff --git a/crates/turbopack-dev-server/src/source/asset_graph.rs b/crates/turbopack-dev-server/src/source/asset_graph.rs index 008ccd530fbc4..1d55208a13b1f 100644 --- a/crates/turbopack-dev-server/src/source/asset_graph.rs +++ b/crates/turbopack-dev-server/src/source/asset_graph.rs @@ -5,8 +5,10 @@ use std::{ use anyhow::Result; use turbo_tasks::{get_invalidator, Invalidator, ValueToString}; -use turbo_tasks_fs::{FileContent, FileContentVc, FileSystemPathVc}; -use turbopack_core::{asset::AssetVc, reference::all_referenced_assets}; +use turbo_tasks_fs::{FileContent, FileSystemPathVc}; +use turbopack_core::{ + asset::AssetVc, reference::all_referenced_assets, version::VersionedContentVc, +}; use super::{ContentSource, ContentSourceVc}; @@ -92,7 +94,7 @@ impl AssetGraphContentSourceVc { #[turbo_tasks::value_impl] impl ContentSource for AssetGraphContentSource { #[turbo_tasks::function] - async fn get(self_vc: AssetGraphContentSourceVc, path: &str) -> Result { + async fn get(self_vc: AssetGraphContentSourceVc, path: &str) -> Result { let assets = self_vc.all_assets_map().strongly_consistent().await?; if let Some(asset) = assets.get(path) { { @@ -106,12 +108,13 @@ impl ContentSource for AssetGraphContentSource { } } } - return Ok(asset.content()); + return Ok(asset.versioned_content()); } Ok(FileContent::NotFound.into()) } + #[turbo_tasks::function] - async fn get_by_id(self_vc: AssetGraphContentSourceVc, id: &str) -> Result { + async fn get_by_id(self_vc: AssetGraphContentSourceVc, id: &str) -> Result { let root_path_str = self_vc.await?.root_path.to_string().await?; if id.starts_with(&*root_path_str) { let path = &id[root_path_str.len()..]; diff --git a/crates/turbopack-dev-server/src/source/combined.rs b/crates/turbopack-dev-server/src/source/combined.rs index 883d6d0b1ede4..cfe97d31a5ccf 100644 --- a/crates/turbopack-dev-server/src/source/combined.rs +++ b/crates/turbopack-dev-server/src/source/combined.rs @@ -1,6 +1,7 @@ use anyhow::Result; +use turbopack_core::version::VersionedContentVc; -use super::{ContentSource, ContentSourceVc, FileContent, FileContentVc}; +use super::{ContentSource, ContentSourceVc, FileContent}; #[turbo_tasks::value(shared)] pub struct CombinedContentSource { @@ -10,20 +11,20 @@ pub struct CombinedContentSource { #[turbo_tasks::value_impl] impl ContentSource for CombinedContentSource { #[turbo_tasks::function] - async fn get(&self, path: &str) -> Result { + async fn get(&self, path: &str) -> Result { for source in self.sources.iter() { let result = source.get(path); - if let FileContent::Content(_) = &*result.await? { + if let FileContent::Content(_) = &*result.content().await? { return Ok(result); } } Ok(FileContent::NotFound.into()) } #[turbo_tasks::function] - async fn get_by_id(&self, id: &str) -> Result { + async fn get_by_id(&self, id: &str) -> Result { for source in self.sources.iter() { let result = source.get_by_id(id); - if let FileContent::Content(_) = &*result.await? { + if let FileContent::Content(_) = &*result.content().await? { return Ok(result); } } diff --git a/crates/turbopack-dev-server/src/source/mod.rs b/crates/turbopack-dev-server/src/source/mod.rs index 8d30175611f2c..52815e78f8b9f 100644 --- a/crates/turbopack-dev-server/src/source/mod.rs +++ b/crates/turbopack-dev-server/src/source/mod.rs @@ -4,12 +4,13 @@ pub mod router; pub mod sub_path; use anyhow::Result; -use turbo_tasks_fs::{FileContent, FileContentVc}; +use turbo_tasks_fs::FileContent; +use turbopack_core::version::VersionedContentVc; #[turbo_tasks::value_trait] pub trait ContentSource { - fn get(&self, path: &str) -> FileContentVc; - fn get_by_id(&self, id: &str) -> FileContentVc; + fn get(&self, path: &str) -> VersionedContentVc; + fn get_by_id(&self, id: &str) -> VersionedContentVc; } #[turbo_tasks::value(shared)] @@ -18,11 +19,11 @@ pub struct NoContentSource; #[turbo_tasks::value_impl] impl ContentSource for NoContentSource { #[turbo_tasks::function] - fn get(&self, _path: &str) -> FileContentVc { + fn get(&self, _path: &str) -> VersionedContentVc { FileContent::NotFound.into() } #[turbo_tasks::function] - fn get_by_id(&self, _id: &str) -> FileContentVc { + fn get_by_id(&self, _id: &str) -> VersionedContentVc { FileContent::NotFound.into() } } diff --git a/crates/turbopack-dev-server/src/source/router.rs b/crates/turbopack-dev-server/src/source/router.rs index 4c4ad580f429a..dbaeb7648c64a 100644 --- a/crates/turbopack-dev-server/src/source/router.rs +++ b/crates/turbopack-dev-server/src/source/router.rs @@ -1,6 +1,7 @@ use anyhow::Result; +use turbopack_core::version::VersionedContentVc; -use super::{ContentSource, ContentSourceVc, FileContent, FileContentVc}; +use super::{ContentSource, ContentSourceVc, FileContent}; #[turbo_tasks::value(shared)] pub struct RouterContentSource { @@ -11,7 +12,7 @@ pub struct RouterContentSource { #[turbo_tasks::value_impl] impl ContentSource for RouterContentSource { #[turbo_tasks::function] - fn get(&self, path: &str) -> FileContentVc { + fn get(&self, path: &str) -> VersionedContentVc { for (route, source) in self.routes.iter() { if path.starts_with(route) { let path = &path[route.len()..]; @@ -21,10 +22,10 @@ impl ContentSource for RouterContentSource { self.fallback.get(path) } #[turbo_tasks::function] - async fn get_by_id(&self, id: &str) -> Result { + async fn get_by_id(&self, id: &str) -> Result { for (_, source) in self.routes.iter() { let result = source.get_by_id(id); - if let FileContent::Content(_) = &*result.await? { + if let FileContent::Content(_) = &*result.content().await? { return Ok(result); } } diff --git a/crates/turbopack-dev-server/src/source/sub_path.rs b/crates/turbopack-dev-server/src/source/sub_path.rs index de943390a3442..56da59c3fc324 100644 --- a/crates/turbopack-dev-server/src/source/sub_path.rs +++ b/crates/turbopack-dev-server/src/source/sub_path.rs @@ -1,6 +1,7 @@ use anyhow::Result; +use turbopack_core::version::VersionedContentVc; -use super::{ContentSource, ContentSourceVc, FileContentVc}; +use super::{ContentSource, ContentSourceVc}; #[turbo_tasks::value(shared)] pub struct SubPathContentSource { @@ -11,11 +12,11 @@ pub struct SubPathContentSource { #[turbo_tasks::value_impl] impl ContentSource for SubPathContentSource { #[turbo_tasks::function] - fn get(&self, path: &str) -> FileContentVc { + fn get(&self, path: &str) -> VersionedContentVc { self.source.get(&[&self.path, path].concat()) } #[turbo_tasks::function] - fn get_by_id(&self, id: &str) -> FileContentVc { + fn get_by_id(&self, id: &str) -> VersionedContentVc { self.source.get_by_id(id) } } diff --git a/crates/turbopack-dev-server/src/update/mod.rs b/crates/turbopack-dev-server/src/update/mod.rs new file mode 100644 index 0000000000000..f07e04a8f32de --- /dev/null +++ b/crates/turbopack-dev-server/src/update/mod.rs @@ -0,0 +1,4 @@ +pub mod server; +pub mod stream; + +pub(super) use server::UpdateServer; diff --git a/crates/turbopack-dev-server/src/update/server.rs b/crates/turbopack-dev-server/src/update/server.rs new file mode 100644 index 0000000000000..736e1da3be0cc --- /dev/null +++ b/crates/turbopack-dev-server/src/update/server.rs @@ -0,0 +1,141 @@ +use anyhow::Result; +use futures::{ + stream::{FuturesUnordered, StreamExt, StreamFuture}, + SinkExt, +}; +use hyper::upgrade::Upgraded; +use hyper_tungstenite::{tungstenite::Message, HyperWebsocket, WebSocketStream}; +use serde::{Deserialize, Serialize}; +use tokio::select; +use turbo_tasks::TurboTasksApi; +use turbopack_core::version::Update; + +use super::stream::UpdateStream; +use crate::source::ContentSourceVc; + +/// A server that listens for updates and sends them to connected clients. +pub(crate) struct UpdateServer { + ws: Option, + streams: FuturesUnordered>, + source: ContentSourceVc, +} + +impl UpdateServer { + /// Create a new update server with the given websocket and content source. + pub fn new(ws: HyperWebsocket, source: ContentSourceVc) -> Self { + Self { + ws: Some(ws), + streams: FuturesUnordered::new(), + source, + } + } + + /// Run the update server loop. + pub fn run(self, tt: &dyn TurboTasksApi) { + tt.run_once_process(Box::pin(async move { + if let Err(err) = self.run_internal().await { + println!("[UpdateServer]: error {}", err); + } + Ok(()) + })); + } + + async fn run_internal(mut self) -> Result<()> { + let mut client: UpdateClient = self.ws.take().unwrap().await?.into(); + + // TODO(alexkirsz) To avoid sending an empty update in the beginning, skip the + // first update. Note that the first update *may not* be empty, but since we + // don't support client HMR yet, this would result in a reload loop. + loop { + select! { + message = client.recv() => { + if let Some(message) = message? { + let content = self.source.get_by_id(&message.id); + let stream = UpdateStream::new(message.id, content).await?; + self.add_stream(stream); + } else { + break + } + } + Some((Some(update), stream)) = self.streams.next() => { + match &*update { + Update::Partial(partial) => { + let partial_instruction = partial.instruction.await?; + client.send_update(stream.id(), ClientUpdateInstructionType::Partial { + instruction: partial_instruction.as_ref(), + }).await?; + } + Update::Total(_total) => { + client.send_update(stream.id(), ClientUpdateInstructionType::Restart).await?; + } + Update::None => {} + } + self.add_stream(stream); + } + else => break + } + } + Ok(()) + } + + fn add_stream(&mut self, stream: UpdateStream) { + self.streams.push(stream.into_future()); + } +} + +struct UpdateClient { + ws: WebSocketStream, +} + +impl UpdateClient { + async fn recv(&mut self) -> Result> { + Ok(if let Some(msg) = self.ws.next().await { + if let Message::Text(msg) = msg? { + let msg = serde_json::from_str(&msg)?; + Some(msg) + } else { + None + } + } else { + None + }) + } + + async fn send_update( + &mut self, + id: &str, + type_: ClientUpdateInstructionType<'_>, + ) -> Result<()> { + let instruction = ClientUpdateInstruction { id, type_ }; + self.ws + .send(Message::text(serde_json::to_string(&instruction)?)) + .await?; + Ok(()) + } +} + +impl From> for UpdateClient { + fn from(ws: WebSocketStream) -> Self { + Self { ws } + } +} + +#[derive(Deserialize)] +#[serde(transparent)] +pub(super) struct ClientMessage { + pub(super) id: String, +} + +#[derive(Serialize)] +pub(super) struct ClientUpdateInstruction<'a> { + pub(super) id: &'a str, + #[serde(flatten, rename = "type")] + pub(super) type_: ClientUpdateInstructionType<'a>, +} + +#[derive(Serialize)] +#[serde(tag = "type", rename_all = "lowercase")] +pub(super) enum ClientUpdateInstructionType<'a> { + Restart, + Partial { instruction: &'a str }, +} diff --git a/crates/turbopack-dev-server/src/update/stream.rs b/crates/turbopack-dev-server/src/update/stream.rs new file mode 100644 index 0000000000000..d91d1253dbd48 --- /dev/null +++ b/crates/turbopack-dev-server/src/update/stream.rs @@ -0,0 +1,110 @@ +use std::{pin::Pin, sync::Mutex}; + +use anyhow::Result; +use futures::{stream::unfold, Stream}; +use tokio::sync::mpsc::Sender; +use turbo_tasks::{get_invalidator, Invalidator, RawVcReadResult, TransientValue}; +use turbopack_core::version::{PartialUpdate, TotalUpdate, Update, VersionVc, VersionedContentVc}; + +#[turbo_tasks::function] +async fn compute_update_stream( + from: VersionVc, + content: VersionedContentVc, + sender: TransientValue>>, +) -> Result<()> { + let update = content.update(from); + sender.send(update.await?).await?; + Ok(()) +} + +#[turbo_tasks::value(serialization = "none", eq = "manual", cell = "new")] +struct VersionState { + #[turbo_tasks(debug_ignore)] + inner: Mutex<(VersionVc, Option)>, +} + +#[turbo_tasks::value_impl] +impl VersionStateVc { + #[turbo_tasks::function] + async fn get(self) -> Result { + let this = self.await?; + let mut lock = this.inner.lock().unwrap(); + lock.1 = Some(get_invalidator()); + Ok(lock.0) + } +} + +impl VersionStateVc { + async fn new(inner: VersionVc) -> Result { + let inner = inner.cell_local().await?; + Ok(Self::cell(VersionState { + inner: Mutex::new((inner, None)), + })) + } + + async fn set(&self, new_inner: VersionVc) -> Result<()> { + let this = self.await?; + let new_inner = new_inner.cell_local().await?; + let mut lock = this.inner.lock().unwrap(); + if let (_, Some(invalidator)) = std::mem::replace(&mut *lock, (new_inner, None)) { + invalidator.invalidate(); + } + Ok(()) + } +} + +pub(super) struct UpdateStream { + id: String, + stream: Pin> + Send + Sync>>, +} + +impl UpdateStream { + pub async fn new(id: String, content: VersionedContentVc) -> Result { + let (sx, rx) = tokio::sync::mpsc::channel(32); + + let version_state = VersionStateVc::new(content.version()).await?; + + compute_update_stream(version_state.get(), content, TransientValue::new(sx)); + + Ok(UpdateStream { + id, + stream: Box::pin(unfold( + (rx, version_state), + |(mut rx, version_state)| async move { + loop { + let update = rx.recv().await.expect("failed to receive update"); + match &*update { + Update::Partial(PartialUpdate { to, .. }) + | Update::Total(TotalUpdate { to }) => { + version_state + .set(*to) + .await + .expect("failed to update version"); + return Some((update, (rx, version_state))); + } + // Do not propagate empty updates. + Update::None => { + continue; + } + } + } + }, + )), + }) + } + + pub fn id(&self) -> &str { + &self.id + } +} + +impl Stream for UpdateStream { + type Item = RawVcReadResult; + + fn poll_next( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + Pin::new(&mut self.get_mut().stream).poll_next(cx) + } +} diff --git a/crates/turbopack-ecmascript/Cargo.toml b/crates/turbopack-ecmascript/Cargo.toml index 96a908b3abb44..de790c5221858 100644 --- a/crates/turbopack-ecmascript/Cargo.toml +++ b/crates/turbopack-ecmascript/Cargo.toml @@ -33,6 +33,7 @@ tokio = "1.11.0" turbo-tasks = { path = "../turbo-tasks" } turbo-tasks-fs = { path = "../turbo-tasks-fs" } turbopack-core = { path = "../turbopack-core" } +turbopack-hash = { path = "../turbopack-hash" } url = "2.2.2" [dependencies.swc_common] diff --git a/crates/turbopack-ecmascript/src/chunk/mod.rs b/crates/turbopack-ecmascript/src/chunk/mod.rs index 0a7eb50e7f63e..cdc4bea44b82a 100644 --- a/crates/turbopack-ecmascript/src/chunk/mod.rs +++ b/crates/turbopack-ecmascript/src/chunk/mod.rs @@ -1,10 +1,18 @@ pub mod loader; -use std::fmt::Write; +use std::{ + collections::{HashMap, HashSet}, + fmt::Write, +}; -use anyhow::Result; +use anyhow::{anyhow, Result}; use serde::{Deserialize, Serialize}; -use turbo_tasks::{primitives::StringVc, trace::TraceRawVcs, ValueToString, ValueToStringVc}; +use turbo_tasks::{ + primitives::{StringVc, StringsVc}, + trace::TraceRawVcs, + util::try_join_all, + ValueToString, ValueToStringVc, +}; use turbo_tasks_fs::{File, FileContent, FileContentVc, FileSystemPathVc}; use turbopack_core::{ asset::{Asset, AssetVc}, @@ -14,12 +22,16 @@ use turbopack_core::{ FromChunkableAsset, ModuleId, ModuleIdVc, }, reference::{AssetReferenceVc, AssetReferencesVc}, + version::{ + PartialUpdate, Update, UpdateVc, Version, VersionVc, VersionedContent, VersionedContentVc, + }, }; +use turbopack_hash::{encode_hex, hash_xxh3_hash64, Xxh3Hash64Hasher}; use self::loader::ChunkGroupLoaderChunkItemVc; use crate::{ references::esm::EsmExportsVc, - utils::{stringify_module_id, stringify_number, stringify_str, FormatIter}, + utils::{stringify_module_id, stringify_str, FormatIter}, }; #[turbo_tasks::value] @@ -73,117 +85,80 @@ impl From> for EcmascriptChunkContentR } } } - #[turbo_tasks::function] async fn ecmascript_chunk_content( context: ChunkingContextVc, entry: AssetVc, ) -> Result { - let res = if let Some(res) = chunk_content::(context, entry).await? { - res - } else { - chunk_content_split::(context, entry).await? - }; - - Ok(EcmascriptChunkContentResultVc::cell(res.into())) -} - -#[turbo_tasks::value_impl] -impl Chunk for EcmascriptChunk {} - -#[turbo_tasks::value_impl] -impl ValueToString for EcmascriptChunk { - #[turbo_tasks::function] - async fn to_string(&self) -> Result { - Ok(StringVc::cell(format!( - "chunk {}", - self.entry.path().to_string().await? - ))) - } + Ok(EcmascriptChunkContentResultVc::cell( + if let Some(res) = chunk_content::(context, entry).await? { + res + } else { + chunk_content_split::(context, entry).await? + } + .into(), + )) } -#[turbo_tasks::function] -async fn module_factory(content: EcmascriptChunkItemContentVc) -> Result { - let content = content.await?; - let mut args = vec![ - "r: __turbopack_require__", - "i: __turbopack_import__", - "s: __turbopack_esm__", - "v: __turbopack_export_value__", - "c: __turbopack_cache__", - "l: __turbopack_load__", - "p: process", - ]; - if content.options.module { - args.push("m: module"); - } - if content.options.exports { - args.push("e: exports"); - } - Ok(StringVc::cell(format!( - "\n{}: (({{ {} }}) => (() => {{\n\n{}\n}})()),\n", - match &*content.id.await? { - ModuleId::Number(n) => stringify_number(*n), - ModuleId::String(s) => stringify_str(s), - }, - FormatIter(|| args.iter().copied().intersperse(", ")), - content.inner_code - ))) +#[turbo_tasks::value] +pub struct EcmascriptChunkContent { + module_factories: Vec<(ModuleId, String)>, + chunk_id: StringVc, + evaluate: Option, } -#[turbo_tasks::value_impl] -impl Asset for EcmascriptChunk { - #[turbo_tasks::function] - fn path(&self) -> FileSystemPathVc { - self.context.as_chunk_path(self.entry.path(), ".js") +impl EcmascriptChunkContent { + async fn new( + context: ChunkingContextVc, + entry: AssetVc, + chunk_id: StringVc, + evaluate: Option, + ) -> Result { + // TODO(alexkirsz) All of this should be done in a transition, otherwise we run + // the risks of values not being strongly consistent with each other. + let chunk_content = ecmascript_chunk_content(context, entry).await?; + let c_context = chunk_context(context); + let module_factories: Vec<_> = try_join_all(chunk_content.chunk_items.iter().map( + |chunk_item| async move { + let content = chunk_item.content(c_context, context); + let factory = module_factory(content); + let content_id = content.await?.id.await?; + Ok((content_id.clone(), factory.await?.clone())) as Result<_> + }, + )) + .await?; + Ok(EcmascriptChunkContent { + module_factories, + chunk_id, + evaluate, + }) } - #[turbo_tasks::function] - async fn content(self_vc: EcmascriptChunkVc) -> Result { - let this = self_vc.await?; - let content = ecmascript_chunk_content(this.context, this.entry); - let c_context = chunk_context(this.context); - let path = self_vc.path(); - let chunk_id = path.to_string(); - let contents = content - .await? - .chunk_items - .iter() - .map(|chunk_item| module_factory(chunk_item.content(c_context, this.context))) - .collect::>(); - let evaluate_chunks = if this.evaluate { - Some(ChunkGroupVc::from_chunk(self_vc.into()).chunks()) - } else { - None - }; + async fn file_content(&self) -> Result { let mut code = format!( "(self.TURBOPACK = self.TURBOPACK || []).push([{}, {{\n", - stringify_str(&chunk_id.await?) + stringify_str(&self.chunk_id.await?) ); - for module_factory in contents.iter() { - code += &*module_factory.await?; + for (id, factory) in &self.module_factories { + code = code + "\n" + &stringify_module_id(id) + ": " + factory + ","; } code += "\n}"; - if let Some(evaluate_chunks) = evaluate_chunks { - let evaluate_chunks = evaluate_chunks.await?; - let mut chunk_ids = Vec::new(); - for c in evaluate_chunks.iter() { - if let Some(ecma_chunk) = EcmascriptChunkVc::resolve_from(c).await? { - if ecma_chunk != self_vc { - chunk_ids.push(stringify_str(&*c.path().to_string().await?)); - } - } - } - - let condition = chunk_ids - .into_iter() - .map(|id| format!(" && chunks.has({})", id)) + if let Some(evaluate) = &self.evaluate { + let evaluate = evaluate.await?; + let condition = evaluate + .chunks_ids + .await? + .iter() + .map(|id| format!(" && chunks.has({})", stringify_str(id))) .collect::>() .join(""); - let module_id = c_context - .id(EcmascriptChunkPlaceableVc::cast_from(this.entry)) - .await?; - let entry_id = stringify_module_id(&module_id); + let entry_id = stringify_module_id(&evaluate.entry_module_id); + // Add a runnable to the chunk that requests the entry module to ensure it gets + // executed when the chunk is evaluated. + // The condition stops the entry module from being executed while chunks it + // depend on have not yet been registered. + // The runnable will run every time a new chunk is `.push`ed to TURBOPACK, until + // all dependent chunks have been evaluated. let _ = write!( code, ", ({{ chunks, getModule }}) => {{ @@ -193,7 +168,8 @@ impl Asset for EcmascriptChunk { ); } code += "]);\n"; - if this.evaluate { + if self.evaluate.is_some() { + // Add the turbopack runtime to the chunk. code += r#"(() => { if(Array.isArray(self.TURBOPACK)) { var array = self.TURBOPACK; @@ -312,7 +288,10 @@ impl Asset for EcmascriptChunk { socket.send(JSON.stringify(chunk)); } socket.onmessage = (event) => { - if(event.data === "refresh") location.reload(); + var data = JSON.parse(event.data); + if (data.type === "restart" || data.type === "partial") { + location.reload(); + } } } } @@ -321,6 +300,228 @@ impl Asset for EcmascriptChunk { Ok(FileContent::Content(File::from_source(code)).into()) } +} + +#[turbo_tasks::function] +async fn module_factory(content: EcmascriptChunkItemContentVc) -> Result { + let content = content.await?; + let mut args = vec![ + "r: __turbopack_require__", + "i: __turbopack_import__", + "s: __turbopack_esm__", + "v: __turbopack_export_value__", + "c: __turbopack_cache__", + "l: __turbopack_load__", + "p: process", + ]; + if content.options.module { + args.push("m: module"); + } + if content.options.exports { + args.push("e: exports"); + } + Ok(StringVc::cell(format!( + "(({{ {} }}) => (() => {{\n\n{}\n}})())", + FormatIter(|| args.iter().copied().intersperse(", ")), + content.inner_code + ))) +} + +#[derive(Serialize)] +struct EcmascriptChunkUpdate { + added: HashMap, + modified: HashMap, + deleted: HashSet, +} + +#[turbo_tasks::value_impl] +impl EcmascriptChunkContentVc { + #[turbo_tasks::function] + async fn version_original(self) -> Result { + let module_factories_hashes = self + .await? + .module_factories + .iter() + .map(|(id, factory)| (id.clone(), hash_xxh3_hash64(factory.as_bytes()))) + .collect(); + Ok(EcmascriptChunkVersion { + module_factories_hashes, + } + .cell()) + } +} + +#[turbo_tasks::value_impl] +impl VersionedContent for EcmascriptChunkContent { + #[turbo_tasks::function] + async fn content(&self) -> Result { + Ok(self.file_content().await?) + } + + #[turbo_tasks::function] + async fn version(self_vc: EcmascriptChunkContentVc) -> Result { + Ok(self_vc.version_original().into()) + } + + #[turbo_tasks::function] + async fn update( + self_vc: EcmascriptChunkContentVc, + from_version: VersionVc, + ) -> Result { + let from_version = EcmascriptChunkVersionVc::resolve_from(from_version) + .await? + .expect("version must be an `EcmascriptChunkVersionVc`"); + let to_version = self_vc.version_original(); + + let to = to_version.await?; + let from = from_version.await?; + let this = self_vc.await?; + + // TODO(alexkirsz) This should probably be stored as a HashMap already. + let module_factories: HashMap<_, _> = this.module_factories.iter().cloned().collect(); + let mut added = HashMap::new(); + let mut modified = HashMap::new(); + let mut deleted = HashSet::new(); + + let consistency_error = + || anyhow!("consistency error: missing module in `EcmascriptChunkContent`"); + + for (id, hash) in &to.module_factories_hashes { + if let Some(old_hash) = from.module_factories_hashes.get(id) { + if old_hash != hash { + modified.insert( + id.clone(), + module_factories + .get(id) + .ok_or_else(consistency_error)? + .clone(), + ); + } + } else { + added.insert( + id.clone(), + module_factories + .get(id) + .ok_or_else(consistency_error)? + .clone(), + ); + } + } + + for id in from.module_factories_hashes.keys() { + if !to.module_factories_hashes.contains_key(id) { + deleted.insert(id.clone()); + } + } + + let update = if added.is_empty() && modified.is_empty() && deleted.is_empty() { + Update::None + } else { + let chunk_update = EcmascriptChunkUpdate { + added, + modified, + deleted, + }; + + Update::Partial(PartialUpdate { + to: to_version.into(), + instruction: StringVc::cell(serde_json::to_string(&chunk_update)?), + }) + }; + + Ok(update.into()) + } +} + +#[turbo_tasks::value] +struct EcmascriptChunkVersion { + module_factories_hashes: HashMap, +} + +#[turbo_tasks::value_impl] +impl Version for EcmascriptChunkVersion { + #[turbo_tasks::function] + async fn id(&self) -> Result { + let sorted_hashes = { + let mut versions: Vec<_> = self.module_factories_hashes.values().copied().collect(); + versions.sort(); + versions + }; + let mut hasher = Xxh3Hash64Hasher::new(); + for hash in sorted_hashes { + hasher.write(&hash.to_le_bytes()); + } + let hash = hasher.finish(); + let hex_hash = encode_hex(hash); + Ok(StringVc::cell(hex_hash)) + } +} + +#[turbo_tasks::value] +struct EcmascriptChunkEvaluate { + chunks_ids: StringsVc, + entry_module_id: ModuleId, +} + +#[turbo_tasks::value_impl] +impl Chunk for EcmascriptChunk {} + +#[turbo_tasks::value_impl] +impl ValueToString for EcmascriptChunk { + #[turbo_tasks::function] + async fn to_string(&self) -> Result { + Ok(StringVc::cell(format!( + "chunk {}", + self.entry.path().to_string().await? + ))) + } +} + +impl EcmascriptChunkVc { + async fn chunk_content(self) -> Result { + let this = self.await?; + let evaluate = if this.evaluate { + let evaluate_chunks = ChunkGroupVc::from_chunk(self.into()).chunks().await?; + let mut chunks_ids = Vec::new(); + for c in evaluate_chunks.iter() { + if let Some(ecma_chunk) = EcmascriptChunkVc::resolve_from(c).await? { + if ecma_chunk != self { + chunks_ids.push(c.path().to_string().await?.clone()); + } + } + } + let c_context = chunk_context(this.context); + let entry_module_id = c_context + .id(EcmascriptChunkPlaceableVc::cast_from(this.entry)) + .await? + .clone(); + Some(EcmascriptChunkEvaluateVc::cell(EcmascriptChunkEvaluate { + chunks_ids: StringsVc::cell(chunks_ids), + entry_module_id, + })) + } else { + None + }; + let path = self.path(); + let chunk_id = path.to_string(); + let content = + EcmascriptChunkContent::new(this.context, this.entry, chunk_id, evaluate).await?; + Ok(content) + } +} + +#[turbo_tasks::value_impl] +impl Asset for EcmascriptChunk { + #[turbo_tasks::function] + fn path(&self) -> FileSystemPathVc { + self.context.as_chunk_path(self.entry.path(), ".js") + } + + #[turbo_tasks::function] + async fn content(self_vc: EcmascriptChunkVc) -> Result { + let content = self_vc.chunk_content().await?; + content.file_content().await + } #[turbo_tasks::function] async fn references(&self) -> Result { @@ -337,6 +538,12 @@ impl Asset for EcmascriptChunk { } Ok(AssetReferencesVc::cell(references)) } + + #[turbo_tasks::function] + async fn versioned_content(self_vc: EcmascriptChunkVc) -> Result { + let content = self_vc.chunk_content().await?; + Ok(content.cell().into()) + } } #[turbo_tasks::value] diff --git a/crates/turbopack-ecmascript/src/lib.rs b/crates/turbopack-ecmascript/src/lib.rs index 9185186fc45ab..3942bba153c39 100644 --- a/crates/turbopack-ecmascript/src/lib.rs +++ b/crates/turbopack-ecmascript/src/lib.rs @@ -278,7 +278,10 @@ impl EcmascriptChunkItem for ModuleChunkItem { .into()) } else { Ok(EcmascriptChunkItemContent { - inner_code: format!("/* unparsable {} */", self.module.path().to_string().await?), + inner_code: format!( + "/* unparseable {} */", + self.module.path().to_string().await? + ), id: chunk_context.id(EcmascriptChunkPlaceableVc::cast_from(self.module)), options: EcmascriptChunkItemOptions { ..Default::default() diff --git a/crates/turbopack-hash/Cargo.toml b/crates/turbopack-hash/Cargo.toml index 2f10cc19321d0..386ced93d7318 100644 --- a/crates/turbopack-hash/Cargo.toml +++ b/crates/turbopack-hash/Cargo.toml @@ -12,3 +12,4 @@ bench = false [dependencies] base16 = "0.2.1" md4 = "0.10.1" +twox-hash = "1.6.3" diff --git a/crates/turbopack-hash/src/hex.rs b/crates/turbopack-hash/src/hex.rs new file mode 100644 index 0000000000000..9fd35fa975f22 --- /dev/null +++ b/crates/turbopack-hash/src/hex.rs @@ -0,0 +1,4 @@ +/// Encodes a 64-bit unsigned integer into a hex string. +pub fn encode_hex(n: u64) -> String { + format!("{:01$x}", n, std::mem::size_of::() * 2) +} diff --git a/crates/turbopack-hash/src/lib.rs b/crates/turbopack-hash/src/lib.rs index 20b841791d01c..363065250a002 100644 --- a/crates/turbopack-hash/src/lib.rs +++ b/crates/turbopack-hash/src/lib.rs @@ -1,4 +1,11 @@ mod base16; +mod hex; mod md4; +mod xxh3_hash64; -pub use crate::{base16::encode_base16, md4::hash_md4}; +pub use crate::{ + base16::encode_base16, + hex::encode_hex, + md4::hash_md4, + xxh3_hash64::{hash_xxh3_hash64, Xxh3Hash64Hasher}, +}; diff --git a/crates/turbopack-hash/src/xxh3_hash64.rs b/crates/turbopack-hash/src/xxh3_hash64.rs new file mode 100644 index 0000000000000..0dd69173be49d --- /dev/null +++ b/crates/turbopack-hash/src/xxh3_hash64.rs @@ -0,0 +1,34 @@ +use std::hash::Hasher; + +use twox_hash::xxh3; + +/// Hash some content with the Xxh3Hash64 non-cryptographic hash function. +pub fn hash_xxh3_hash64(input: &[u8]) -> u64 { + xxh3::hash64(input) +} + +/// Xxh3Hash64 hasher. +pub struct Xxh3Hash64Hasher(xxh3::Hash64); + +impl Xxh3Hash64Hasher { + /// Create a new hasher. + pub fn new() -> Self { + Self(xxh3::Hash64::with_seed(0)) + } + + /// Add input bytes to the hash. + pub fn write(&mut self, input: &[u8]) { + self.0.write(input); + } + + /// Finish the hash computation and return the digest. + pub fn finish(&mut self) -> u64 { + self.0.finish() + } +} + +impl Default for Xxh3Hash64Hasher { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/turbopack/src/rebase/mod.rs b/crates/turbopack/src/rebase/mod.rs index 906187ef48300..6d6e9954f2e86 100644 --- a/crates/turbopack/src/rebase/mod.rs +++ b/crates/turbopack/src/rebase/mod.rs @@ -37,7 +37,7 @@ impl Asset for RebasedAsset { } #[turbo_tasks::function] - async fn content(&self) -> FileContentVc { + fn content(&self) -> FileContentVc { self.source.content() }