Introduce FileContent, Versioned, and refactor dev server updates (#209)

This prepares the way for HMR (#160) by letting us diff assets between
versions.

1. Add `Asset::versioned_content` which returns a `VersionedContentVc`.
2. `VersionedContent`s have a built-in versioning mechanism as they must implement `version() -> VersionVc`. `Version` is a trait, so `VersionVc` can contain a specific version implementation by asset type. This is particularly important because... 
3. A `VersionedContentVc` can be diffed with a `VersionVc` from the same underlying `VersionedContent` type with `content.update(from: version)`. This returns an `UpdateVc` which describes the steps necessary to update from one verson to the next. In the case of ES chunks, this will be a map of added, and modified module IDs, with their respective factories, and a set of deleted module IDs.
4. Implement diffing for ES chunks.

crates/next-dev/src/turbo_tasks_viz.rs

@@ -3,8 +3,9 @@ use std::{str::FromStr, sync::Arc};
 use anyhow::Result;
 use mime::Mime;
 use turbo_tasks::TurboTasks;
-use turbo_tasks_fs::{File, FileContent, FileContentVc};
+use turbo_tasks_fs::{File, FileContent};
 use turbo_tasks_memory::{stats::Stats, viz, MemoryBackend};
+use turbopack_core::version::VersionedContentVc;
 use turbopack_dev_server::source::{ContentSource, ContentSourceVc};
 
 #[turbo_tasks::value(serialization = "none", eq = "manual", cell = "new", into = "new")]
@@ -22,7 +23,7 @@ impl TurboTasksSourceVc {
 #[turbo_tasks::value_impl]
 impl ContentSource for TurboTasksSource {
     #[turbo_tasks::function]
-    fn get(&self, path: &str) -> Result<FileContentVc> {
+    fn get(&self, path: &str) -> Result<VersionedContentVc> {
         let tt = &self.turbo_tasks;
         if path == "graph" {
             let mut stats = Stats::new();
@@ -56,7 +57,7 @@ impl ContentSource for TurboTasksSource {
     }
 
     #[turbo_tasks::function]
-    fn get_by_id(&self, _id: &str) -> FileContentVc {
+    fn get_by_id(&self, _id: &str) -> VersionedContentVc {
         FileContent::NotFound.into()
     }
 }

crates/turbo-tasks-fs/src/lib.rs

@@ -886,8 +886,9 @@ impl From<Permissions> for fs::Permissions {
 }
 
 #[turbo_tasks::value(shared)]
+#[derive(Clone)]
 pub enum FileContent {
-    Content(#[turbo_tasks(debug_ignore)] File),
+    Content(File),
     NotFound,
 }
 
@@ -902,6 +903,7 @@ pub enum LinkContent {
 #[derive(Clone)]
 pub struct File {
     meta: FileMeta,
+    #[turbo_tasks(debug_ignore)]
     content: Vec<u8>,
 }
 

crates/turbopack-core/Cargo.toml

@@ -23,6 +23,7 @@ serde_regex = "1.1.0"
 tokio = "1.11.0"
 turbo-tasks = { path = "../turbo-tasks" }
 turbo-tasks-fs = { path = "../turbo-tasks-fs" }
+turbopack-hash = { path = "../turbopack-hash" }
 url = "2.2.2"
 
 [build-dependencies]

crates/turbopack-core/src/asset.rs

@@ -1,7 +1,10 @@
 use anyhow::Result;
 use turbo_tasks_fs::{FileContentVc, FileSystemPathVc};
 
-use crate::reference::AssetReferencesVc;
+use crate::{
+    reference::AssetReferencesVc,
+    version::{VersionedContentVc, VersionedFileContentVc},
+};
 
 /// A list of [Asset]s
 #[turbo_tasks::value(shared, transparent)]
@@ -31,4 +34,9 @@ pub trait Asset {
 
     /// Other things (most likely [Asset]s) referenced from this [Asset].
     fn references(&self) -> AssetReferencesVc;
+
+    /// The content of the [Asset] alongside its version.
+    async fn versioned_content(&self) -> Result<VersionedContentVc> {
+        Ok(VersionedFileContentVc::new(self.content()).await?.into())
+    }
 }

crates/turbopack-core/src/chunk/mod.rs

@@ -18,6 +18,7 @@ use crate::{
 /// A module id, which can be a number or string
 #[turbo_tasks::value(shared)]
 #[derive(Debug, Clone, Hash)]
+#[serde(untagged)]
 pub enum ModuleId {
     Number(u32),
     String(String),

crates/turbopack-core/src/issue/mod.rs

@@ -319,8 +319,8 @@ impl IssueSourceVc {
         }
         Ok(Self::cell(
             if let FileLinesContent::Lines(lines) = &*asset.content().lines().await? {
-                let start = find_line_and_column(lines, start);
-                let end = find_line_and_column(lines, end);
+                let start = find_line_and_column(lines.as_ref(), start);
+                let end = find_line_and_column(lines.as_ref(), end);
                 IssueSource { asset, start, end }
             } else {
                 IssueSource {

crates/turbopack-core/src/lib.rs

@@ -11,6 +11,7 @@ pub mod resolve;
 pub mod source_asset;
 pub mod target;
 mod utils;
+pub mod version;
 
 pub fn register() {
     turbo_tasks::register();

crates/turbopack-core/src/source_asset.rs

@@ -1,3 +1,4 @@
+use anyhow::Result;
 use turbo_tasks_fs::{FileContentVc, FileSystemPathVc};
 
 use crate::{

crates/turbopack-core/src/version.rs

@@ -0,0 +1,151 @@
+use anyhow::{anyhow, Result};
+use serde::{Deserialize, Serialize};
+use turbo_tasks::{debug::ValueDebugFormat, primitives::StringVc, trace::TraceRawVcs};
+use turbo_tasks_fs::{FileContent, FileContentVc};
+use turbopack_hash::{encode_hex, hash_xxh3_hash64};
+
+/// The content of an [Asset] alongside its version.
+#[turbo_tasks::value_trait]
+pub trait VersionedContent {
+    /// The content of the [Asset].
+    fn content(&self) -> FileContentVc;
+
+    /// Get a unique identifier of the version as a string. There is no way
+    /// to convert a version identifier back to the original `VersionedContent`,
+    /// so the original object needs to be stored somewhere.
+    fn version(&self) -> VersionVc;
+
+    /// Describes how to update the content from an earlier version to the
+    /// latest available one.
+    async fn update(self_vc: VersionedContentVc, from: VersionVc) -> Result<UpdateVc> {
+        // By default, since we can't make any assumptions about the versioning
+        // scheme of the content, we ask for a full invalidation, except in the
+        // case where versions are the same. And we can't compare `VersionVc`s
+        // directly since `.cell_local()` breaks referential equality checks.
+        let from_id = from.id();
+        let to = self_vc.version();
+        let to_id = to.id();
+        Ok(if *from_id.await? == *to_id.await? {
+            Update::None.into()
+        } else {
+            Update::Total(TotalUpdate { to }).into()
+        })
+    }
+}
+
+/// A versioned file content.
+#[turbo_tasks::value]
+pub struct VersionedFileContent {
+    // We can't store a `FileContentVc` directly because we don't want
+    // `VersionedFileContentVc` to invalidate when the content changes.
+    // Otherwise, reading `content` and `version` at two different instants in
+    // time might return inconsistent values.
+    file_content: FileContent,
+}
+
+#[turbo_tasks::value_impl]
+impl VersionedContent for VersionedFileContent {
+    #[turbo_tasks::function]
+    fn content(&self) -> FileContentVc {
+        FileContentVc::cell(self.file_content.clone())
+    }
+
+    #[turbo_tasks::function]
+    async fn version(&self) -> Result<VersionVc> {
+        Ok(FileHashVersionVc::compute(&self.file_content).await?.into())
+    }
+}
+
+impl VersionedFileContentVc {
+    /// Creates a new [VersionedFileContentVc] from a [FileContentVc].
+    pub async fn new(file_content: FileContentVc) -> Result<Self> {
+        let file_content = file_content.strongly_consistent().await?.clone();
+        Ok(Self::cell(VersionedFileContent { file_content }))
+    }
+}
+
+impl From<FileContent> for VersionedFileContentVc {
+    fn from(file_content: FileContent) -> Self {
+        VersionedFileContent { file_content }.cell()
+    }
+}
+
+impl From<FileContent> for VersionedContentVc {
+    fn from(file_content: FileContent) -> Self {
+        VersionedFileContent { file_content }.cell().into()
+    }
+}
+
+/// Describes the current version of an object, and how to update them from an
+/// earlier version.
+#[turbo_tasks::value_trait]
+pub trait Version {
+    /// Get a unique identifier of the version as a string. There is no way
+    /// to convert an id back to its original `Version`, so the original object
+    /// needs to be stored somewhere.
+    fn id(&self) -> StringVc;
+}
+
+/// Describes an update to a versioned object.
+#[turbo_tasks::value(shared)]
+#[derive(Debug)]
+pub enum Update {
+    /// The asset can't be meaningfully updated while the app is running, so the
+    /// whole thing needs to be replaced.
+    Total(TotalUpdate),
+
+    /// The asset can (potentially) be updated to a new version by applying a
+    /// specific set of instructions.
+    Partial(PartialUpdate),
+
+    /// No update required.
+    None,
+}
+
+/// A total update to a versioned object.
+#[derive(PartialEq, Eq, Debug, Clone, TraceRawVcs, ValueDebugFormat, Serialize, Deserialize)]
+pub struct TotalUpdate {
+    /// The version this update will bring the object to.
+    pub to: VersionVc,
+}
+
+/// A partial update to a versioned object.
+#[derive(PartialEq, Eq, Debug, Clone, TraceRawVcs, ValueDebugFormat, Serialize, Deserialize)]
+pub struct PartialUpdate {
+    /// The version this update will bring the object to.
+    pub to: VersionVc,
+    /// The instructions to be passed to a remote system in order to update the
+    /// versioned object.
+    // TODO(alexkirsz) Should this be a serde_json::Value?
+    pub instruction: StringVc,
+}
+
+/// [`Version`] implementation that hashes a file at a given path and returns
+/// the hex encoded hash as a version identifier.
+#[turbo_tasks::value]
+#[derive(Clone)]
+pub struct FileHashVersion {
+    hash: String,
+}
+
+impl FileHashVersionVc {
+    /// Computes a new [`FileHashVersionVc`] from a path.
+    pub async fn compute(file_content: &FileContent) -> Result<Self> {
+        match file_content {
+            FileContent::Content(file) => {
+                let hash = hash_xxh3_hash64(file.content());
+                let hex_hash = encode_hex(hash);
+                Ok(Self::cell(FileHashVersion { hash: hex_hash }))
+            }
+            FileContent::NotFound => Err(anyhow!("file not found")),
+        }
+    }
+}
+
+#[turbo_tasks::value_impl]
+impl Version for FileHashVersion {
+    #[turbo_tasks::function]
+    async fn id(&self) -> Result<StringVc> {
+        Ok(StringVc::cell(self.hash.clone()))
+    }
+}

crates/turbopack-dev-server/Cargo.toml

@@ -15,10 +15,10 @@ event-listener = "2.5.2"
 futures = "0.3.21"
 hyper = { version = "0.14", features = ["full"] }
 hyper-tungstenite = "0.8.1"
-json = "0.12.4"
 lazy_static = "1.4.0"
 mime_guess = "2.0.4"
 serde = "1.0.136"
+serde_json = "1.0.83"
 tokio = "1.11.0"
 turbo-tasks = { path = "../turbo-tasks" }
 turbo-tasks-fs = { path = "../turbo-tasks-fs" }