feat: add redact to wasm

.changeset/grumpy-chicken-beam.md

@@ -0,0 +1,5 @@
+---
+"loro-crdt": patch
+---
+
+Feat: redact

.github/workflows/rust.yml

@@ -40,7 +40,7 @@ jobs:
         with:
           version: "0.2.100"
       - uses: Swatinem/rust-cache@v2
-      - name: Check
+      - name: Clippy Check
         run: cargo clippy --all-features -- -Dwarnings
       - name: Run rust tests
-        run: deno task test-all
+        run: GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} deno task test-all

crates/loro-ffi/src/doc.rs

@@ -345,10 +345,36 @@ impl LoroDoc {
         start_vv: &VersionVector,
         end_vv: &VersionVector,
     ) -> String {
-        let json = self
-            .doc
-            .export_json_updates_without_peer_compression(&start_vv.into(), &end_vv.into());
-        serde_json::to_string(&json).unwrap()
+        serde_json::to_string(
+            &self
+                .doc
+                .export_json_updates_without_peer_compression(&start_vv.into(), &end_vv.into()),
+        )
+        .unwrap()
+    }
+
+    /// Redacts sensitive content in JSON updates within the specified version range.
+    ///
+    /// This function allows you to share document history while removing potentially sensitive content.
+    /// It preserves the document structure and collaboration capabilities while replacing content with
+    /// placeholders according to these redaction rules:
+    ///
+    /// - Preserves delete and move operations
+    /// - Replaces text insertion content with the Unicode replacement character
+    /// - Substitutes list and map insert values with null
+    /// - Maintains structure of child containers
+    /// - Replaces text mark values with null
+    /// - Preserves map keys and text annotation keys
+    pub fn redact_json_updates(
+        &self,
+        json: &str,
+        version_range: VersionRange,
+    ) -> Result<String, LoroError> {
+        let mut schema: JsonSchema =
+            serde_json::from_str(json).map_err(|_e| LoroError::InvalidJsonSchema)?;
+        loro::json::redact(&mut schema, version_range)
+            .map_err(|e| LoroError::Unknown(e.to_string().into_boxed_str()))?;
+        Ok(serde_json::to_string(&schema).unwrap())
     }
 
     /// Export the readable [`Change`]s in the given [`IdSpan`]

crates/loro-internal/Cargo.toml

@@ -65,7 +65,7 @@ unexpected_cfgs = { level = "warn", check-cfg = ['cfg(loom)'] }
 version = "0.2.15"
 features = ["js"]
 
-[target.'cfg(all(target_arch = "wasm32", not(feature = "wasm")))'.dependencies]
+[target.'cfg(all(target_arch = "wasm32", not(features = "wasm")))'.dependencies]
 wasm-bindgen = "0.2.100"
 
 [dev-dependencies]

crates/loro-internal/VERSION

@@ -1 +1 @@
-1.5.0
+1.5.5

crates/loro-internal/src/container/richtext/tracker.rs

@@ -220,7 +220,6 @@ impl Tracker {
         self.applied_vv.extend_to_include_end_id(end_id);
     }
 
-    #[must_use]
     fn skip_applied(
         &mut self,
         op_id: ID,

crates/loro-internal/src/oplog/change_store.rs

@@ -1240,6 +1240,7 @@ impl ChangesBlock {
         self.estimated_size > MAX_BLOCK_SIZE
     }
 
+    #[allow(clippy::result_large_err)]
     fn push_change(
         self: &mut Arc<Self>,
         change: Change,

crates/loro-internal/src/state/counter_state.rs

@@ -38,7 +38,6 @@ impl ContainerState for CounterState {
         false
     }
 
-    #[must_use]
     fn apply_diff_and_convert(&mut self, diff: InternalDiff, _ctx: DiffApplyContext) -> Diff {
         if let InternalDiff::Counter(diff) = diff {
             self.value += diff;

crates/loro-wasm/package.json

@@ -36,12 +36,15 @@
   "author": "",
   "license": "MIT",
   "devDependencies": {
+    "@actions/core": "^1.11.1",
+    "@actions/github": "^6.0.1",
     "@rollup/plugin-alias": "^5.1.1",
     "@rollup/plugin-node-resolve": "^15.0.1",
     "@rollup/plugin-typescript": "^12.1.1",
     "@rollup/plugin-wasm": "^6.2.2",
     "@typescript-eslint/parser": "^6.2.0",
     "@vitest/ui": "^1.0.4",
+    "brotli-wasm": "^3.0.1",
     "esbuild": "^0.18.20",
     "eslint": "^8.46.0",
     "loro-crdt-alpha-4": "npm:loro-crdt@=1.0.0-alpha.4",

crates/loro-wasm/scripts/build.ts

@@ -1,5 +1,7 @@
 import * as path from "https://deno.land/std@0.105.0/path/mod.ts";
-import { gunzip, gzip } from "https://deno.land/x/compress@v0.4.5/mod.ts";
+import { gzip } from "https://deno.land/x/compress@v0.4.5/mod.ts";
+import brotliPromise from "npm:brotli-wasm";
+import { getOctokit } from "npm:@actions/github";
 const __dirname = path.dirname(path.fromFileUrl(import.meta.url));
 
 // deno run -A build.ts debug
@@ -16,7 +18,16 @@ const TARGETS = ["bundler", "nodejs", "web"];
 const startTime = performance.now();
 const LoroWasmDir = path.resolve(__dirname, "..");
 
-console.log(LoroWasmDir);
+// Check if running in CI
+const isCI = Deno.env.get("CI") === "true";
+const githubToken = Deno.env.get("GITHUB_TOKEN");
+const githubEventPath = Deno.env.get("GITHUB_EVENT_PATH");
+
+console.log({
+  isCI,
+  githubToken: !!githubToken,
+  githubEventPath: githubEventPath,
+});
 async function build() {
   await cargoBuild();
   const target = Deno.args[1];
@@ -58,15 +69,85 @@ async function build() {
   );
 
   if (profile === "release") {
-    const wasm = await Deno.readFile(path.resolve(LoroWasmDir, "bundler", "loro_wasm_bg.wasm"));
-    console.log("Wasm size: ", (wasm.length / 1024).toFixed(2), "KB");
+    const wasm = await Deno.readFile(
+      path.resolve(LoroWasmDir, "bundler", "loro_wasm_bg.wasm"),
+    );
+    const wasmSize = (wasm.length / 1024).toFixed(2);
+    console.log("Wasm size: ", wasmSize, "KB");
+
     const gzipped = await gzip(wasm);
-    console.log("Gzipped size: ", (gzipped.length / 1024).toFixed(2), "KB");
+    const gzipSize = (gzipped.length / 1024).toFixed(2);
+    console.log("Gzipped size: ", gzipSize, "KB");
+
+    // Use brotli-wasm for brotli compression
+    const brotli = await brotliPromise;
+    const brotliCompressed = brotli.compress(wasm);
+    const brotliSize = (brotliCompressed.length / 1024).toFixed(2);
+    console.log("Brotli size: ", brotliSize, "KB");
+
+    // Report sizes to PR if in CI
+    if (isCI && githubToken && githubEventPath) {
+      console.log("Creating comment for PR");
+      try {
+        // Parse GitHub event data
+        const event = JSON.parse(await Deno.readTextFile(githubEventPath));
+        console.log("event", event);
+        if (event.pull_request) {
+          const prNumber = event.pull_request.number;
+          const repo = event.repository.full_name;
+          const [owner, repoName] = repo.split("/");
+
+          const commentBody = `## WASM Size Report
+<!-- loro-wasm-size-report -->
+- Original size: ${wasmSize} KB
+- Gzipped size: ${gzipSize} KB
+- Brotli size: ${brotliSize} KB`;
+
+          // Initialize Octokit client
+          const octokit = getOctokit(githubToken);
+
+          // Find if we already have a comment with our marker
+          const { data: comments } = await octokit.rest.issues.listComments({
+            owner,
+            repo: repoName,
+            issue_number: prNumber,
+          });
+
+          const sizeReportMarker = "<!-- loro-wasm-size-report -->";
+          const existingComment = comments.find((comment) =>
+            comment.body?.includes(sizeReportMarker)
+          );
+
+          if (existingComment) {
+            // Update existing comment
+            await octokit.rest.issues.updateComment({
+              owner,
+              repo: repoName,
+              comment_id: existingComment.id,
+              body: commentBody,
+            });
+            console.log("Updated existing WASM size report comment");
+          } else {
+            // Create new comment
+            await octokit.rest.issues.createComment({
+              owner,
+              repo: repoName,
+              issue_number: prNumber,
+              body: commentBody,
+            });
+            console.log("Created new WASM size report comment");
+          }
+        }
+      } catch (error) {
+        console.error("Failed to report sizes to PR:", error);
+      }
+    }
   }
 }
 
 async function cargoBuild() {
-  const cmd = `cargo build --target wasm32-unknown-unknown --profile ${profile}`;
+  const cmd =
+    `cargo build --target wasm32-unknown-unknown --profile ${profile}`;
   console.log(cmd);
   const status = await Deno.run({
     cmd: cmd.split(" "),
@@ -94,7 +175,8 @@ async function buildTarget(target: string) {
   }
 
   // TODO: polyfill FinalizationRegistry
-  const cmd = `wasm-bindgen --weak-refs --target ${target} --out-dir ${target} ../../target/wasm32-unknown-unknown/${profileDir}/loro_wasm.wasm`;
+  const cmd =
+    `wasm-bindgen --weak-refs --target ${target} --out-dir ${target} ../../target/wasm32-unknown-unknown/${profileDir}/loro_wasm.wasm`;
   console.log(">", cmd);
   await Deno.run({ cmd: cmd.split(" "), cwd: LoroWasmDir }).status();
   console.log();

crates/loro-wasm/scripts/deno.json

@@ -0,0 +1,3 @@
+{
+    "nodeModules": "auto"
+}