diff --git a/lerna.json b/lerna.json index d8bda8c8aadc..94685b1489c9 100644 --- a/lerna.json +++ b/lerna.json @@ -15,5 +15,5 @@ "registry": "https://registry.npmjs.org/" } }, - "version": "16.4.0-canary.13" + "version": "16.4.0-canary.14" } \ No newline at end of file diff --git a/packages/create-next-app/package.json b/packages/create-next-app/package.json index 0170e8b2698d..2a13d4c0ceea 100644 --- a/packages/create-next-app/package.json +++ b/packages/create-next-app/package.json @@ -1,6 +1,6 @@ { "name": "create-next-app", - "version": "16.4.0-canary.13", + "version": "16.4.0-canary.14", "keywords": [ "react", "next", diff --git a/packages/devlow-bench/package.json b/packages/devlow-bench/package.json index 01e7daf053a5..2238cfc83dad 100644 --- a/packages/devlow-bench/package.json +++ b/packages/devlow-bench/package.json @@ -1,7 +1,7 @@ { "name": "@vercel/devlow-bench", "private": true, - "version": "16.4.0-canary.13", + "version": "16.4.0-canary.14", "description": "Benchmarking tool for the developer workflow", "repository": { "type": "git", diff --git a/packages/eslint-config-next/package.json b/packages/eslint-config-next/package.json index 99167e1361af..0886e7c514eb 100644 --- a/packages/eslint-config-next/package.json +++ b/packages/eslint-config-next/package.json @@ -1,6 +1,6 @@ { "name": "eslint-config-next", - "version": "16.4.0-canary.13", + "version": "16.4.0-canary.14", "description": "ESLint configuration used by Next.js.", "license": "MIT", "repository": { @@ -12,7 +12,7 @@ "dist" ], "dependencies": { - "@next/eslint-plugin-next": "16.4.0-canary.13", + "@next/eslint-plugin-next": "16.4.0-canary.14", "eslint-import-resolver-node": "^0.3.6", "eslint-import-resolver-typescript": "^3.5.2", "eslint-plugin-import": "^2.32.0", diff --git a/packages/eslint-plugin-internal/package.json b/packages/eslint-plugin-internal/package.json index 5ea9085fc54d..2ce80eb07c6c 100644 --- a/packages/eslint-plugin-internal/package.json +++ b/packages/eslint-plugin-internal/package.json @@ -1,7 +1,7 @@ { "name": "@next/eslint-plugin-internal", "private": true, - "version": "16.4.0-canary.13", + "version": "16.4.0-canary.14", "description": "ESLint plugin for working on Next.js.", "exports": { ".": "./src/eslint-plugin-internal.js" diff --git a/packages/eslint-plugin-next/package.json b/packages/eslint-plugin-next/package.json index fcacc7ceb566..dddeea02e316 100644 --- a/packages/eslint-plugin-next/package.json +++ b/packages/eslint-plugin-next/package.json @@ -1,6 +1,6 @@ { "name": "@next/eslint-plugin-next", - "version": "16.4.0-canary.13", + "version": "16.4.0-canary.14", "description": "ESLint plugin for Next.js.", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/font/package.json b/packages/font/package.json index 4d70e2f680a9..54bffa791b5d 100644 --- a/packages/font/package.json +++ b/packages/font/package.json @@ -1,7 +1,7 @@ { "name": "@next/font", "private": true, - "version": "16.4.0-canary.13", + "version": "16.4.0-canary.14", "repository": { "url": "vercel/next.js", "directory": "packages/font" diff --git a/packages/next-bundle-analyzer/package.json b/packages/next-bundle-analyzer/package.json index bfcf5c0c89c0..efb00a82898f 100644 --- a/packages/next-bundle-analyzer/package.json +++ b/packages/next-bundle-analyzer/package.json @@ -1,6 +1,6 @@ { "name": "@next/bundle-analyzer", - "version": "16.4.0-canary.13", + "version": "16.4.0-canary.14", "main": "index.js", "types": "index.d.ts", "license": "MIT", diff --git a/packages/next-codemod/package.json b/packages/next-codemod/package.json index 2556233cb123..b48199582175 100644 --- a/packages/next-codemod/package.json +++ b/packages/next-codemod/package.json @@ -1,6 +1,6 @@ { "name": "@next/codemod", - "version": "16.4.0-canary.13", + "version": "16.4.0-canary.14", "license": "MIT", "repository": { "type": "git", diff --git a/packages/next-env/package.json b/packages/next-env/package.json index d865cc9213dd..0e8770141b0a 100644 --- a/packages/next-env/package.json +++ b/packages/next-env/package.json @@ -1,6 +1,6 @@ { "name": "@next/env", - "version": "16.4.0-canary.13", + "version": "16.4.0-canary.14", "keywords": [ "react", "next", diff --git a/packages/next-mdx/package.json b/packages/next-mdx/package.json index bfbdb6799944..b9b89a12817b 100644 --- a/packages/next-mdx/package.json +++ b/packages/next-mdx/package.json @@ -1,6 +1,6 @@ { "name": "@next/mdx", - "version": "16.4.0-canary.13", + "version": "16.4.0-canary.14", "main": "index.js", "license": "MIT", "repository": { diff --git a/packages/next-playwright/package.json b/packages/next-playwright/package.json index d0faf73cabf1..f13b70bf3a85 100644 --- a/packages/next-playwright/package.json +++ b/packages/next-playwright/package.json @@ -1,6 +1,6 @@ { "name": "@next/playwright", - "version": "16.4.0-canary.13", + "version": "16.4.0-canary.14", "repository": { "url": "vercel/next.js", "directory": "packages/next-playwright" diff --git a/packages/next-plugin-storybook/package.json b/packages/next-plugin-storybook/package.json index 0eb292f5d323..6a51b3cefe28 100644 --- a/packages/next-plugin-storybook/package.json +++ b/packages/next-plugin-storybook/package.json @@ -1,6 +1,6 @@ { "name": "@next/plugin-storybook", - "version": "16.4.0-canary.13", + "version": "16.4.0-canary.14", "repository": { "url": "vercel/next.js", "directory": "packages/next-plugin-storybook" diff --git a/packages/next-polyfill-module/package.json b/packages/next-polyfill-module/package.json index 076c6d07be59..7a1391f1d90f 100644 --- a/packages/next-polyfill-module/package.json +++ b/packages/next-polyfill-module/package.json @@ -1,6 +1,6 @@ { "name": "@next/polyfill-module", - "version": "16.4.0-canary.13", + "version": "16.4.0-canary.14", "description": "A standard library polyfill for ES Modules supporting browsers (Edge 16+, Firefox 60+, Chrome 61+, Safari 10.1+)", "main": "dist/polyfill-module.js", "license": "MIT", diff --git a/packages/next-polyfill-nomodule/package.json b/packages/next-polyfill-nomodule/package.json index 166ae464fa1f..1929aab70c72 100644 --- a/packages/next-polyfill-nomodule/package.json +++ b/packages/next-polyfill-nomodule/package.json @@ -1,6 +1,6 @@ { "name": "@next/polyfill-nomodule", - "version": "16.4.0-canary.13", + "version": "16.4.0-canary.14", "description": "A polyfill for non-dead, nomodule browsers.", "main": "dist/polyfill-nomodule.js", "license": "MIT", diff --git a/packages/next-routing/package.json b/packages/next-routing/package.json index 8c948c18dad2..121343cf7edc 100644 --- a/packages/next-routing/package.json +++ b/packages/next-routing/package.json @@ -1,6 +1,6 @@ { "name": "@next/routing", - "version": "16.4.0-canary.13", + "version": "16.4.0-canary.14", "keywords": [ "react", "next", diff --git a/packages/next-rspack/package.json b/packages/next-rspack/package.json index afd9ae7774c1..996606cdf6a0 100644 --- a/packages/next-rspack/package.json +++ b/packages/next-rspack/package.json @@ -1,6 +1,6 @@ { "name": "next-rspack", - "version": "16.4.0-canary.13", + "version": "16.4.0-canary.14", "repository": { "url": "vercel/next.js", "directory": "packages/next-rspack" diff --git a/packages/next-swc/package.json b/packages/next-swc/package.json index 9157d61808d4..0abbd2a3aa5f 100644 --- a/packages/next-swc/package.json +++ b/packages/next-swc/package.json @@ -1,6 +1,6 @@ { "name": "@next/swc", - "version": "16.4.0-canary.13", + "version": "16.4.0-canary.14", "private": true, "files": [ "native/" diff --git a/packages/next/package.json b/packages/next/package.json index 104bc1c724d1..575bf03d3bbb 100644 --- a/packages/next/package.json +++ b/packages/next/package.json @@ -1,6 +1,6 @@ { "name": "next", - "version": "16.4.0-canary.13", + "version": "16.4.0-canary.14", "description": "The React Framework", "main": "./dist/server/next.js", "license": "MIT", @@ -100,7 +100,7 @@ ] }, "dependencies": { - "@next/env": "16.4.0-canary.13", + "@next/env": "16.4.0-canary.14", "@swc/helpers": "0.5.23", "baseline-browser-mapping": "^2.9.19", "caniuse-lite": "^1.0.30001579", @@ -164,11 +164,11 @@ "@modelcontextprotocol/sdk": "1.18.1", "@mswjs/interceptors": "0.42.0", "@napi-rs/triples": "1.2.0", - "@next/font": "16.4.0-canary.13", - "@next/polyfill-module": "16.4.0-canary.13", - "@next/polyfill-nomodule": "16.4.0-canary.13", - "@next/react-refresh-utils": "16.4.0-canary.13", - "@next/swc": "16.4.0-canary.13", + "@next/font": "16.4.0-canary.14", + "@next/polyfill-module": "16.4.0-canary.14", + "@next/polyfill-nomodule": "16.4.0-canary.14", + "@next/react-refresh-utils": "16.4.0-canary.14", + "@next/swc": "16.4.0-canary.14", "@opentelemetry/api": "1.6.0", "@playwright/test": "1.61.0", "@rspack/core": "1.6.7", diff --git a/packages/next/src/server/use-cache/use-cache-wrapper.ts b/packages/next/src/server/use-cache/use-cache-wrapper.ts index ac70b2895356..0b329ce824ec 100644 --- a/packages/next/src/server/use-cache/use-cache-wrapper.ts +++ b/packages/next/src/server/use-cache/use-cache-wrapper.ts @@ -348,8 +348,6 @@ const crossRequestPendingCacheInvocations = new Map< Promise >() -const isEdgeRuntime = process.env.NEXT_RUNTIME === 'edge' - // The first argument at each call site is the full directive that produced // the invocation, e.g. "'use cache'" or "'use cache: remote'". const debug = process.env.NEXT_PRIVATE_DEBUG_CACHE @@ -3576,9 +3574,7 @@ export async function cache( // to be added to the consumer. Instead, we'll wait for any ClientReference to be emitted // which themselves will handle the preloading. moduleLoading: null, - moduleMap: isEdgeRuntime - ? clientReferenceManifest.edgeRscModuleMapping - : clientReferenceManifest.rscModuleMapping, + moduleMap: clientReferenceManifest.rscModuleMapping, serverModuleMap: getServerModuleMap(), } diff --git a/packages/react-refresh-utils/package.json b/packages/react-refresh-utils/package.json index e7f4df33494a..614f93001ff9 100644 --- a/packages/react-refresh-utils/package.json +++ b/packages/react-refresh-utils/package.json @@ -1,6 +1,6 @@ { "name": "@next/react-refresh-utils", - "version": "16.4.0-canary.13", + "version": "16.4.0-canary.14", "description": "An experimental package providing utilities for React Refresh.", "repository": { "url": "vercel/next.js", diff --git a/packages/third-parties/package.json b/packages/third-parties/package.json index 8f8a3893b435..18a291a61c9e 100644 --- a/packages/third-parties/package.json +++ b/packages/third-parties/package.json @@ -1,6 +1,6 @@ { "name": "@next/third-parties", - "version": "16.4.0-canary.13", + "version": "16.4.0-canary.14", "repository": { "url": "vercel/next.js", "directory": "packages/third-parties" @@ -26,7 +26,7 @@ "third-party-capital": "1.0.20" }, "devDependencies": { - "next": "16.4.0-canary.13", + "next": "16.4.0-canary.14", "outdent": "0.8.0", "prettier": "2.5.1", "typescript": "6.0.2" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b3672791afe3..ce8f9ba0ea4a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1027,7 +1027,7 @@ importers: packages/eslint-config-next: dependencies: '@next/eslint-plugin-next': - specifier: 16.4.0-canary.13 + specifier: 16.4.0-canary.14 version: link:../eslint-plugin-next eslint: specifier: '>=9.0.0' @@ -1110,7 +1110,7 @@ importers: packages/next: dependencies: '@next/env': - specifier: 16.4.0-canary.13 + specifier: 16.4.0-canary.14 version: link:../next-env '@swc/helpers': specifier: 0.5.23 @@ -1231,19 +1231,19 @@ importers: specifier: 1.2.0 version: 1.2.0 '@next/font': - specifier: 16.4.0-canary.13 + specifier: 16.4.0-canary.14 version: link:../font '@next/polyfill-module': - specifier: 16.4.0-canary.13 + specifier: 16.4.0-canary.14 version: link:../next-polyfill-module '@next/polyfill-nomodule': - specifier: 16.4.0-canary.13 + specifier: 16.4.0-canary.14 version: link:../next-polyfill-nomodule '@next/react-refresh-utils': - specifier: 16.4.0-canary.13 + specifier: 16.4.0-canary.14 version: link:../react-refresh-utils '@next/swc': - specifier: 16.4.0-canary.13 + specifier: 16.4.0-canary.14 version: link:../next-swc '@opentelemetry/api': specifier: 1.6.0 @@ -1986,7 +1986,7 @@ importers: version: 1.0.20 devDependencies: next: - specifier: 16.4.0-canary.13 + specifier: 16.4.0-canary.14 version: link:../next outdent: specifier: 0.8.0 diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/eviction.rs b/turbopack/crates/turbo-tasks-backend/src/backend/eviction.rs index db2a7fc89879..7007a8f38d02 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/eviction.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/eviction.rs @@ -129,8 +129,11 @@ impl EvictionControl { evict } - /// Call after completing an eviction cycle. Seeds the memory floor with the - /// post-eviction usage; later cycles lower it further as memory settles. + /// Call after completing an eviction cycle and after mimalloc is cleaned up + /// with [`TurboMalloc::collect`], since the freed memory is only reflected + /// in [`TurboMalloc::memory_usage`] once it has been. Seeds the memory floor + /// with the post-eviction usage; later cycles lower it further as memory + /// settles. pub(crate) fn record_eviction(&mut self) { self.memory_floor = Some(TurboMalloc::memory_usage()); } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index d31cdaf8ab58..f761b9354760 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -3044,8 +3044,6 @@ impl TurboTasksBackend { // memory so racing with execution is as likely to save time as // cost it. self.storage.evict_after_snapshot(background_span.id()); - // Sample the post-eviction floor as the new baseline. - eviction_control.record_eviction(); true } else { false @@ -3098,6 +3096,12 @@ impl TurboTasksBackend { { TurboMalloc::collect(true); } + + // Sample the new baseline after the collect above, which is what + // makes the evicted memory show up in `memory_usage`. + if ran_eviction { + eviction_control.record_eviction(); + } } } } diff --git a/turbopack/crates/turbo-tasks-malloc/src/counter.rs b/turbopack/crates/turbo-tasks-malloc/src/counter.rs index a61e55c27232..4a94073ae86a 100644 --- a/turbopack/crates/turbo-tasks-malloc/src/counter.rs +++ b/turbopack/crates/turbo-tasks-malloc/src/counter.rs @@ -1,30 +1,111 @@ -use std::{ - cell::UnsafeCell, - ptr::NonNull, - sync::atomic::{AtomicUsize, Ordering}, -}; +//! Allocation accounting. +//! +//! Every build tracks per-thread allocation totals, which the tracing layer reads through +//! [`allocation_counters`] to attribute allocations to spans. +//! +//! Builds without the `custom_allocator` feature additionally maintain a process-wide counter of +//! live bytes, which backs [`crate::TurboMalloc::memory_usage`]. With mimalloc that figure comes +//! from the allocator instead, so [`global`] is not compiled in: the atomic would otherwise be +//! contended by every thread on every allocation, and the thread-local buffering that makes it +//! affordable is inlined into every allocation site in the binary. +use std::{cell::UnsafeCell, ptr::NonNull}; + +#[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] +pub use self::global::get; use crate::AllocationCounters; -/// Tracks the current total amount of memory allocated through all the [ThreadLocalCounter] -/// instances. This is an overestimate as individual threads 'preallocate' a [TARGET_BUFFER] bytes -/// to reduce the number of global synchronizations. This means at any given time this might -/// overcount by up to [MAX_BUFFER] bytes for each thread. -static ALLOCATED: AtomicUsize = AtomicUsize::new(0); -const KB: usize = 1024; -/// When global counter is updates we will keep a thread-local buffer of this -/// size. -const TARGET_BUFFER: usize = 100 * KB; -/// When the thread-local buffer would exceed this size, we will update the -/// global counter. -const MAX_BUFFER: usize = 200 * KB; +/// The process-wide live-bytes counter, and the buffering that keeps updating it affordable. +/// +/// Only compiled without the `custom_allocator` feature; see the module docs. Each thread holds +/// its buffer in its own [`ThreadLocalCounter`] and passes it in, so the counter's state lives in +/// exactly one place. +#[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] +mod global { + use std::sync::atomic::{AtomicUsize, Ordering}; + + /// Tracks the current total amount of memory allocated through all the + /// [`super::ThreadLocalCounter`] instances. This is an overestimate as individual threads + /// 'preallocate' a [TARGET_BUFFER] bytes to reduce the number of global synchronizations. + /// This means at any given time this might overcount by up to [MAX_BUFFER] bytes for each + /// thread. + static ALLOCATED: AtomicUsize = AtomicUsize::new(0); + const KB: usize = 1024; + /// When global counter is updates we will keep a thread-local buffer of this + /// size. + pub const TARGET_BUFFER: usize = 100 * KB; + /// When the thread-local buffer would exceed this size, we will update the + /// global counter. + pub const MAX_BUFFER: usize = 200 * KB; + + /// Live bytes (allocations minus deallocations) across all threads. + pub fn get() -> usize { + ALLOCATED.load(Ordering::Relaxed) + } + + /// Takes `size` from the global counter, refilling `buffer` while it is there. + /// + /// Kept out of the allocator's inlined hot path: the buffer means this runs about once per + /// [`TARGET_BUFFER`] bytes rather than once per allocation. + #[inline(never)] + pub fn refill(buffer: &mut usize, size: usize) { + debug_assert!(*buffer < size); + let offset = size - *buffer + TARGET_BUFFER; + *buffer = TARGET_BUFFER; + ALLOCATED.fetch_add(offset, Ordering::Relaxed); + } + + /// Returns everything buffered above [`TARGET_BUFFER`] to the global counter. + #[inline(never)] + pub fn flush_excess(buffer: &mut usize) { + debug_assert!(*buffer > MAX_BUFFER); + let offset = *buffer - TARGET_BUFFER; + *buffer = TARGET_BUFFER; + ALLOCATED.fetch_sub(offset, Ordering::Relaxed); + } + + /// Returns everything buffered, for a thread that is going away. + pub fn flush_all(buffer: &mut usize) { + if *buffer > 0 { + ALLOCATED.fetch_sub(*buffer, Ordering::Relaxed); + *buffer = 0; + } + } + + impl super::ThreadLocalCounter { + /// Charges `size` against this thread's buffer, refilling it from the global counter when + /// it runs dry. Does nothing with `custom_allocator`, where there is no global + /// counter. + #[inline(always)] + pub(super) fn buffered_add(&mut self, size: usize) { + if self.buffer >= size { + self.buffer -= size; + } else { + refill(&mut self.buffer, size); + } + } + + /// Returns `size` to this thread's buffer, flushing the excess to the global counter once + /// the buffer grows past [`global::MAX_BUFFER`]. Does nothing with + /// `custom_allocator`. + #[inline(always)] + pub(super) fn buffered_remove(&mut self, size: usize) { + self.buffer += size; + if self.buffer > MAX_BUFFER { + flush_excess(&mut self.buffer); + } + } + } +} +/// Per-thread allocation and deallocation totals. #[derive(Default)] struct ThreadLocalCounter { /// Thread-local buffer of allocated bytes that have been added to the /// global counter desprite not being allocated yet. It is unsigned so that /// means the global counter is always equal or greater than the real /// value. + #[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] buffer: usize, allocation_counters: AllocationCounters, } @@ -32,29 +113,28 @@ struct ThreadLocalCounter { impl ThreadLocalCounter { const fn new() -> Self { Self { + #[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] buffer: 0, allocation_counters: AllocationCounters::new(), } } + #[inline(always)] fn add(&mut self, size: usize) { self.allocation_counters.allocations += size; self.allocation_counters.allocation_count += 1; - if self.buffer >= size { - self.buffer -= size; - } else { - add_slow(self, size); - } + + #[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] + self.buffered_add(size); } #[inline(always)] fn remove(&mut self, size: usize) { self.allocation_counters.deallocations += size; self.allocation_counters.deallocation_count += 1; - self.buffer += size; - if self.buffer > MAX_BUFFER { - remove_slow(self); - } + + #[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] + self.buffered_remove(size); } #[inline(always)] @@ -63,62 +143,28 @@ impl ThreadLocalCounter { self.allocation_counters.deallocation_count += 1; self.allocation_counters.allocations += new_size; self.allocation_counters.allocation_count += 1; - match old_size.cmp(&new_size) { - std::cmp::Ordering::Equal => {} - std::cmp::Ordering::Less => { - let size = new_size - old_size; - if self.buffer >= size { - self.buffer -= size; - } else { - add_slow(self, size); - } - } - std::cmp::Ordering::Greater => { - let size = old_size - new_size; - self.buffer += size; - if self.buffer > MAX_BUFFER { - remove_slow(self); - } + + #[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] + { + match old_size.cmp(&new_size) { + std::cmp::Ordering::Equal => {} + std::cmp::Ordering::Less => self.buffered_add(new_size - old_size), + std::cmp::Ordering::Greater => self.buffered_remove(old_size - new_size), } } } fn unload(&mut self) { - if self.buffer > 0 { - ALLOCATED.fetch_sub(self.buffer, Ordering::Relaxed); - self.buffer = 0; - } + #[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] + global::flush_all(&mut self.buffer); self.allocation_counters = AllocationCounters::default(); } } -// Keep the uncommon atomic updates out of the allocator's inlined hot path. -#[cold] -#[inline(never)] -fn add_slow(local: &mut ThreadLocalCounter, size: usize) { - debug_assert!(local.buffer < size); - let offset = size - local.buffer + TARGET_BUFFER; - local.buffer = TARGET_BUFFER; - ALLOCATED.fetch_add(offset, Ordering::Relaxed); -} - -#[cold] -#[inline(never)] -fn remove_slow(local: &mut ThreadLocalCounter) { - debug_assert!(local.buffer > MAX_BUFFER); - let offset = local.buffer - TARGET_BUFFER; - local.buffer = TARGET_BUFFER; - ALLOCATED.fetch_sub(offset, Ordering::Relaxed); -} - thread_local! { static LOCAL_COUNTER: UnsafeCell = const {UnsafeCell::new(ThreadLocalCounter::new())}; } -pub fn get() -> usize { - ALLOCATED.load(Ordering::Relaxed) -} - pub fn allocation_counters() -> AllocationCounters { with_local_counter(|local| local.allocation_counters.clone()) } @@ -155,8 +201,8 @@ pub fn update(old_size: usize, new_size: usize) { with_local_counter(|local| local.update(old_size, new_size)); } -/// Flushes the thread-local buffer to the global counter. This should be called -/// e. g. when a thread is stopped or goes to sleep for a long time. +/// Clears this thread's counters. Called when a thread stops, so a recycled thread does not +/// inherit the previous occupant's totals. pub fn flush() { with_local_counter(|local| local.unload()); } @@ -166,42 +212,115 @@ mod tests { use super::*; #[test] - fn counting() { - let mut expected = get(); - add(100); - // Initial change should fill up the buffer - expected += TARGET_BUFFER + 100; - assert_eq!(get(), expected); + fn counts_allocations_and_deallocations() { + let start = allocation_counters(); + add(100); - // Further changes should use the buffer - assert_eq!(get(), expected); - add(MAX_BUFFER); - // Large changes should require more buffer space - expected += 100 + MAX_BUFFER; - assert_eq!(get(), expected); + add(250); remove(100); - // Small changes should use the buffer - // buffer size is now TARGET_BUFFER + 100 - assert_eq!(get(), expected); - remove(MAX_BUFFER); - // The buffer should not grow over MAX_BUFFER - // buffer size would be TARGET_BUFFER + 100 + MAX_BUFFER - // but it will be reduce to TARGET_BUFFER - // this means the global counter should reduce by 100 + MAX_BUFFER - expected -= MAX_BUFFER + 100; - assert_eq!(get(), expected); - - update(100, 200); - // Small reallocations should use the buffer. - assert_eq!(get(), expected); - update(0, MAX_BUFFER); - // Growing beyond the buffer should require more buffer space. The prior small growth - // consumed another 100 bytes from the buffer. - expected += MAX_BUFFER + 100; - assert_eq!(get(), expected); - update(MAX_BUFFER + 1, 0); - // Shrinking beyond MAX_BUFFER should flush the excess. - expected -= MAX_BUFFER + 1; - assert_eq!(get(), expected); + + let after = allocation_counters(); + assert_eq!(after.allocations - start.allocations, 350); + assert_eq!(after.allocation_count - start.allocation_count, 2); + assert_eq!(after.deallocations - start.deallocations, 100); + assert_eq!(after.deallocation_count - start.deallocation_count, 1); + } + + #[test] + fn update_counts_both_sides() { + let start = allocation_counters(); + + update(40, 100); + + let after = allocation_counters(); + assert_eq!(after.allocations - start.allocations, 100); + assert_eq!(after.allocation_count - start.allocation_count, 1); + assert_eq!(after.deallocations - start.deallocations, 40); + assert_eq!(after.deallocation_count - start.deallocation_count, 1); + } + + /// `reset_allocation_counters` restores a previously captured value, which is how the tracing + /// layer excludes its own writes from a span's totals. + #[test] + fn reset_restores_a_captured_value() { + let start = allocation_counters(); + add(4096); + assert!(allocation_counters().allocations > start.allocations); + + reset_allocation_counters(start.clone()); + assert_eq!(allocation_counters().allocations, start.allocations); + assert_eq!( + allocation_counters().allocation_count, + start.allocation_count + ); + } + + /// `flush` is called when a thread stops so a thread reusing the slot starts clean. + #[test] + fn flush_clears_this_threads_counters() { + std::thread::spawn(|| { + add(1234); + assert!(allocation_counters().allocations >= 1234); + flush(); + let cleared = allocation_counters(); + assert_eq!(cleared.allocations, 0); + assert_eq!(cleared.allocation_count, 0); + assert_eq!(cleared.deallocations, 0); + assert_eq!(cleared.deallocation_count, 0); + }) + .join() + .unwrap(); + } + + /// The buffered global counter only exists without the `custom_allocator` feature. + /// + /// Asserts the buffering arithmetic on a [`ThreadLocalCounter`] directly: how much a thread + /// keeps buffered, and therefore when it has to touch the global. The global itself is not + /// read — it is process-wide, and this binary installs [`crate::TurboMalloc`] as its global + /// allocator, so every other thread moves it concurrently. `buffer` is thread-local and + /// exact. + #[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] + #[test] + fn counting() { + use super::global::{MAX_BUFFER, TARGET_BUFFER}; + + let mut local = ThreadLocalCounter::new(); + + // A fresh counter has nothing buffered, so the first allocation has to reach the global, + // taking a full TARGET_BUFFER while it is there. + local.add(100); + assert_eq!(local.buffer, TARGET_BUFFER); + + // Further small allocations come straight out of the buffer. + local.add(100); + assert_eq!(local.buffer, TARGET_BUFFER - 100); + + // An allocation larger than the buffer refills it from the global. + local.add(MAX_BUFFER); + assert_eq!(local.buffer, TARGET_BUFFER); + + // Frees go back into the buffer while it stays under MAX_BUFFER. + local.remove(100); + assert_eq!(local.buffer, TARGET_BUFFER + 100); + + // Past MAX_BUFFER the excess is flushed back to the global, down to TARGET_BUFFER. + local.remove(MAX_BUFFER); + assert_eq!(local.buffer, TARGET_BUFFER); + + // A reallocation that grows by less than the buffer is served locally. + local.update(100, 200); + assert_eq!(local.buffer, TARGET_BUFFER - 100); + + // One that grows beyond it refills. + local.update(0, MAX_BUFFER); + assert_eq!(local.buffer, TARGET_BUFFER); + + // One that shrinks beyond MAX_BUFFER flushes the excess. + local.update(MAX_BUFFER + 1, 0); + assert_eq!(local.buffer, TARGET_BUFFER); + + // Unloading returns whatever is still buffered. + local.unload(); + assert_eq!(local.buffer, 0); } } diff --git a/turbopack/crates/turbo-tasks-malloc/src/lib.rs b/turbopack/crates/turbo-tasks-malloc/src/lib.rs index 9b72c9583fac..70e16c7cd577 100644 --- a/turbopack/crates/turbo-tasks-malloc/src/lib.rs +++ b/turbopack/crates/turbo-tasks-malloc/src/lib.rs @@ -7,7 +7,7 @@ use std::{ ops::{Add, AddAssign}, }; -use self::counter::{add, flush, get, remove, update}; +use self::counter::{add, flush, remove, update}; #[derive(Default, Clone, Debug)] pub struct AllocationInfo { @@ -85,16 +85,46 @@ impl AllocationCounters { pub struct TurboMalloc; impl TurboMalloc { - /// Returns the current amount of live memory (bytes allocated minus freed) - /// tracked across all threads. + /// Returns the bytes mimalloc currently has committed from the OS. This measures what the + /// allocator holds rather than the process's total footprint, and it does not track frees in + /// lock step, since mimalloc reuses and purges pages on its own schedule. /// - /// For efficiency reasons every thread only synchronizes with this counter after ~100K bytes of - /// allocations or deallocations. So this could be off by as much as 100K*number of thread in - /// either direction. + /// See `current_commit` in [`mi_process_info`], which documents each figure mimalloc reports. + /// + /// [`mi_process_info`]: https://docs.rs/libmimalloc-sys/latest/libmimalloc_sys/fn.mi_process_info.html + /// + /// Without the `custom_allocator` feature this is a process-wide live-bytes counter instead, + /// which is approximate because threads buffer their updates. pub fn memory_usage() -> usize { - get() + #[cfg(all(feature = "custom_allocator", not(target_family = "wasm")))] + { + // `current_commit` is a relaxed atomic load, but `mi_process_info` also calls + // `_mi_prim_process_info`, which is a `getrusage` (plus a `task_info` on macOS). All + // eight out-params are optional, so ask only for the one we use. + let mut current_commit = 0usize; + // Safety: every out-param is either null or a valid `usize` we own. + unsafe { + libmimalloc_sys::mi_process_info( + /* elapsed_msecs */ std::ptr::null_mut(), + /* user_msecs */ std::ptr::null_mut(), + /* system_msecs */ std::ptr::null_mut(), + /* current_rss */ std::ptr::null_mut(), + /* peak_rss */ std::ptr::null_mut(), + &mut current_commit, + /* peak_commit */ std::ptr::null_mut(), + /* page_faults */ std::ptr::null_mut(), + ); + } + current_commit + } + #[cfg(not(all(feature = "custom_allocator", not(target_family = "wasm"))))] + { + self::counter::get() + } } + /// Clears the calling thread's allocation counters. Call this when a thread is about to stop, + /// so a thread that reuses its slot does not inherit the previous totals. pub fn thread_stop() { flush(); } @@ -205,6 +235,36 @@ unsafe impl GlobalAlloc for TurboMalloc { mod tests { use super::TurboMalloc; + // `memory_usage` reports what *this* allocator has committed, so the test binary has to + // actually route its allocations through it. Without this the `vec!` below goes to the + // system allocator and mimalloc's counter never moves. + #[global_allocator] + static ALLOC: TurboMalloc = TurboMalloc; + + /// Guards against the counter silently becoming unavailable. + #[test] + fn memory_usage_is_reported_and_tracks_a_large_allocation() { + let before = TurboMalloc::memory_usage(); + assert!(before > 0, "a running process has live memory"); + + // Large enough to dwarf whatever else the test process does concurrently, and written to + // so the pages are actually committed. + const SIZE: usize = 256 * 1024 * 1024; + let mut buffer = vec![0u8; SIZE]; + for chunk in buffer.chunks_mut(4096) { + chunk[0] = 1; + } + std::hint::black_box(&buffer); + + let after = TurboMalloc::memory_usage(); + assert!( + after >= before + SIZE / 2, + "expected a rise of at least {} bytes, got {before} -> {after}", + SIZE / 2 + ); + drop(buffer); + } + #[test] fn memory_pressure_is_in_range() { let value = TurboMalloc::memory_pressure();