[libpas] Use VA-based memory-zeroing for very large allocations

https://bugs.webkit.org/show_bug.cgi?id=259938
rdar://113577292

Reviewed by Keith Miller.

Let's try using VA-based zeroing for very large allocations in libpas.
VA-based one has some cost, but still useful if the allocated memory is not used
so much actually. We already have a path avoiding this zeroing if the allocated VA
is already zero, but if not, let's take this path to zeroing them via mmap.

* Source/bmalloc/CMakeLists.txt:
* Source/bmalloc/bmalloc.xcodeproj/project.pbxproj:
* Source/bmalloc/libpas/libpas.xcodeproj/project.pbxproj:
* Source/bmalloc/libpas/src/libpas/pas_allocation_result.c: Added.
(pas_allocation_result_zero_large_slow):
* Source/bmalloc/libpas/src/libpas/pas_allocation_result.h:
(pas_allocation_result_zero):
* Source/bmalloc/libpas/src/libpas/pas_internal_config.h:

Canonical link: https://commits.webkit.org/266698@main

Source/bmalloc/CMakeLists.txt

@@ -56,6 +56,7 @@ set(bmalloc_SOURCES
     libpas/src/libpas/pas_alignment.c
     libpas/src/libpas/pas_all_heaps.c
     libpas/src/libpas/pas_allocation_callbacks.c
+    libpas/src/libpas/pas_allocation_result.c
     libpas/src/libpas/pas_all_shared_page_directories.c
     libpas/src/libpas/pas_baseline_allocator.c
     libpas/src/libpas/pas_baseline_allocator_table.c

Source/bmalloc/bmalloc.xcodeproj/project.pbxproj

@@ -667,6 +667,7 @@
 		E328D84D23CEB38900545B18 /* Packed.h in Headers */ = {isa = PBXBuildFile; fileRef = E328D84C23CEB38900545B18 /* Packed.h */; settings = {ATTRIBUTES = (Private, ); }; };
 		E378A9DF246B68720029C2BB /* ObjectTypeTable.cpp in Sources */ = {isa = PBXBuildFile; fileRef = E378A9DE246B686A0029C2BB /* ObjectTypeTable.cpp */; };
 		E378A9E0246B68750029C2BB /* ObjectTypeTable.h in Headers */ = {isa = PBXBuildFile; fileRef = E378A9DD246B686A0029C2BB /* ObjectTypeTable.h */; settings = {ATTRIBUTES = (Private, ); }; };
+		E39ECE2D2A82360400BF878E /* pas_allocation_result.c in Sources */ = {isa = PBXBuildFile; fileRef = E39ECE2C2A82360400BF878E /* pas_allocation_result.c */; };
 		E3A413C9226061140037F470 /* IsoSharedPageInlines.h in Headers */ = {isa = PBXBuildFile; fileRef = E3A413C8226061140037F470 /* IsoSharedPageInlines.h */; settings = {ATTRIBUTES = (Private, ); }; };
 		E3F24402225D2C0100A0E0C3 /* IsoSharedPage.h in Headers */ = {isa = PBXBuildFile; fileRef = E3F24401225D2C0100A0E0C3 /* IsoSharedPage.h */; settings = {ATTRIBUTES = (Private, ); }; };
 		E3F24404225D2C7600A0E0C3 /* IsoSharedPage.cpp in Sources */ = {isa = PBXBuildFile; fileRef = E3F24403225D2C7600A0E0C3 /* IsoSharedPage.cpp */; };
@@ -1352,6 +1353,7 @@
 		E378A9DD246B686A0029C2BB /* ObjectTypeTable.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = ObjectTypeTable.h; path = bmalloc/ObjectTypeTable.h; sourceTree = "<group>"; };
 		E378A9DE246B686A0029C2BB /* ObjectTypeTable.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = ObjectTypeTable.cpp; path = bmalloc/ObjectTypeTable.cpp; sourceTree = "<group>"; };
 		E38A9E3B27426514000BBD49 /* pas_platform.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = pas_platform.h; path = libpas/src/libpas/pas_platform.h; sourceTree = "<group>"; };
+		E39ECE2C2A82360400BF878E /* pas_allocation_result.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = pas_allocation_result.c; path = libpas/src/libpas/pas_allocation_result.c; sourceTree = "<group>"; };
 		E3A413C8226061140037F470 /* IsoSharedPageInlines.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = IsoSharedPageInlines.h; path = bmalloc/IsoSharedPageInlines.h; sourceTree = "<group>"; };
 		E3F24401225D2C0100A0E0C3 /* IsoSharedPage.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = IsoSharedPage.h; path = bmalloc/IsoSharedPage.h; sourceTree = "<group>"; };
 		E3F24403225D2C7600A0E0C3 /* IsoSharedPage.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = IsoSharedPage.cpp; path = bmalloc/IsoSharedPage.cpp; sourceTree = "<group>"; };
@@ -1515,6 +1517,7 @@
 				0FC409892451496000876DA0 /* pas_allocation_callbacks.h */,
 				0FC409812451495F00876DA0 /* pas_allocation_config.h */,
 				0FC409772451495F00876DA0 /* pas_allocation_kind.h */,
+				E39ECE2C2A82360400BF878E /* pas_allocation_result.c */,
 				0FC409BE2451496300876DA0 /* pas_allocation_result.h */,
 				0FC409AA2451496200876DA0 /* pas_allocator_counts.h */,
 				0F7549802486973C002A4C7D /* pas_allocator_index.h */,
@@ -2845,6 +2848,7 @@
 				DD4BED2529CBA49700398E35 /* pas_all_heaps.c in Sources */,
 				DD4BED8829CBA49700398E35 /* pas_all_shared_page_directories.c in Sources */,
 				DD4BED6629CBA49700398E35 /* pas_allocation_callbacks.c in Sources */,
+				E39ECE2D2A82360400BF878E /* pas_allocation_result.c in Sources */,
 				DD4BECDF29CBA49700398E35 /* pas_baseline_allocator.c in Sources */,
 				DD4BEDAD29CBA49700398E35 /* pas_baseline_allocator_table.c in Sources */,
 				DD4BEC4F29CBA49700398E35 /* pas_basic_heap_config_enumerator_data.c in Sources */,

Source/bmalloc/libpas/libpas.xcodeproj/project.pbxproj

@@ -553,7 +553,7 @@
 		2B2E2FD22949A41100F85C38 /* pas_malloc_stack_logging.h in Headers */ = {isa = PBXBuildFile; fileRef = 2B2E2FD02949A41100F85C38 /* pas_malloc_stack_logging.h */; };
 		2B6055E42805368B00C8BDAC /* BmallocTests.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2B6055E32805368B00C8BDAC /* BmallocTests.cpp */; };
 		2BDF4F4529E8B36F0056BF50 /* pas_report_crash.h in Headers */ = {isa = PBXBuildFile; fileRef = 2BDF4F4429E8B36F0056BF50 /* pas_report_crash.h */; };
-		2BDF4F4729E8B3AD0056BF50 /* pas_report_crash.c in Sources */ = {isa = PBXBuildFile; fileRef = 2BDF4F4629E8B3AD0056BF50 /* pas_report_crash.c */; };
+		2BDF4F4729E8B3AD0056BF50 /* (null) in Sources */ = {isa = PBXBuildFile; };
 		2BF3F5B529A0051F005361FD /* EnumerationTests.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2BF3F5B429A0051F005361FD /* EnumerationTests.cpp */; };
 		2C11E88E2728A783002162D0 /* bmalloc_type.h in Headers */ = {isa = PBXBuildFile; fileRef = 2C11E88C2728A783002162D0 /* bmalloc_type.h */; };
 		2C11E88F2728A783002162D0 /* pas_simple_type.c in Sources */ = {isa = PBXBuildFile; fileRef = 2C11E88D2728A783002162D0 /* pas_simple_type.c */; };
@@ -590,6 +590,7 @@
 		2CB9B15B278F85EE003A8C1B /* pas_lenient_compact_ptr_inlines.h in Headers */ = {isa = PBXBuildFile; fileRef = 2CB9B157278F85EE003A8C1B /* pas_lenient_compact_ptr_inlines.h */; };
 		2CB9B15D278F861B003A8C1B /* LotsOfHeapsAndThreads.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2CB9B15C278F861B003A8C1B /* LotsOfHeapsAndThreads.cpp */; };
 		2CE2AE35275A953E00D02BBC /* BitfitTests.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 2CE2AE34275A953E00D02BBC /* BitfitTests.cpp */; };
+		E3096D4C2A82357800BC4CA0 /* pas_allocation_result.c in Sources */ = {isa = PBXBuildFile; fileRef = E3096D4B2A82357800BC4CA0 /* pas_allocation_result.c */; };
 		E3AA9B8328C724D8005DF9D6 /* pas_darwin_spi.h in Headers */ = {isa = PBXBuildFile; fileRef = E3AA9B8228C724D8005DF9D6 /* pas_darwin_spi.h */; };
 /* End PBXBuildFile section */
 
@@ -1302,6 +1303,7 @@
 		2CB9B157278F85EE003A8C1B /* pas_lenient_compact_ptr_inlines.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_lenient_compact_ptr_inlines.h; sourceTree = "<group>"; };
 		2CB9B15C278F861B003A8C1B /* LotsOfHeapsAndThreads.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = LotsOfHeapsAndThreads.cpp; sourceTree = "<group>"; };
 		2CE2AE34275A953E00D02BBC /* BitfitTests.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = BitfitTests.cpp; sourceTree = "<group>"; };
+		E3096D4B2A82357800BC4CA0 /* pas_allocation_result.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = pas_allocation_result.c; sourceTree = "<group>"; };
 		E3AA9B8228C724D8005DF9D6 /* pas_darwin_spi.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = pas_darwin_spi.h; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
@@ -1490,6 +1492,7 @@
 				0F47C31D23396E0B002A1CF7 /* pas_allocation_callbacks.h */,
 				0F78088C22FA534100F37451 /* pas_allocation_config.h */,
 				0F47C31823396D1F002A1CF7 /* pas_allocation_kind.h */,
+				E3096D4B2A82357800BC4CA0 /* pas_allocation_result.c */,
 				0FC681B221065A00003C6A13 /* pas_allocation_result.h */,
 				0FDBB07A22BBFF83006BA5FC /* pas_allocator_counts.h */,
 				0F2150EA2484BB2A000D634B /* pas_allocator_index.h */,
@@ -2700,6 +2703,7 @@
 			isa = PBXSourcesBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
+				2BDF4F4729E8B3AD0056BF50 /* (null) in Sources */,
 				0F8A806625EECC1D00790B4A /* bmalloc_heap.c in Sources */,
 				0F8A806825EECC1D00790B4A /* bmalloc_heap_config.c in Sources */,
 				2C11E8912728A893002162D0 /* bmalloc_type.c in Sources */,
@@ -2719,6 +2723,7 @@
 				0FE7EDB522960142004F4166 /* pas_all_heaps.c in Sources */,
 				0FD48B4623A9ABB30026C46D /* pas_all_shared_page_directories.c in Sources */,
 				0F47C31E23396E0B002A1CF7 /* pas_allocation_callbacks.c in Sources */,
+				E3096D4C2A82357800BC4CA0 /* pas_allocation_result.c in Sources */,
 				0FD48B6323A9ABB30026C46D /* pas_baseline_allocator.c in Sources */,
 				0FD48B3F23A9ABB30026C46D /* pas_baseline_allocator_table.c in Sources */,
 				0F59A39C266844A300A38657 /* pas_basic_heap_config_enumerator_data.c in Sources */,
@@ -2811,7 +2816,6 @@
 				0FD48B3023A9ABB30026C46D /* pas_random.c in Sources */,
 				0FF08F3422A58F1200386575 /* pas_red_black_tree.c in Sources */,
 				0FA5E4592492D5BA00CE962A /* pas_redundant_local_allocator_node.c in Sources */,
-				2BDF4F4729E8B3AD0056BF50 /* pas_report_crash.c in Sources */,
 				0F5B6091235E88EF00CAE629 /* pas_reserved_memory_provider.c in Sources */,
 				0F4F60FA25979BC8008B4A82 /* pas_root.c in Sources */,
 				0F19326D22F73E8500FBA713 /* pas_scavenger.c in Sources */,

Source/bmalloc/libpas/src/libpas/pas_allocation_result.c

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2023 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pas_config.h"
+
+#if LIBPAS_ENABLED
+
+#include "pas_allocation_result.h"
+#include "pas_page_malloc.h"
+
+pas_allocation_result pas_allocation_result_zero_large_slow(pas_allocation_result result, size_t size)
+{
+    size_t page_size;
+
+    page_size = pas_page_malloc_alignment();
+    if (pas_is_aligned(size, page_size) && pas_is_aligned(result.begin, page_size))
+        pas_page_malloc_zero_fill((void*)result.begin, size);
+    else
+        pas_zero_memory((void*)result.begin, size);
+    return pas_allocation_result_create_success_with_zero_mode(result.begin, pas_zero_mode_is_all_zero);
+}
+
+#endif /* LIBPAS_ENABLED */

Source/bmalloc/libpas/src/libpas/pas_allocation_result.h

@@ -27,6 +27,7 @@
 #define PAS_ALLOCATION_RESULT_H
 
 #include <errno.h>
+#include "pas_internal_config.h"
 #include "pas_utils.h"
 #include "pas_zero_mode.h"
 
@@ -78,17 +79,22 @@ pas_allocation_result_identity(pas_allocation_result result)
     return result;
 }
 
+PAS_API pas_allocation_result pas_allocation_result_zero_large_slow(pas_allocation_result result, size_t size);
+
 static PAS_ALWAYS_INLINE pas_allocation_result
 pas_allocation_result_zero(pas_allocation_result result,
                            size_t size)
 {
-    if (result.did_succeed
-        && result.zero_mode == pas_zero_mode_may_have_non_zero) {
-        pas_zero_memory((void*)result.begin, size);
-        return pas_allocation_result_create_success_with_zero_mode(
-            result.begin, pas_zero_mode_is_all_zero);
-    }
-    return result;
+    if (PAS_UNLIKELY(!result.did_succeed))
+        return result;
+    if (result.zero_mode == pas_zero_mode_is_all_zero)
+        return result;
+
+    if (size >= (1ULL << PAS_VA_BASED_ZERO_MEMORY_SHIFT))
+        return pas_allocation_result_zero_large_slow(result, size);
+
+    pas_zero_memory((void*)result.begin, size);
+    return pas_allocation_result_create_success_with_zero_mode(result.begin, pas_zero_mode_is_all_zero);
 }
 
 static PAS_ALWAYS_INLINE pas_allocation_result

Source/bmalloc/libpas/src/libpas/pas_internal_config.h

@@ -65,6 +65,9 @@
 #define PAS_MIN_ALIGN_SHIFT              4
 #define PAS_MIN_ALIGN                    ((size_t)1 << PAS_MIN_ALIGN_SHIFT)
 
+/* Use VA-based memory zeroing when the allocation size exceeds this threshold. */
+#define PAS_VA_BASED_ZERO_MEMORY_SHIFT   24
+
 /* This is the same as PAS_BITVECTOR_WORD_SHIFT, which is a nice performance optimization but
    isn't necessary. It's a performance optimization because there are specialized fast code
    paths for sharing_shift == PAS_BITVECTOR_WORD_SHIFT. */