From d77f4913a18ecce8c4be95cbaa4299ff1521dc10 Mon Sep 17 00:00:00 2001 From: Teagan Strickland Date: Fri, 11 Oct 2019 11:28:57 +0000 Subject: [PATCH] [vm/compiler] Further compress the information previously in StackMaps. Lifting the PC offset in a2bb730 was a small, lightweight change that gave us big gains, at least on 32-bit architectures. Here, we make much more invasive changes that will improve the amount of memory used by the information previously stored in StackMap objects. Instead of allocating separate objects for StackMaps, we instead compress all StackMap information for a given Code object into a single object (CompressedStackMaps, or CSM for short). This replaces the Array used to store PC offsets (as Smis) and the individual StackMap objects. While we lose all canonicalization for individual StackMap entries, the drop in space required to store stack map information more than offsets that. ----- The impact on AOT snapshot size when compiling the Flutter Gallery in release mode: armv7: Total size -2.58% (Isolate RO: +14.46%, Isolate snapshot: -22.93%) armv8: Total size -1.85% (Isolate RO: +15.69%, Isolate snapshot: -22.97%) The impact on in-memory, not on-disk, size for the Flutter Gallery as seen in the Observatory while running a profile (not release) build: armv7: Drops from 7.1 MB to 6.2MB (-0.9 MB) armv8: Drops from 13.5MB to 11.7MB (-1.8 MB) ----- Bug: https://github.com/dart-lang/sdk/issues/35274 Cq-Include-Trybots: luci.dart.try:vm-kernel-precomp-linux-debug-x64-try,vm-kernel-precomp-linux-debug-simarm_x64-try Change-Id: Ie3bb898d557215146260a560423f5fa27bdff512 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/119640 Commit-Queue: Teagan Strickland Reviewed-by: Martin Kustermann Reviewed-by: Ryan Macnak --- runtime/vm/bitmap.cc | 32 ++++ runtime/vm/bitmap.h | 2 + runtime/vm/bitmap_test.cc | 48 ++++- runtime/vm/class_id.h | 2 +- runtime/vm/clustered_snapshot.cc | 16 +- runtime/vm/code_descriptors.cc | 122 ++++++++---- runtime/vm/code_descriptors.h | 65 +++++-- runtime/vm/code_descriptors_test.cc | 13 +- runtime/vm/compiler/assembler/disassembler.cc | 13 +- .../compiler/backend/flow_graph_compiler.cc | 31 ++- .../vm/compiler/backend/flow_graph_compiler.h | 10 +- runtime/vm/image_snapshot.cc | 34 ++-- runtime/vm/object.cc | 178 +++++++----------- runtime/vm/object.h | 79 ++++---- runtime/vm/object_service.cc | 2 +- runtime/vm/program_visitor.cc | 83 ++++---- runtime/vm/program_visitor.h | 2 +- runtime/vm/raw_object.cc | 11 +- runtime/vm/raw_object.h | 44 +++-- runtime/vm/raw_object_fields.cc | 2 +- runtime/vm/raw_object_snapshot.cc | 2 +- runtime/vm/snapshot.h | 2 +- runtime/vm/stack_frame.cc | 24 ++- runtime/vm/symbols.h | 2 +- 24 files changed, 454 insertions(+), 365 deletions(-) diff --git a/runtime/vm/bitmap.cc b/runtime/vm/bitmap.cc index 79bab3c9514f..d5e25dd2be48 100644 --- a/runtime/vm/bitmap.cc +++ b/runtime/vm/bitmap.cc @@ -77,6 +77,38 @@ void BitmapBuilder::Print() const { } } +void BitmapBuilder::AppendAsBytesTo(GrowableArray* bytes) const { + // Early return if there are no bits in the payload to copy. + if (Length() == 0) return; + + const intptr_t total_size = + Utils::RoundUp(Length(), kBitsPerByte) / kBitsPerByte; + intptr_t payload_size; + intptr_t extra_size; + if (total_size > data_size_in_bytes_) { + // A [BitmapBuilder] does not allocate storage for the trailing 0 bits in + // the backing store, so we need to add additional empty bytes here. + payload_size = data_size_in_bytes_; + extra_size = total_size - data_size_in_bytes_; + } else { + payload_size = total_size; + extra_size = 0; + } + for (intptr_t i = 0; i < payload_size; i++) { + bytes->Add(data_[i]); + } + for (intptr_t i = 0; i < extra_size; i++) { + bytes->Add(0U); + } + // Make sure any bits in the payload beyond the bit length are cleared to + // ensure deterministic snapshots. +#if defined(DEBUG) + if (Length() % kBitsPerByte == 0) return; + const int8_t mask = (1 << (Length() % kBitsPerByte)) - 1; + ASSERT(bytes->Last() == (bytes->Last() & mask)); +#endif +} + bool BitmapBuilder::GetBit(intptr_t bit_offset) const { if (!InRange(bit_offset)) { return false; diff --git a/runtime/vm/bitmap.h b/runtime/vm/bitmap.h index 469153b88032..878a834b6087 100644 --- a/runtime/vm/bitmap.h +++ b/runtime/vm/bitmap.h @@ -6,6 +6,7 @@ #define RUNTIME_VM_BITMAP_H_ #include "vm/allocation.h" +#include "vm/growable_array.h" #include "vm/thread_state.h" #include "vm/zone.h" @@ -43,6 +44,7 @@ class BitmapBuilder : public ZoneAllocated { void SetRange(intptr_t min, intptr_t max, bool value); void Print() const; + void AppendAsBytesTo(GrowableArray* bytes) const; private: static const intptr_t kInitialSizeInBytes = 16; diff --git a/runtime/vm/bitmap_test.cc b/runtime/vm/bitmap_test.cc index a9a1addc6833..135d0a6c3cb8 100644 --- a/runtime/vm/bitmap_test.cc +++ b/runtime/vm/bitmap_test.cc @@ -3,12 +3,25 @@ // BSD-style license that can be found in the LICENSE file. #include "vm/bitmap.h" + #include "platform/assert.h" +#include "vm/code_descriptors.h" #include "vm/object.h" #include "vm/unit_test.h" namespace dart { +// 0x4 is just a placeholder PC offset because no entry of a CSM should +// have a PC offset of 0, otherwise internal assumptions break. +static const intptr_t kTestPcOffset = 0x4; +static const intptr_t kTestSpillSlotBitCount = 0; + +static RawCompressedStackMaps* MapsFromBuilder(BitmapBuilder* bmap) { + CompressedStackMapsBuilder builder; + builder.AddEntry(kTestPcOffset, bmap, kTestSpillSlotBitCount); + return builder.Finalize(); +} + ISOLATE_UNIT_TEST_CASE(BitmapBuilder) { // Test basic bit map builder operations. BitmapBuilder* builder1 = new BitmapBuilder(); @@ -36,16 +49,23 @@ ISOLATE_UNIT_TEST_CASE(BitmapBuilder) { EXPECT_EQ(value, builder1->Get(i)); value = !value; } - // Create a StackMap object from the builder and verify its contents. - const StackMap& stackmap1 = StackMap::Handle(StackMap::New(builder1, 0)); - EXPECT_EQ(1024, stackmap1.Length()); - OS::PrintErr("%s\n", stackmap1.ToCString()); + + // Create a CompressedStackMaps object and verify its contents. + const auto& maps1 = CompressedStackMaps::Handle(MapsFromBuilder(builder1)); + CompressedStackMapsIterator it1(maps1); + EXPECT(it1.MoveNext()); + + EXPECT_EQ(kTestPcOffset, it1.pc_offset()); + EXPECT_EQ(kTestSpillSlotBitCount, it1.spill_slot_bit_count()); + EXPECT_EQ(1024, it1.length()); value = true; for (int32_t i = 0; i < 1024; i++) { - EXPECT_EQ(value, stackmap1.IsObject(i)); + EXPECT_EQ(value, it1.IsObject(i)); value = !value; } + EXPECT(!it1.MoveNext()); + // Test the SetRange function in the builder. builder1->SetRange(0, 256, false); EXPECT_EQ(1024, builder1->Length()); @@ -62,18 +82,26 @@ ISOLATE_UNIT_TEST_CASE(BitmapBuilder) { for (int32_t i = 1025; i <= 2048; i++) { EXPECT(!builder1->Get(i)); } - const StackMap& stackmap2 = StackMap::Handle(StackMap::New(builder1, 0)); - EXPECT_EQ(2049, stackmap2.Length()); + + const auto& maps2 = CompressedStackMaps::Handle(MapsFromBuilder(builder1)); + CompressedStackMapsIterator it2(maps2); + EXPECT(it2.MoveNext()); + + EXPECT_EQ(kTestPcOffset, it2.pc_offset()); + EXPECT_EQ(kTestSpillSlotBitCount, it2.spill_slot_bit_count()); + EXPECT_EQ(2049, it2.length()); for (int32_t i = 0; i <= 256; i++) { - EXPECT(!stackmap2.IsObject(i)); + EXPECT(!it2.IsObject(i)); } for (int32_t i = 257; i <= 1024; i++) { - EXPECT(stackmap2.IsObject(i)); + EXPECT(it2.IsObject(i)); } for (int32_t i = 1025; i <= 2048; i++) { - EXPECT(!stackmap2.IsObject(i)); + EXPECT(!it2.IsObject(i)); } + EXPECT(!it2.MoveNext()); + // Test using SetLength to shorten the builder, followed by lengthening. builder1->SetLength(747); EXPECT_EQ(747, builder1->Length()); diff --git a/runtime/vm/class_id.h b/runtime/vm/class_id.h index 19998825655d..353039d6864b 100644 --- a/runtime/vm/class_id.h +++ b/runtime/vm/class_id.h @@ -31,7 +31,7 @@ namespace dart { V(ObjectPool) \ V(PcDescriptors) \ V(CodeSourceMap) \ - V(StackMap) \ + V(CompressedStackMaps) \ V(LocalVarDescriptors) \ V(ExceptionHandlers) \ V(Context) \ diff --git a/runtime/vm/clustered_snapshot.cc b/runtime/vm/clustered_snapshot.cc index 3a24079bf5af..45f050975fb3 100644 --- a/runtime/vm/clustered_snapshot.cc +++ b/runtime/vm/clustered_snapshot.cc @@ -1387,7 +1387,7 @@ class CodeSerializationCluster : public SerializationCluster { #else s->Push(code->ptr()->catch_entry_.variables_); #endif - s->Push(code->ptr()->stackmaps_); + s->Push(code->ptr()->compressed_stackmaps_); if (!FLAG_dwarf_stack_traces) { s->Push(code->ptr()->inlined_id_to_function_); s->Push(code->ptr()->code_source_map_); @@ -1445,7 +1445,7 @@ class CodeSerializationCluster : public SerializationCluster { #else WriteField(code, catch_entry_.variables_); #endif - WriteField(code, stackmaps_); + WriteField(code, compressed_stackmaps_); if (FLAG_dwarf_stack_traces) { WriteFieldValue(inlined_id_to_function_, Array::null()); WriteFieldValue(code_source_map_, CodeSourceMap::null()); @@ -1555,7 +1555,8 @@ class CodeDeserializationCluster : public DeserializationCluster { code->ptr()->catch_entry_.variables_ = reinterpret_cast(d->ReadRef()); #endif - code->ptr()->stackmaps_ = reinterpret_cast(d->ReadRef()); + code->ptr()->compressed_stackmaps_ = + reinterpret_cast(d->ReadRef()); code->ptr()->inlined_id_to_function_ = reinterpret_cast(d->ReadRef()); code->ptr()->code_source_map_ = @@ -1902,7 +1903,7 @@ class PcDescriptorsDeserializationCluster : public DeserializationCluster { }; #if !defined(DART_PRECOMPILED_RUNTIME) -// PcDescriptor, StackMap, OneByteString, TwoByteString +// PcDescriptor, CompressedStackMaps, OneByteString, TwoByteString class RODataSerializationCluster : public SerializationCluster { public: RODataSerializationCluster(const char* name, intptr_t cid) @@ -4447,8 +4448,9 @@ SerializationCluster* Serializer::NewClusterForClass(intptr_t cid) { return new (Z) RODataSerializationCluster("(RO)PcDescriptors", cid); case kCodeSourceMapCid: return new (Z) RODataSerializationCluster("(RO)CodeSourceMap", cid); - case kStackMapCid: - return new (Z) RODataSerializationCluster("(RO)StackMap", cid); + case kCompressedStackMapsCid: + return new (Z) + RODataSerializationCluster("(RO)CompressedStackMaps", cid); case kOneByteStringCid: return new (Z) RODataSerializationCluster("(RO)OneByteString", cid); case kTwoByteStringCid: @@ -5070,7 +5072,7 @@ DeserializationCluster* Deserializer::ReadCluster() { switch (cid) { case kPcDescriptorsCid: case kCodeSourceMapCid: - case kStackMapCid: + case kCompressedStackMapsCid: case kOneByteStringCid: case kTwoByteStringCid: return new (Z) RODataDeserializationCluster(); diff --git a/runtime/vm/code_descriptors.cc b/runtime/vm/code_descriptors.cc index fd5fd33bbf6a..a01bd88d8d47 100644 --- a/runtime/vm/code_descriptors.cc +++ b/runtime/vm/code_descriptors.cc @@ -46,46 +46,102 @@ RawPcDescriptors* DescriptorList::FinalizePcDescriptors(uword entry_point) { return PcDescriptors::New(&encoded_data_); } -void StackMapTableBuilder::AddEntry(intptr_t pc_offset, - BitmapBuilder* bitmap, - intptr_t register_bit_count) { - ASSERT(Smi::IsValid(pc_offset)); - pc_offset_ = Smi::New(pc_offset); - stack_map_ = StackMap::New(bitmap, register_bit_count); - list_.Add(pc_offset_, Heap::kOld); - list_.Add(stack_map_, Heap::kOld); -} - -bool StackMapTableBuilder::Verify() { - intptr_t num_entries = Length(); - for (intptr_t i = 1; i < num_entries; i++) { - pc_offset_ = OffsetAt(i - 1); - auto const offset1 = pc_offset_.Value(); - pc_offset_ = OffsetAt(i); - auto const offset2 = pc_offset_.Value(); - // Ensure there are no duplicates and the entries are sorted. - if (offset1 >= offset2) return false; +// Encode unsigned integer |value| in LEB128 format and store into |data|. +static void EncodeLEB128(GrowableArray* data, uintptr_t value) { + while (true) { + uint8_t part = value & 0x7f; + value >>= 7; + if (value != 0) part |= 0x80; + data->Add(part); + if (value == 0) break; } - return true; } -RawArray* StackMapTableBuilder::FinalizeStackMaps(const Code& code) { - ASSERT(Verify()); - intptr_t num_entries = Length(); - if (num_entries == 0) { - return Object::empty_array().raw(); - } - return Array::MakeFixedLength(list_); +void CompressedStackMapsBuilder::AddEntry(intptr_t pc_offset, + BitmapBuilder* bitmap, + intptr_t spill_slot_bit_count) { + ASSERT(bitmap != nullptr); + ASSERT(pc_offset > last_pc_offset_); + ASSERT(spill_slot_bit_count >= 0 && spill_slot_bit_count <= bitmap->Length()); + auto const pc_delta = pc_offset - last_pc_offset_; + auto const non_spill_slot_bit_count = bitmap->Length() - spill_slot_bit_count; + EncodeLEB128(&encoded_bytes_, pc_delta); + EncodeLEB128(&encoded_bytes_, spill_slot_bit_count); + EncodeLEB128(&encoded_bytes_, non_spill_slot_bit_count); + bitmap->AppendAsBytesTo(&encoded_bytes_); + last_pc_offset_ = pc_offset; +} + +RawCompressedStackMaps* CompressedStackMapsBuilder::Finalize() const { + if (encoded_bytes_.length() == 0) return CompressedStackMaps::null(); + return CompressedStackMaps::New(encoded_bytes_); } -RawSmi* StackMapTableBuilder::OffsetAt(intptr_t index) const { - pc_offset_ ^= list_.At(2 * index); - return pc_offset_.raw(); +// Decode unsigned integer in LEB128 format from |data| and update |byte_index|. +static uintptr_t DecodeLEB128(const uint8_t* data, + const intptr_t data_length, + intptr_t* byte_index) { + ASSERT(*byte_index < data_length); + uword shift = 0; + uintptr_t value = 0; + uint8_t part = 0; + do { + part = data[(*byte_index)++]; + value |= static_cast(part & 0x7f) << shift; + shift += 7; + } while ((part & 0x80) != 0); + + return value; +} + +bool CompressedStackMapsIterator::MoveNext() { + // Empty CompressedStackMaps are represented as null values. + if (maps_.IsNull() || next_offset_ >= maps_.payload_size()) return false; + intptr_t offset = next_offset_; + + // We decode three LEB128 encoded integers after this, so there should be + // at least three bytes remaining in the payload. + ASSERT(offset <= maps_.payload_size() - 3); + auto const pc_delta = + DecodeLEB128(maps_.Payload(), maps_.payload_size(), &offset); + ASSERT(pc_delta <= kIntptrMax); + + ASSERT(offset <= maps_.payload_size() - 2); + auto const spill_slot_bit_count = + DecodeLEB128(maps_.Payload(), maps_.payload_size(), &offset); + ASSERT(spill_slot_bit_count <= kIntptrMax); + + ASSERT(offset <= maps_.payload_size() - 1); + auto const non_spill_slot_bit_count = + DecodeLEB128(maps_.Payload(), maps_.payload_size(), &offset); + ASSERT(non_spill_slot_bit_count <= kIntptrMax); + + const auto stackmap_bits = spill_slot_bit_count + non_spill_slot_bit_count; + const intptr_t stackmap_size = + Utils::RoundUp(stackmap_bits, kBitsPerByte) >> kBitsPerByteLog2; + const intptr_t space_remaining = maps_.payload_size() - offset; + if (stackmap_size > space_remaining) return false; + + // Now that the current entry has been completely decoded without errors, set + // the fields appropriately. + ASSERT(current_pc_offset_ < (kIntptrMax - static_cast(pc_delta))); + current_pc_offset_ += pc_delta; + current_spill_slot_bit_count_ = spill_slot_bit_count; + current_non_spill_slot_bit_count_ = non_spill_slot_bit_count; + current_bits_offset_ = offset; + next_offset_ = offset + stackmap_size; + + return true; } -RawStackMap* StackMapTableBuilder::MapAt(intptr_t index) const { - stack_map_ ^= list_.At(2 * index + 1); - return stack_map_.raw(); +bool CompressedStackMapsIterator::IsObject(intptr_t bit_index) const { + ASSERT(HasLoadedEntry()); + ASSERT(bit_index >= 0 && bit_index < length()); + const intptr_t byte_index = bit_index >> kBitsPerByteLog2; + const intptr_t bit_remainder = bit_index & (kBitsPerByte - 1); + uint8_t byte_mask = 1U << bit_remainder; + uint8_t byte = maps_.Payload()[current_bits_offset_ + byte_index]; + return (byte & byte_mask) != 0; } RawExceptionHandlers* ExceptionHandlerList::FinalizeExceptionHandlers( diff --git a/runtime/vm/code_descriptors.h b/runtime/vm/code_descriptors.h index e77a651b74a2..a1ba7287b1e9 100644 --- a/runtime/vm/code_descriptors.h +++ b/runtime/vm/code_descriptors.h @@ -44,32 +44,61 @@ class DescriptorList : public ZoneAllocated { DISALLOW_COPY_AND_ASSIGN(DescriptorList); }; -class StackMapTableBuilder : public ZoneAllocated { +class CompressedStackMapsBuilder : public ZoneAllocated { public: - StackMapTableBuilder() - : pc_offset_(Smi::ZoneHandle()), - stack_map_(StackMap::ZoneHandle()), - list_(GrowableObjectArray::ZoneHandle( - GrowableObjectArray::New(Heap::kOld))) {} - ~StackMapTableBuilder() {} + CompressedStackMapsBuilder() : encoded_bytes_() {} void AddEntry(intptr_t pc_offset, BitmapBuilder* bitmap, - intptr_t register_bit_count); + intptr_t spill_slot_bit_count); - bool Verify(); + RawCompressedStackMaps* Finalize() const; - RawArray* FinalizeStackMaps(const Code& code); + private: + intptr_t last_pc_offset_ = 0; + GrowableArray encoded_bytes_; + DISALLOW_COPY_AND_ASSIGN(CompressedStackMapsBuilder); +}; + +class CompressedStackMapsIterator : public ValueObject { + public: + explicit CompressedStackMapsIterator(const CompressedStackMaps& maps) + : maps_(maps) {} + + bool MoveNext(); + bool Find(intptr_t pc_offset) { + ASSERT(pc_offset > 0); + do { + if (current_pc_offset_ == pc_offset) return true; + } while (MoveNext()); + return false; + } + + intptr_t pc_offset() const { + ASSERT(HasLoadedEntry()); + return current_pc_offset_; + } + intptr_t length() const { + ASSERT(HasLoadedEntry()); + return current_spill_slot_bit_count_ + current_non_spill_slot_bit_count_; + } + intptr_t spill_slot_bit_count() const { + ASSERT(HasLoadedEntry()); + return current_spill_slot_bit_count_; + } + bool IsObject(intptr_t bit_offset) const; private: - intptr_t Length() const { return list_.Length() / 2; } - RawSmi* OffsetAt(intptr_t index) const; - RawStackMap* MapAt(intptr_t index) const; - - Smi& pc_offset_; - StackMap& stack_map_; - GrowableObjectArray& list_; - DISALLOW_COPY_AND_ASSIGN(StackMapTableBuilder); + // Since PC offsets are return addresses, we're guaranteed that the PC offset + // for a particular stack map entry must be > 0. + bool HasLoadedEntry() const { return current_pc_offset_ > 0; } + + const CompressedStackMaps& maps_; + intptr_t next_offset_ = 0; + intptr_t current_pc_offset_ = 0; + intptr_t current_spill_slot_bit_count_ = -1; + intptr_t current_non_spill_slot_bit_count_ = -1; + intptr_t current_bits_offset_ = -1; }; class ExceptionHandlerList : public ZoneAllocated { diff --git a/runtime/vm/code_descriptors_test.cc b/runtime/vm/code_descriptors_test.cc index 5d712edf0e62..d959adff7e50 100644 --- a/runtime/vm/code_descriptors_test.cc +++ b/runtime/vm/code_descriptors_test.cc @@ -84,10 +84,8 @@ TEST_CASE(StackMapGC) { // Build and setup a stackmap for the call to 'func' in 'A.foo' in order // to test the traversal of stack maps when a GC happens. - StackMapTableBuilder* stackmap_table_builder = new StackMapTableBuilder(); - EXPECT(stackmap_table_builder != NULL); BitmapBuilder* stack_bitmap = new BitmapBuilder(); - EXPECT(stack_bitmap != NULL); + EXPECT(stack_bitmap != nullptr); stack_bitmap->Set(0, false); // var i. stack_bitmap->Set(1, true); // var s1. stack_bitmap->Set(2, false); // var k. @@ -99,16 +97,17 @@ TEST_CASE(StackMapGC) { PcDescriptors::Handle(code.pc_descriptors()); int call_count = 0; PcDescriptors::Iterator iter(descriptors, RawPcDescriptors::kUnoptStaticCall); + CompressedStackMapsBuilder compressed_maps_builder; while (iter.MoveNext()) { - stackmap_table_builder->AddEntry(iter.PcOffset(), stack_bitmap, 0); + compressed_maps_builder.AddEntry(iter.PcOffset(), stack_bitmap, 0); ++call_count; } // We can't easily check that we put the stackmap at the correct pc, but // we did if there was exactly one call seen. EXPECT(call_count == 1); - const Array& stack_maps = - Array::Handle(stackmap_table_builder->FinalizeStackMaps(code)); - code.set_stackmaps(stack_maps); + const auto& compressed_maps = + CompressedStackMaps::Handle(compressed_maps_builder.Finalize()); + code.set_compressed_stackmaps(compressed_maps); // Now invoke 'A.moo' and it will trigger a GC when the native function // is called, this should then cause the stack map of function 'A.foo' diff --git a/runtime/vm/compiler/assembler/disassembler.cc b/runtime/vm/compiler/assembler/disassembler.cc index 64cd1affb1b5..b74310c7b834 100644 --- a/runtime/vm/compiler/assembler/disassembler.cc +++ b/runtime/vm/compiler/assembler/disassembler.cc @@ -279,15 +279,10 @@ void Disassembler::DisassembleCodeHelper(const char* function_fullname, #endif // !defined(DART_PRECOMPILED_RUNTIME) THR_Print("StackMaps for function '%s' {\n", function_fullname); - if (code.stackmaps() != Array::null()) { - const Array& stackmap_table = Array::Handle(zone, code.stackmaps()); - auto& offset = Smi::Handle(zone); - StackMap& map = StackMap::Handle(zone); - for (intptr_t i = 0; i < stackmap_table.Length(); i += 2) { - offset ^= stackmap_table.At(i); - map ^= stackmap_table.At(i + 1); - THR_Print("0x%08" Px ": %s\n", offset.Value(), map.ToCString()); - } + if (code.compressed_stackmaps() != CompressedStackMaps::null()) { + const auto& stackmaps = + CompressedStackMaps::Handle(zone, code.compressed_stackmaps()); + THR_Print("%s\n", stackmaps.ToCString()); } THR_Print("}\n"); diff --git a/runtime/vm/compiler/backend/flow_graph_compiler.cc b/runtime/vm/compiler/backend/flow_graph_compiler.cc index 0f5f984ee05e..ecc7b504ba2c 100644 --- a/runtime/vm/compiler/backend/flow_graph_compiler.cc +++ b/runtime/vm/compiler/backend/flow_graph_compiler.cc @@ -119,12 +119,12 @@ FlowGraphCompiler::FlowGraphCompiler( parsed_function_(parsed_function), flow_graph_(*flow_graph), block_order_(*flow_graph->CodegenBlockOrder(is_optimizing)), - current_block_(NULL), - exception_handlers_list_(NULL), - pc_descriptors_list_(NULL), - stackmap_table_builder_(NULL), - code_source_map_builder_(NULL), - catch_entry_moves_maps_builder_(NULL), + current_block_(nullptr), + exception_handlers_list_(nullptr), + pc_descriptors_list_(nullptr), + compressed_stackmaps_builder_(nullptr), + code_source_map_builder_(nullptr), + catch_entry_moves_maps_builder_(nullptr), block_info_(block_order_.length()), deopt_infos_(), static_calls_target_table_(), @@ -932,11 +932,8 @@ void FlowGraphCompiler::RecordSafepoint(LocationSummary* locs, bitmap->Set(bitmap->Length(), true); } - // The slow path area Outside the spill area contains are live registers - // and pushed arguments for calls inside the slow path. - intptr_t slow_path_bit_count = bitmap->Length() - spill_area_size; - stackmap_table_builder()->AddEntry(assembler()->CodeSize(), bitmap, - slow_path_bit_count); + compressed_stackmaps_builder()->AddEntry(assembler()->CodeSize(), bitmap, + spill_area_size); } } @@ -1106,12 +1103,14 @@ RawArray* FlowGraphCompiler::CreateDeoptInfo(compiler::Assembler* assembler) { } void FlowGraphCompiler::FinalizeStackMaps(const Code& code) { - if (stackmap_table_builder_ == NULL) { - code.set_stackmaps(Object::null_array()); + if (compressed_stackmaps_builder_ == NULL) { + code.set_compressed_stackmaps( + CompressedStackMaps::Handle(CompressedStackMaps::null())); } else { - // Finalize the stack map array and add it to the code object. - code.set_stackmaps( - Array::Handle(stackmap_table_builder_->FinalizeStackMaps(code))); + // Finalize the compressed stack maps and add it to the code object. + const auto& maps = + CompressedStackMaps::Handle(compressed_stackmaps_builder_->Finalize()); + code.set_compressed_stackmaps(maps); } } diff --git a/runtime/vm/compiler/backend/flow_graph_compiler.h b/runtime/vm/compiler/backend/flow_graph_compiler.h index 710dc8cd6484..884bef320986 100644 --- a/runtime/vm/compiler/backend/flow_graph_compiler.h +++ b/runtime/vm/compiler/backend/flow_graph_compiler.h @@ -1034,11 +1034,11 @@ class FlowGraphCompiler : public ValueObject { intptr_t GetOptimizationThreshold() const; - StackMapTableBuilder* stackmap_table_builder() { - if (stackmap_table_builder_ == NULL) { - stackmap_table_builder_ = new StackMapTableBuilder(); + CompressedStackMapsBuilder* compressed_stackmaps_builder() { + if (compressed_stackmaps_builder_ == NULL) { + compressed_stackmaps_builder_ = new CompressedStackMapsBuilder(); } - return stackmap_table_builder_; + return compressed_stackmaps_builder_; } // TODO(vegorov) re-enable frame state tracking on DBC. It is @@ -1100,7 +1100,7 @@ class FlowGraphCompiler : public ValueObject { BlockEntryInstr* current_block_; ExceptionHandlerList* exception_handlers_list_; DescriptorList* pc_descriptors_list_; - StackMapTableBuilder* stackmap_table_builder_; + CompressedStackMapsBuilder* compressed_stackmaps_builder_; CodeSourceMapBuilder* code_source_map_builder_; CatchEntryMovesMapBuilder* catch_entry_moves_maps_builder_; GrowableArray block_info_; diff --git a/runtime/vm/image_snapshot.cc b/runtime/vm/image_snapshot.cc index de688a11e939..cd951089200c 100644 --- a/runtime/vm/image_snapshot.cc +++ b/runtime/vm/image_snapshot.cc @@ -185,11 +185,11 @@ int32_t ImageWriter::GetTextOffsetFor(RawInstructions* instructions, } #if defined(IS_SIMARM_X64) -static intptr_t StackMapSizeInSnapshot(intptr_t len_in_bits) { - const intptr_t len_in_bytes = - Utils::RoundUp(len_in_bits, kBitsPerByte) / kBitsPerByte; +static intptr_t CompressedStackMapsSizeInSnapshot(intptr_t payload_size) { + // We do not need to round the non-payload size up to a word boundary because + // currently sizeof(RawCompressedStackMaps) is 12, even on 64-bit. const intptr_t unrounded_size_in_bytes = - 2 * compiler::target::kWordSize + len_in_bytes; + compiler::target::kWordSize + sizeof(uint32_t) + payload_size; return Utils::RoundUp(unrounded_size_in_bytes, compiler::target::ObjectAlignment::kObjectAlignment); } @@ -232,9 +232,10 @@ intptr_t ImageWriter::SizeInSnapshot(RawObject* raw_object) { const classid_t cid = raw_object->GetClassId(); switch (cid) { - case kStackMapCid: { - RawStackMap* raw_map = static_cast(raw_object); - return StackMapSizeInSnapshot(raw_map->ptr()->length_); + case kCompressedStackMapsCid: { + RawCompressedStackMaps* raw_maps = + static_cast(raw_object); + return CompressedStackMapsSizeInSnapshot(raw_maps->ptr()->payload_size_); } case kOneByteStringCid: case kTwoByteStringCid: { @@ -431,23 +432,22 @@ void ImageWriter::WriteROData(WriteStream* stream) { #endif #if defined(IS_SIMARM_X64) - if (obj.IsStackMap()) { - const StackMap& map = StackMap::Cast(obj); + if (obj.IsCompressedStackMaps()) { + const CompressedStackMaps& map = CompressedStackMaps::Cast(obj); // Header layout is the same between 32-bit and 64-bit architecture, but // we need to recalcuate the size in words. - const intptr_t len_in_bits = map.Length(); - const intptr_t len_in_bytes = - Utils::RoundUp(len_in_bits, kBitsPerByte) / kBitsPerByte; - const intptr_t size_in_bytes = StackMapSizeInSnapshot(len_in_bits); + const intptr_t payload_size = map.payload_size(); + const intptr_t size_in_bytes = + CompressedStackMapsSizeInSnapshot(payload_size); marked_tags = RawObject::SizeTag::update(size_in_bytes * 2, marked_tags); stream->WriteTargetWord(marked_tags); - stream->WriteFixed(map.Length()); - stream->WriteFixed(map.SlowPathBitCount()); - stream->WriteBytes(map.raw()->ptr()->data(), len_in_bytes); + stream->WriteFixed(payload_size); + // We do not need to align the stream to a word boundary on 64-bit because + // sizeof(RawCompressedStackMaps) is 12, even there. + stream->WriteBytes(map.raw()->ptr()->data(), payload_size); stream->Align(compiler::target::ObjectAlignment::kObjectAlignment); - } else if (obj.IsString()) { const String& str = String::Cast(obj); RELEASE_ASSERT(String::GetCachedHash(str.raw()) != 0); diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc index 46efcb651c4f..c443b9aabb3f 100644 --- a/runtime/vm/object.cc +++ b/runtime/vm/object.cc @@ -153,7 +153,8 @@ RawClass* Object::object_pool_class_ = reinterpret_cast(RAW_NULL); RawClass* Object::pc_descriptors_class_ = reinterpret_cast(RAW_NULL); RawClass* Object::code_source_map_class_ = reinterpret_cast(RAW_NULL); -RawClass* Object::stackmap_class_ = reinterpret_cast(RAW_NULL); +RawClass* Object::compressed_stackmaps_class_ = + reinterpret_cast(RAW_NULL); RawClass* Object::var_descriptors_class_ = reinterpret_cast(RAW_NULL); RawClass* Object::exception_handlers_class_ = @@ -642,8 +643,8 @@ void Object::Init(Isolate* isolate) { cls = Class::New(isolate); code_source_map_class_ = cls.raw(); - cls = Class::New(isolate); - stackmap_class_ = cls.raw(); + cls = Class::New(isolate); + compressed_stackmaps_class_ = cls.raw(); cls = Class::New(isolate); var_descriptors_class_ = cls.raw(); @@ -1035,7 +1036,7 @@ void Object::Cleanup() { object_pool_class_ = reinterpret_cast(RAW_NULL); pc_descriptors_class_ = reinterpret_cast(RAW_NULL); code_source_map_class_ = reinterpret_cast(RAW_NULL); - stackmap_class_ = reinterpret_cast(RAW_NULL); + compressed_stackmaps_class_ = reinterpret_cast(RAW_NULL); var_descriptors_class_ = reinterpret_cast(RAW_NULL); exception_handlers_class_ = reinterpret_cast(RAW_NULL); context_class_ = reinterpret_cast(RAW_NULL); @@ -1135,7 +1136,7 @@ void Object::FinalizeVMIsolate(Isolate* isolate) { SET_CLASS_NAME(object_pool, ObjectPool); SET_CLASS_NAME(code_source_map, CodeSourceMap); SET_CLASS_NAME(pc_descriptors, PcDescriptors); - SET_CLASS_NAME(stackmap, StackMap); + SET_CLASS_NAME(compressed_stackmaps, CompressedStackMaps); SET_CLASS_NAME(var_descriptors, LocalVarDescriptors); SET_CLASS_NAME(exception_handlers, ExceptionHandlers); SET_CLASS_NAME(context, Context); @@ -1222,12 +1223,12 @@ void Object::FinalizeReadOnlyObject(RawObject* object) { ASSERT(size <= map->HeapSize()); memset(reinterpret_cast(RawObject::ToAddr(map) + size), 0, map->HeapSize() - size); - } else if (cid == kStackMapCid) { - RawStackMap* map = StackMap::RawCast(object); - intptr_t size = StackMap::UnroundedSize(map); - ASSERT(size <= map->HeapSize()); - memset(reinterpret_cast(RawObject::ToAddr(map) + size), 0, - map->HeapSize() - size); + } else if (cid == kCompressedStackMapsCid) { + RawCompressedStackMaps* maps = CompressedStackMaps::RawCast(object); + intptr_t size = CompressedStackMaps::UnroundedSize(maps); + ASSERT(size <= maps->HeapSize()); + memset(reinterpret_cast(RawObject::ToAddr(maps) + size), 0, + maps->HeapSize() - size); } else if (cid == kPcDescriptorsCid) { RawPcDescriptors* desc = PcDescriptors::RawCast(object); intptr_t size = PcDescriptors::UnroundedSize(desc); @@ -4005,8 +4006,8 @@ RawString* Class::GenerateUserVisibleName() const { return Symbols::CodeSourceMap().raw(); case kPcDescriptorsCid: return Symbols::PcDescriptors().raw(); - case kStackMapCid: - return Symbols::StackMap().raw(); + case kCompressedStackMapsCid: + return Symbols::CompressedStackMaps().raw(); case kLocalVarDescriptorsCid: return Symbols::LocalVarDescriptors().raw(); case kExceptionHandlersCid: @@ -12820,84 +12821,70 @@ const char* CodeSourceMap::ToCString() const { return "CodeSourceMap"; } -bool StackMap::GetBit(intptr_t bit_index) const { - ASSERT(InRange(bit_index)); - int byte_index = bit_index >> kBitsPerByteLog2; - int bit_remainder = bit_index & (kBitsPerByte - 1); - uint8_t byte_mask = 1U << bit_remainder; - uint8_t byte = raw_ptr()->data()[byte_index]; - return (byte & byte_mask) != 0; -} - -void StackMap::SetBit(intptr_t bit_index, bool value) const { - ASSERT(InRange(bit_index)); - int byte_index = bit_index >> kBitsPerByteLog2; - int bit_remainder = bit_index & (kBitsPerByte - 1); - uint8_t byte_mask = 1U << bit_remainder; - NoSafepointScope no_safepoint; - uint8_t* byte_addr = UnsafeMutableNonPointer(&raw_ptr()->data()[byte_index]); - if (value) { - *byte_addr |= byte_mask; - } else { - *byte_addr &= ~byte_mask; +intptr_t CompressedStackMaps::Hash() const { + uint32_t hash = 0; + for (intptr_t i = 0; i < payload_size(); i++) { + uint8_t byte = Payload()[i]; + hash = CombineHashes(hash, byte); } + return FinalizeHash(hash, kHashBits); } -RawStackMap* StackMap::New(BitmapBuilder* bmap, intptr_t slow_path_bit_count) { - ASSERT(Object::stackmap_class() != Class::null()); - ASSERT(bmap != NULL); - StackMap& result = StackMap::Handle(); - // Guard against integer overflow of the instance size computation. - intptr_t length = bmap->Length(); - intptr_t payload_size = Utils::RoundUp(length, kBitsPerByte) / kBitsPerByte; - if ((length < 0) || (length > kMaxUint16) || - (payload_size > kMaxLengthInBytes)) { - // This should be caught before we reach here. - FATAL1("Fatal error in StackMap::New: invalid length %" Pd "\n", length); - } - if ((slow_path_bit_count < 0) || (slow_path_bit_count > kMaxUint16)) { - // This should be caught before we reach here. - FATAL1("Fatal error in StackMap::New: invalid slow_path_bit_count %" Pd - "\n", - slow_path_bit_count); +RawCompressedStackMaps* CompressedStackMaps::New( + const GrowableArray& payload) { + ASSERT(Object::compressed_stackmaps_class() != Class::null()); + auto& result = CompressedStackMaps::Handle(); + + const uintptr_t payload_size = payload.length(); + if (payload_size > kMaxInt32) { + FATAL1( + "Fatal error in CompressedStackMaps::New: " + "invalid payload size %" Pu "\n", + payload_size); } { - // StackMap data objects are associated with a code object, allocate them - // in old generation. + // CompressedStackMaps data objects are associated with a code object, + // allocate them in old generation. RawObject* raw = Object::Allocate( - StackMap::kClassId, StackMap::InstanceSize(length), Heap::kOld); + CompressedStackMaps::kClassId, + CompressedStackMaps::InstanceSize(payload_size), Heap::kOld); NoSafepointScope no_safepoint; result ^= raw; - result.SetLength(length); - } - if (payload_size > 0) { - // Ensure leftover bits are deterministic. - result.raw()->ptr()->data()[payload_size - 1] = 0; - } - for (intptr_t i = 0; i < length; ++i) { - result.SetBit(i, bmap->Get(i)); + result.set_payload_size(payload_size); } - result.SetSlowPathBitCount(slow_path_bit_count); + result.SetPayload(payload); + return result.raw(); } -const char* StackMap::ToCString() const { - if (IsNull()) return "{null}"; - // Guard against integer overflow in the computation of alloc_size. - // - // TODO(kmillikin): We could just truncate the string if someone - // tries to print a 2 billion plus entry stackmap. - if (Length() > kIntptrMax) { - FATAL1("Length() is unexpectedly large (%" Pd ")", Length()); +void CompressedStackMaps::SetPayload( + const GrowableArray& payload) const { + auto const array_length = payload.length(); + ASSERT(array_length <= payload_size()); + + NoSafepointScope no_safepoint; + uint8_t* payload_start = UnsafeMutableNonPointer(raw_ptr()->data()); + for (intptr_t i = 0; i < array_length; i++) { + payload_start[i] = payload.At(i); } - Thread* thread = Thread::Current(); - intptr_t alloc_size = Length() + 1; - char* chars = thread->zone()->Alloc(alloc_size); - for (intptr_t i = 0; i < Length(); i++) { - chars[i] = IsObject(i) ? '1' : '0'; +} + +const char* CompressedStackMaps::ToCString() const { + ZoneTextBuffer b(Thread::Current()->zone(), 100); + CompressedStackMapsIterator it(*this); + bool first_entry = true; + while (it.MoveNext()) { + if (first_entry) { + first_entry = false; + } else { + b.AddString("\n"); + } + b.Printf("0x%" Pp ": ", it.pc_offset()); + for (intptr_t i = 0, n = it.length(); i < n; i++) { + b.AddString(it.IsObject(i) ? "1" : "0"); + } } - chars[alloc_size - 1] = '\0'; - return chars; + return b.buffer(); } RawString* LocalVarDescriptors::GetName(intptr_t var_index) const { @@ -14461,9 +14448,9 @@ void Code::set_is_alive(bool value) const { set_state_bits(AliveBit::update(value, raw_ptr()->state_bits_)); } -void Code::set_stackmaps(const Array& maps) const { +void Code::set_compressed_stackmaps(const CompressedStackMaps& maps) const { ASSERT(maps.IsOld()); - StorePointer(&raw_ptr()->stackmaps_, maps.raw()); + StorePointer(&raw_ptr()->compressed_stackmaps_, maps.raw()); } #if !defined(DART_PRECOMPILED_RUNTIME) && !defined(DART_PRECOMPILER) @@ -15130,39 +15117,6 @@ void Code::SetActiveInstructions(const Instructions& instructions) const { #endif } -RawStackMap* Code::GetStackMap(uint32_t pc_offset, - Array* maps, - StackMap* map) const { - // This code is used during iterating frames during a GC and hence it - // should not in turn start a GC. - NoSafepointScope no_safepoint; - if (stackmaps() == Array::null()) { - // No stack maps are present in the code object which means this - // frame relies on tagged pointers. - return StackMap::null(); - } - // A stack map is present in the code object, use the stack map to visit - // frame slots which are marked as having objects. - *maps = stackmaps(); - *map = StackMap::null(); - for (intptr_t i = 0; i < maps->Length(); i += 2) { - // The reinterpret_cast from Smi::RawCast is inlined here because in - // debug mode, it creates Handles due to the ASSERT. - const uint32_t offset = - ValueFromRawSmi(reinterpret_cast(maps->At(i))); - if (offset != pc_offset) continue; - *map ^= maps->At(i + 1); - ASSERT(!map->IsNull()); - return map->raw(); - } - // If we are missing a stack map, this must either be unoptimized code, or - // the entry to an osr function. (In which case all stack slots are - // considered to have tagged pointers.) - // Running with --verify-on-transition should hit this. - ASSERT(!is_optimized() || (pc_offset == EntryPoint() - PayloadStart())); - return StackMap::null(); -} - void Code::GetInlinedFunctionsAtInstruction( intptr_t pc_offset, GrowableArray* functions, diff --git a/runtime/vm/object.h b/runtime/vm/object.h index 2afaacd842da..c64ba148866b 100644 --- a/runtime/vm/object.h +++ b/runtime/vm/object.h @@ -470,7 +470,9 @@ class Object { static RawClass* object_pool_class() { return object_pool_class_; } static RawClass* pc_descriptors_class() { return pc_descriptors_class_; } static RawClass* code_source_map_class() { return code_source_map_class_; } - static RawClass* stackmap_class() { return stackmap_class_; } + static RawClass* compressed_stackmaps_class() { + return compressed_stackmaps_class_; + } static RawClass* var_descriptors_class() { return var_descriptors_class_; } static RawClass* exception_handlers_class() { return exception_handlers_class_; @@ -739,7 +741,8 @@ class Object { static RawClass* object_pool_class_; // Class of the ObjectPool vm object. static RawClass* pc_descriptors_class_; // Class of PcDescriptors vm object. static RawClass* code_source_map_class_; // Class of CodeSourceMap vm object. - static RawClass* stackmap_class_; // Class of StackMap vm object. + static RawClass* + compressed_stackmaps_class_; // Class of CompressedStackMaps. static RawClass* var_descriptors_class_; // Class of LocalVarDescriptors. static RawClass* exception_handlers_class_; // Class of ExceptionHandlers. static RawClass* deopt_info_class_; // Class of DeoptInfo. @@ -5143,62 +5146,52 @@ class CodeSourceMap : public Object { friend class Object; }; -class StackMap : public Object { +class CompressedStackMaps : public Object { public: - bool IsObject(intptr_t index) const { - ASSERT(InRange(index)); - return GetBit(index); - } + static const intptr_t kHashBits = 30; - intptr_t Length() const { return raw_ptr()->length_; } + intptr_t payload_size() const { return raw_ptr()->payload_size_; } - intptr_t SlowPathBitCount() const { return raw_ptr()->slow_path_bit_count_; } - void SetSlowPathBitCount(intptr_t bit_count) const { - ASSERT(bit_count <= kMaxUint16); - StoreNonPointer(&raw_ptr()->slow_path_bit_count_, bit_count); - } - - bool Equals(const StackMap& other) const { - if (Length() != other.Length()) { - return false; - } + bool Equals(const CompressedStackMaps& other) const { + if (payload_size() != other.payload_size()) return false; NoSafepointScope no_safepoint; - return memcmp(raw_ptr(), other.raw_ptr(), InstanceSize(Length())) == 0; + return memcmp(raw_ptr(), other.raw_ptr(), InstanceSize(payload_size())) == + 0; } + intptr_t Hash() const; - static const intptr_t kMaxLengthInBytes = kSmiMax; - - static intptr_t UnroundedSize(RawStackMap* map) { - return UnroundedSize(map->ptr()->length_); + static intptr_t UnroundedSize(RawCompressedStackMaps* maps) { + return UnroundedSize(maps->ptr()->payload_size_); } - static intptr_t UnroundedSize(intptr_t len) { - // The stackmap payload is in an array of bytes. - intptr_t payload_size = Utils::RoundUp(len, kBitsPerByte) / kBitsPerByte; - return sizeof(RawStackMap) + payload_size; + static intptr_t UnroundedSize(intptr_t length) { + return sizeof(RawCompressedStackMaps) + length; } static intptr_t InstanceSize() { - ASSERT(sizeof(RawStackMap) == OFFSET_OF_RETURNED_VALUE(RawStackMap, data)); + ASSERT(sizeof(RawCompressedStackMaps) == + OFFSET_OF_RETURNED_VALUE(RawCompressedStackMaps, data)); return 0; } static intptr_t InstanceSize(intptr_t length) { return RoundedAllocationSize(UnroundedSize(length)); } - static RawStackMap* New(BitmapBuilder* bmap, intptr_t slow_path_bit_count); private: - void SetLength(intptr_t length) const { - ASSERT(length <= kMaxUint16); - StoreNonPointer(&raw_ptr()->length_, length); - } + // The encoding logic for CompressedStackMaps entries is in + // CompressedStackMapsBuilder, and the decoding logic is in + // CompressedStackMapsIterator. + static RawCompressedStackMaps* New(const GrowableArray& bytes); - bool InRange(intptr_t index) const { return index < Length(); } + void set_payload_size(intptr_t payload_size) const { + StoreNonPointer(&raw_ptr()->payload_size_, payload_size); + } - bool GetBit(intptr_t bit_index) const; - void SetBit(intptr_t bit_index, bool value) const; + const uint8_t* Payload() const { return raw_ptr()->data(); } + void SetPayload(const GrowableArray& payload) const; - FINAL_HEAP_OBJECT_IMPLEMENTATION(StackMap, Object); - friend class BitmapBuilder; + FINAL_HEAP_OBJECT_IMPLEMENTATION(CompressedStackMaps, Object); friend class Class; + friend class CompressedStackMapsBuilder; + friend class CompressedStackMapsIterator; }; class ExceptionHandlers : public Object { @@ -5401,11 +5394,11 @@ class Code : public Object { void set_catch_entry_moves_maps(const TypedData& maps) const; #endif - RawArray* stackmaps() const { return raw_ptr()->stackmaps_; } - void set_stackmaps(const Array& maps) const; - RawStackMap* GetStackMap(uint32_t pc_offset, - Array* stackmaps, - StackMap* map) const; + RawCompressedStackMaps* compressed_stackmaps() const { + return raw_ptr()->compressed_stackmaps_; + } + void set_compressed_stackmaps(const CompressedStackMaps& maps) const; + enum CallKind { kPcRelativeCall = 1, kPcRelativeTailCall = 2, diff --git a/runtime/vm/object_service.cc b/runtime/vm/object_service.cc index 1c1289449f13..02c90ac9dc36 100644 --- a/runtime/vm/object_service.cc +++ b/runtime/vm/object_service.cc @@ -742,7 +742,7 @@ void CodeSourceMap::PrintJSONImpl(JSONStream* stream, bool ref) const { Object::PrintJSONImpl(stream, ref); } -void StackMap::PrintJSONImpl(JSONStream* stream, bool ref) const { +void CompressedStackMaps::PrintJSONImpl(JSONStream* stream, bool ref) const { Object::PrintJSONImpl(stream, ref); } diff --git a/runtime/vm/program_visitor.cc b/runtime/vm/program_visitor.cc index 0763f16ec053..1b827d74fede 100644 --- a/runtime/vm/program_visitor.cc +++ b/runtime/vm/program_visitor.cc @@ -201,42 +201,39 @@ void ProgramVisitor::ShareMegamorphicBuckets() { } } -class StackMapKeyValueTrait { +class CompressedStackMapsKeyValueTrait { public: // Typedefs needed for the DirectChainedHashMap template. - typedef const StackMap* Key; - typedef const StackMap* Value; - typedef const StackMap* Pair; + typedef const CompressedStackMaps* Key; + typedef const CompressedStackMaps* Value; + typedef const CompressedStackMaps* Pair; static Key KeyOf(Pair kv) { return kv; } static Value ValueOf(Pair kv) { return kv; } - static inline intptr_t Hashcode(Key key) { - intptr_t hash = key->SlowPathBitCount(); - hash = CombineHashes(hash, key->Length()); - return FinalizeHash(hash, kBitsPerWord - 1); - } + static inline intptr_t Hashcode(Key key) { return key->Hash(); } static inline bool IsKeyEqual(Pair pair, Key key) { return pair->Equals(*key); } }; -typedef DirectChainedHashMap StackMapSet; +typedef DirectChainedHashMap + CompressedStackMapsSet; -void ProgramVisitor::DedupStackMaps() { - class DedupStackMapsVisitor : public FunctionVisitor { +void ProgramVisitor::DedupCompressedStackMaps() { + class DedupCompressedStackMapsVisitor : public FunctionVisitor { public: - explicit DedupStackMapsVisitor(Zone* zone) + explicit DedupCompressedStackMapsVisitor(Zone* zone) : zone_(zone), - canonical_stackmaps_(), + canonical_compressed_stackmaps_set_(), code_(Code::Handle(zone)), - stackmaps_(Array::Handle(zone)), - stackmap_(StackMap::Handle(zone)) {} + compressed_stackmaps_(CompressedStackMaps::Handle(zone)) {} - void AddStackMap(const StackMap& stackmap) { - canonical_stackmaps_.Insert(&StackMap::ZoneHandle(zone_, stackmap.raw())); + void AddCompressedStackMaps(const CompressedStackMaps& maps) { + canonical_compressed_stackmaps_set_.Insert( + &CompressedStackMaps::ZoneHandle(zone_, maps.raw())); } void Visit(const Function& function) { @@ -244,43 +241,40 @@ void ProgramVisitor::DedupStackMaps() { return; } code_ = function.CurrentCode(); - stackmaps_ = code_.stackmaps(); - if (stackmaps_.IsNull()) return; - for (intptr_t i = 1; i < stackmaps_.Length(); i += 2) { - stackmap_ ^= stackmaps_.At(i); - stackmap_ = DedupStackMap(stackmap_); - stackmaps_.SetAt(i, stackmap_); - } - } - - RawStackMap* DedupStackMap(const StackMap& stackmap) { - const StackMap* canonical_stackmap = - canonical_stackmaps_.LookupValue(&stackmap); - if (canonical_stackmap == NULL) { - AddStackMap(stackmap); - return stackmap.raw(); + compressed_stackmaps_ = code_.compressed_stackmaps(); + if (compressed_stackmaps_.IsNull()) return; + compressed_stackmaps_ = DedupCompressedStackMaps(compressed_stackmaps_); + code_.set_compressed_stackmaps(compressed_stackmaps_); + } + + RawCompressedStackMaps* DedupCompressedStackMaps( + const CompressedStackMaps& maps) { + auto const canonical_maps = + canonical_compressed_stackmaps_set_.LookupValue(&maps); + if (canonical_maps == nullptr) { + AddCompressedStackMaps(maps); + return maps.raw(); } else { - return canonical_stackmap->raw(); + return canonical_maps->raw(); } } private: Zone* zone_; - StackMapSet canonical_stackmaps_; + CompressedStackMapsSet canonical_compressed_stackmaps_set_; Code& code_; - Array& stackmaps_; - StackMap& stackmap_; + CompressedStackMaps& compressed_stackmaps_; }; - DedupStackMapsVisitor visitor(Thread::Current()->zone()); + DedupCompressedStackMapsVisitor visitor(Thread::Current()->zone()); if (Snapshot::IncludesCode(Dart::vm_snapshot_kind())) { // Prefer existing objects in the VM isolate. const Array& object_table = Object::vm_isolate_snapshot_object_table(); Object& object = Object::Handle(); for (intptr_t i = 0; i < object_table.Length(); i++) { object = object_table.At(i); - if (object.IsStackMap()) { - visitor.AddStackMap(StackMap::Cast(object)); + if (object.IsCompressedStackMaps()) { + visitor.AddCompressedStackMaps(CompressedStackMaps::Cast(object)); } } } @@ -620,11 +614,6 @@ void ProgramVisitor::DedupLists() { void Visit(const Function& function) { code_ = function.CurrentCode(); if (!code_.IsNull()) { - list_ = code_.stackmaps(); - if (!list_.IsNull()) { - list_ = DedupList(list_); - code_.set_stackmaps(list_); - } list_ = code_.inlined_id_to_function(); if (!list_.IsNull()) { list_ = DedupList(list_); @@ -768,7 +757,7 @@ class CodeKeyValueTrait { if (pair->pc_descriptors() != key->pc_descriptors()) { return false; } - if (pair->stackmaps() != key->stackmaps()) { + if (pair->compressed_stackmaps() != key->compressed_stackmaps()) { return false; } if (pair->catch_entry_moves_maps() != key->catch_entry_moves_maps()) { @@ -906,7 +895,7 @@ void ProgramVisitor::Dedup() { BindStaticCalls(); ShareMegamorphicBuckets(); - DedupStackMaps(); + DedupCompressedStackMaps(); DedupPcDescriptors(); NOT_IN_PRECOMPILED(DedupDeoptEntries()); #if defined(DART_PRECOMPILER) diff --git a/runtime/vm/program_visitor.h b/runtime/vm/program_visitor.h index 3ba35396341d..d18b60dcf56e 100644 --- a/runtime/vm/program_visitor.h +++ b/runtime/vm/program_visitor.h @@ -33,7 +33,7 @@ class ProgramVisitor : public AllStatic { #if !defined(DART_PRECOMPILED_RUNTIME) static void BindStaticCalls(); static void ShareMegamorphicBuckets(); - static void DedupStackMaps(); + static void DedupCompressedStackMaps(); static void DedupPcDescriptors(); static void DedupDeoptEntries(); #if defined(DART_PRECOMPILER) diff --git a/runtime/vm/raw_object.cc b/runtime/vm/raw_object.cc index 69dd302edf7c..624b55b359d0 100644 --- a/runtime/vm/raw_object.cc +++ b/runtime/vm/raw_object.cc @@ -176,10 +176,11 @@ intptr_t RawObject::HeapSizeFromClass() const { instance_size = CodeSourceMap::InstanceSize(length); break; } - case kStackMapCid: { - const RawStackMap* map = reinterpret_cast(this); - intptr_t length = map->ptr()->length_; - instance_size = StackMap::InstanceSize(length); + case kCompressedStackMapsCid: { + const RawCompressedStackMaps* maps = + reinterpret_cast(this); + intptr_t length = maps->ptr()->payload_size_; + instance_size = CompressedStackMaps::InstanceSize(length); break; } case kLocalVarDescriptorsCid: { @@ -531,7 +532,7 @@ NULL_VISITOR(DynamicLibrary) VARIABLE_NULL_VISITOR(Instructions, Instructions::Size(raw_obj)) VARIABLE_NULL_VISITOR(PcDescriptors, raw_obj->ptr()->length_) VARIABLE_NULL_VISITOR(CodeSourceMap, raw_obj->ptr()->length_) -VARIABLE_NULL_VISITOR(StackMap, raw_obj->ptr()->length_) +VARIABLE_NULL_VISITOR(CompressedStackMaps, raw_obj->ptr()->payload_size_) VARIABLE_NULL_VISITOR(OneByteString, Smi::Value(raw_obj->ptr()->length_)) VARIABLE_NULL_VISITOR(TwoByteString, Smi::Value(raw_obj->ptr()->length_)) // Abstract types don't have their visitor called. diff --git a/runtime/vm/raw_object.h b/runtime/vm/raw_object.h index 99e2eaf1a653..c6243307cd0b 100644 --- a/runtime/vm/raw_object.h +++ b/runtime/vm/raw_object.h @@ -1371,9 +1371,7 @@ class RawCode : public RawObject { RawTypedData* catch_entry_moves_maps_; RawSmi* variables_; } catch_entry_; - // The stackmaps_ array contains alternating Smi and StackMap values, where - // each Smi value is the PC offset for the following StackMap value. - RawArray* stackmaps_; + RawCompressedStackMaps* compressed_stackmaps_; RawArray* inlined_id_to_function_; RawCodeSourceMap* code_source_map_; NOT_IN_PRECOMPILED(RawInstructions* active_instructions_); @@ -1616,21 +1614,35 @@ class RawCodeSourceMap : public RawObject { friend class ImageWriter; }; -// StackMap is an immutable representation of the layout of the stack at a -// PC. The stack map representation consists of a bit map which marks each -// live object index starting from the base of the frame. -// -// The bit map representation is optimized for dense and small bit maps, without -// any upper bound. -class RawStackMap : public RawObject { - RAW_HEAP_OBJECT_IMPLEMENTATION(StackMap); +// RawCompressedStackMaps is a compressed representation of the stack maps +// for certain PC offsets into a set of instructions, where a stack map is a bit +// map that marks each live object index starting from the base of the frame. +class RawCompressedStackMaps : public RawObject { + RAW_HEAP_OBJECT_IMPLEMENTATION(CompressedStackMaps); VISIT_NOTHING(); - uint16_t length_; // Length of payload, in bits. - uint16_t slow_path_bit_count_; // Slow path live values, included in length_. - // ARM64 requires register_bit_count_ to be as large as 96. + uint32_t payload_size_; // Length of the encoded payload, in bytes. - // Variable length data follows here (bitmap of the stack layout). + // Variable length data follows here. The payload consists of entries with + // the following information: + // + // * A header containing the following three pieces of information: + // * An unsigned integer representing the PC offset as a delta from the + // PC offset of the previous entry (from 0 for the first entry). + // * An unsigned integer representing the number of bits used for + // register entries. + // * An unsigned integer representing the number of bits used for spill + // slot entries. + // * The body containing the bits for the stack map. The length of the body + // in bits is the sum of the register and spill slot bit counts. + // + // Each unsigned integer is LEB128 encoded, as generally they tend to fit in + // a single byte or two. Thus, entry headers are not a fixed length, and + // currently there is no random access of entries. In addition, PC offsets + // are currently encoded as deltas, which also inhibits random access without + // accessing previous entries. That means to find an entry for a given PC + // offset, a linear search must be done where the payload is decoded + // up to the entry whose PC offset is >= the given PC. uint8_t* data() { OPEN_ARRAY_START(uint8_t, uint8_t); } const uint8_t* data() const { OPEN_ARRAY_START(uint8_t, uint8_t); } @@ -2792,7 +2804,7 @@ inline bool RawObject::IsVariableSizeClassId(intptr_t index) { RawObject::IsTypedDataClassId(index) || (index == kContextCid) || (index == kTypeArgumentsCid) || (index == kInstructionsCid) || (index == kObjectPoolCid) || (index == kPcDescriptorsCid) || - (index == kCodeSourceMapCid) || (index == kStackMapCid) || + (index == kCodeSourceMapCid) || (index == kCompressedStackMapsCid) || (index == kLocalVarDescriptorsCid) || (index == kExceptionHandlersCid) || (index == kCodeCid) || (index == kContextScopeCid) || (index == kInstanceCid) || diff --git a/runtime/vm/raw_object_fields.cc b/runtime/vm/raw_object_fields.cc index 3dc239b69eca..c818091a5876 100644 --- a/runtime/vm/raw_object_fields.cc +++ b/runtime/vm/raw_object_fields.cc @@ -99,7 +99,7 @@ namespace dart { F(Code, owner_) \ F(Code, exception_handlers_) \ F(Code, pc_descriptors_) \ - F(Code, stackmaps_) \ + F(Code, compressed_stackmaps_) \ F(Code, inlined_id_to_function_) \ F(Code, code_source_map_) \ F(Bytecode, object_pool_) \ diff --git a/runtime/vm/raw_object_snapshot.cc b/runtime/vm/raw_object_snapshot.cc index 67731410f955..56e29f1b4c7a 100644 --- a/runtime/vm/raw_object_snapshot.cc +++ b/runtime/vm/raw_object_snapshot.cc @@ -516,6 +516,7 @@ MESSAGE_SNAPSHOT_UNREACHABLE(Bytecode); MESSAGE_SNAPSHOT_UNREACHABLE(ClosureData); MESSAGE_SNAPSHOT_UNREACHABLE(Code); MESSAGE_SNAPSHOT_UNREACHABLE(CodeSourceMap); +MESSAGE_SNAPSHOT_UNREACHABLE(CompressedStackMaps); MESSAGE_SNAPSHOT_UNREACHABLE(Error); MESSAGE_SNAPSHOT_UNREACHABLE(ExceptionHandlers); MESSAGE_SNAPSHOT_UNREACHABLE(FfiTrampolineData); @@ -537,7 +538,6 @@ MESSAGE_SNAPSHOT_UNREACHABLE(RedirectionData); MESSAGE_SNAPSHOT_UNREACHABLE(Script); MESSAGE_SNAPSHOT_UNREACHABLE(SignatureData); MESSAGE_SNAPSHOT_UNREACHABLE(SingleTargetCache); -MESSAGE_SNAPSHOT_UNREACHABLE(StackMap); MESSAGE_SNAPSHOT_UNREACHABLE(String); MESSAGE_SNAPSHOT_UNREACHABLE(SubtypeTestCache); MESSAGE_SNAPSHOT_UNREACHABLE(TypedDataBase); diff --git a/runtime/vm/snapshot.h b/runtime/vm/snapshot.h index 1f7beaa843fd..61b90a945751 100644 --- a/runtime/vm/snapshot.h +++ b/runtime/vm/snapshot.h @@ -50,6 +50,7 @@ class RawClass; class RawClosure; class RawClosureData; class RawCodeSourceMap; +class RawCompressedStackMaps; class RawContext; class RawContextScope; class RawDouble; @@ -85,7 +86,6 @@ class RawScript; class RawSignatureData; class RawSendPort; class RawSmi; -class RawStackMap; class RawStackTrace; class RawSubtypeTestCache; class RawTwoByteString; diff --git a/runtime/vm/stack_frame.cc b/runtime/vm/stack_frame.cc index 2bdde950c98f..227dbbeb60cc 100644 --- a/runtime/vm/stack_frame.cc +++ b/runtime/vm/stack_frame.cc @@ -287,12 +287,11 @@ void StackFrame::VisitObjectPointers(ObjectPointerVisitor* visitor) { if (!code.IsNull()) { // Optimized frames have a stack map. We need to visit the frame based // on the stack map. - Array maps; - maps = Array::null(); - StackMap map; + CompressedStackMaps maps; + maps = code.compressed_stackmaps(); + CompressedStackMapsIterator it(maps); const uword start = Instructions::PayloadStart(code.instructions()); - map = code.GetStackMap(pc() - start, &maps, &map); - if (!map.IsNull()) { + if (it.Find(pc() - start)) { #if !defined(TARGET_ARCH_DBC) if (is_interpreted()) { UNIMPLEMENTED(); @@ -311,11 +310,11 @@ void StackFrame::VisitObjectPointers(ObjectPointerVisitor* visitor) { // The spill slots and any saved registers are described in the stack // map. The outgoing arguments are assumed to be tagged; the number // of outgoing arguments is not explicitly tracked. - intptr_t length = map.Length(); + // Spill slots are at the 'bottom' of the frame. - intptr_t spill_slot_count = length - map.SlowPathBitCount(); + intptr_t spill_slot_count = it.spill_slot_bit_count(); for (intptr_t bit = 0; bit < spill_slot_count; ++bit) { - if (map.IsObject(bit)) { + if (it.IsObject(bit)) { visitor->VisitPointer(last); } --last; @@ -323,8 +322,8 @@ void StackFrame::VisitObjectPointers(ObjectPointerVisitor* visitor) { // The live registers at the 'top' of the frame comprise the rest of the // stack map. - for (intptr_t bit = length - 1; bit >= spill_slot_count; --bit) { - if (map.IsObject(bit)) { + for (intptr_t bit = it.length() - 1; bit >= spill_slot_count; --bit) { + if (it.IsObject(bit)) { visitor->VisitPointer(first); } ++first; @@ -363,12 +362,11 @@ void StackFrame::VisitObjectPointers(ObjectPointerVisitor* visitor) { // The DBC registers are described in the stack map. // The outgoing arguments are assumed to be tagged; the number // of outgoing arguments is not explicitly tracked. - ASSERT(map.SlowPathBitCount() == 0); // Visit DBC registers that contain tagged values. - intptr_t length = map.Length(); + intptr_t length = it.length(); for (intptr_t bit = 0; bit < length; ++bit) { - if (map.IsObject(bit)) { + if (it.IsObject(bit)) { visitor->VisitPointer(first + bit); } } diff --git a/runtime/vm/symbols.h b/runtime/vm/symbols.h index c95b589f2593..cdbd8095e1fc 100644 --- a/runtime/vm/symbols.h +++ b/runtime/vm/symbols.h @@ -57,6 +57,7 @@ class ObjectPointerVisitor; V(CompleterFuture, "future") \ V(CompleterGetFuture, "get:future") \ V(CompleterSyncConstructor, "Completer.sync") \ + V(CompressedStackMaps, "CompressedStackMaps") \ V(ConstructorStacktracePrefix, "new ") \ V(Context, "Context") \ V(ContextScope, "ContextScope") \ @@ -245,7 +246,6 @@ class ObjectPointerVisitor; V(SpaceIsFromSpace, " is from ") \ V(SpaceOfSpace, " of ") \ V(SpaceWhereNewLine, " where\n") \ - V(StackMap, "StackMap") \ V(StackOverflowError, "StackOverflowError") \ V(StackTraceParameter, ":stack_trace") \ V(StackTraceVar, ":stack_trace_var") \