diff --git a/src/hotspot/share/gc/g1/g1CardTable.hpp b/src/hotspot/share/gc/g1/g1CardTable.hpp index aa3b7946618c3..3540bb9411a81 100644 --- a/src/hotspot/share/gc/g1/g1CardTable.hpp +++ b/src/hotspot/share/gc/g1/g1CardTable.hpp @@ -84,6 +84,7 @@ class G1CardTable : public CardTable { } static CardValue g1_young_card_val() { return g1_young_gen; } + static CardValue g1_scanned_card_val() { return g1_card_already_scanned; } void verify_g1_young_region(MemRegion mr) PRODUCT_RETURN; void g1_mark_as_young(const MemRegion& mr); @@ -103,8 +104,8 @@ class G1CardTable : public CardTable { // be inaccurate as it does not perform the dirtying atomically. inline size_t mark_region_dirty(size_t start_card_index, size_t num_cards); - // Mark the given range of cards as Scanned. All of these cards must be Dirty. - inline void mark_as_scanned(size_t start_card_index, size_t num_cards); + // Change the given range of dirty cards to "which". All of these cards must be Dirty. + inline void change_dirty_cards_to(size_t start_card_index, size_t num_cards, CardValue which); inline uint region_idx_for(CardValue* p); diff --git a/src/hotspot/share/gc/g1/g1CardTable.inline.hpp b/src/hotspot/share/gc/g1/g1CardTable.inline.hpp index 9e95cfc3630cd..ae9637a0d1456 100644 --- a/src/hotspot/share/gc/g1/g1CardTable.inline.hpp +++ b/src/hotspot/share/gc/g1/g1CardTable.inline.hpp @@ -77,14 +77,14 @@ inline size_t G1CardTable::mark_region_dirty(size_t start_card_index, size_t num return result; } -inline void G1CardTable::mark_as_scanned(size_t start_card_index, size_t num_cards) { +inline void G1CardTable::change_dirty_cards_to(size_t start_card_index, size_t num_cards, CardValue which) { CardValue* start = &_byte_map[start_card_index]; CardValue* const end = start + num_cards; while (start < end) { CardValue value = *start; assert(value == dirty_card_val(), "Must have been dirty %d start " PTR_FORMAT " " PTR_FORMAT, value, p2i(start), p2i(end)); - *start++ = g1_card_already_scanned; + *start++ = which; } } diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp index 14c534527886f..d55d10a25ebf5 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp @@ -3045,10 +3045,11 @@ void G1CollectedHeap::do_collection_pause_at_safepoint_helper(double target_paus collection_set()->optional_region_length()); pre_evacuate_collection_set(evacuation_info, &per_thread_states); + bool may_do_optional_evacuation = _collection_set.optional_region_length() != 0; // Actually do the work... - evacuate_initial_collection_set(&per_thread_states); + evacuate_initial_collection_set(&per_thread_states, may_do_optional_evacuation); - if (_collection_set.optional_region_length() != 0) { + if (may_do_optional_evacuation) { evacuate_optional_collection_set(&per_thread_states); } post_evacuate_collection_set(evacuation_info, &rdcqs, &per_thread_states); @@ -3814,10 +3815,11 @@ class G1EvacuateRegionsBaseTask : public AbstractGangTask { class G1EvacuateRegionsTask : public G1EvacuateRegionsBaseTask { G1RootProcessor* _root_processor; + bool _has_optional_evacuation_work; void scan_roots(G1ParScanThreadState* pss, uint worker_id) { _root_processor->evacuate_roots(pss, worker_id); - _g1h->rem_set()->scan_heap_roots(pss, worker_id, G1GCPhaseTimes::ScanHR, G1GCPhaseTimes::ObjCopy); + _g1h->rem_set()->scan_heap_roots(pss, worker_id, G1GCPhaseTimes::ScanHR, G1GCPhaseTimes::ObjCopy, _has_optional_evacuation_work); _g1h->rem_set()->scan_collection_set_regions(pss, worker_id, G1GCPhaseTimes::ScanHR, G1GCPhaseTimes::CodeRoots, G1GCPhaseTimes::ObjCopy); } @@ -3838,13 +3840,16 @@ class G1EvacuateRegionsTask : public G1EvacuateRegionsBaseTask { G1ParScanThreadStateSet* per_thread_states, G1ScannerTasksQueueSet* task_queues, G1RootProcessor* root_processor, - uint num_workers) : + uint num_workers, + bool has_optional_evacuation_work) : G1EvacuateRegionsBaseTask("G1 Evacuate Regions", per_thread_states, task_queues, num_workers), - _root_processor(root_processor) + _root_processor(root_processor), + _has_optional_evacuation_work(has_optional_evacuation_work) { } }; -void G1CollectedHeap::evacuate_initial_collection_set(G1ParScanThreadStateSet* per_thread_states) { +void G1CollectedHeap::evacuate_initial_collection_set(G1ParScanThreadStateSet* per_thread_states, + bool has_optional_evacuation_work) { G1GCPhaseTimes* p = phase_times(); { @@ -3859,7 +3864,12 @@ void G1CollectedHeap::evacuate_initial_collection_set(G1ParScanThreadStateSet* p Ticks start_processing = Ticks::now(); { G1RootProcessor root_processor(this, num_workers); - G1EvacuateRegionsTask g1_par_task(this, per_thread_states, _task_queues, &root_processor, num_workers); + G1EvacuateRegionsTask g1_par_task(this, + per_thread_states, + _task_queues, + &root_processor, + num_workers, + has_optional_evacuation_work); task_time = run_task_timed(&g1_par_task); // Closing the inner scope will execute the destructor for the G1RootProcessor object. // To extract its code root fixup time we measure total time of this scope and @@ -3869,12 +3879,14 @@ void G1CollectedHeap::evacuate_initial_collection_set(G1ParScanThreadStateSet* p p->record_initial_evac_time(task_time.seconds() * 1000.0); p->record_or_add_code_root_fixup_time((total_processing - task_time).seconds() * 1000.0); + + rem_set()->complete_evac_phase(has_optional_evacuation_work); } class G1EvacuateOptionalRegionsTask : public G1EvacuateRegionsBaseTask { void scan_roots(G1ParScanThreadState* pss, uint worker_id) { - _g1h->rem_set()->scan_heap_roots(pss, worker_id, G1GCPhaseTimes::OptScanHR, G1GCPhaseTimes::OptObjCopy); + _g1h->rem_set()->scan_heap_roots(pss, worker_id, G1GCPhaseTimes::OptScanHR, G1GCPhaseTimes::OptObjCopy, true /* remember_already_scanned_cards */); _g1h->rem_set()->scan_collection_set_regions(pss, worker_id, G1GCPhaseTimes::OptScanHR, G1GCPhaseTimes::OptCodeRoots, G1GCPhaseTimes::OptObjCopy); } @@ -3934,6 +3946,8 @@ void G1CollectedHeap::evacuate_optional_collection_set(G1ParScanThreadStateSet* evacuate_next_optional_regions(per_thread_states); phase_times()->record_or_add_optional_evac_time((Ticks::now() - start).seconds() * 1000.0); } + + rem_set()->complete_evac_phase(true /* has_more_than_one_evacuation_phase */); } _collection_set.abandon_optional_collection_set(per_thread_states); diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp index 94b1b2b97189c..4c98d5f38d9e1 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp @@ -785,7 +785,23 @@ class G1CollectedHeap : public CollectedHeap { void calculate_collection_set(G1EvacuationInfo& evacuation_info, double target_pause_time_ms); // Actually do the work of evacuating the parts of the collection set. - void evacuate_initial_collection_set(G1ParScanThreadStateSet* per_thread_states); + // The has_optional_evacuation_work flag for the initial collection set + // evacuation indicates whether one or more optional evacuation steps may + // follow. + // If not set, G1 can avoid clearing the card tables of regions that we scan + // for roots from the heap: when scanning the card table for dirty cards after + // all remembered sets have been dumped onto it, for optional evacuation we + // mark these cards as "Scanned" to know that we do not need to re-scan them + // in the additional optional evacuation passes. This means that in the "Clear + // Card Table" phase we need to clear those marks. However, if there is no + // optional evacuation, g1 can immediately clean the dirty cards it encounters + // as nobody else will be looking at them again, saving the clear card table + // work later. + // This case is very common (young only collections and most mixed gcs), so + // depending on the ratio between scanned and evacuated regions (which g1 always + // needs to clear), this is a big win. + void evacuate_initial_collection_set(G1ParScanThreadStateSet* per_thread_states, + bool has_optional_evacuation_work); void evacuate_optional_collection_set(G1ParScanThreadStateSet* per_thread_states); private: // Evacuate the next set of optional regions. diff --git a/src/hotspot/share/gc/g1/g1RemSet.cpp b/src/hotspot/share/gc/g1/g1RemSet.cpp index da72d81825463..98809fc08b461 100644 --- a/src/hotspot/share/gc/g1/g1RemSet.cpp +++ b/src/hotspot/share/gc/g1/g1RemSet.cpp @@ -131,8 +131,17 @@ class G1RemSetScanState : public CHeapObj { } private: - // The complete set of regions which card table needs to be cleared at the end of GC because - // we scribbled all over them. + // The complete set of regions which card table needs to be cleared at the end + // of GC because we scribbled over these card tables. + // + // Regions may be added for two reasons: + // - they were part of the collection set: they may contain g1_young_card_val + // or regular card marks that we never scan so we must always clear their card + // table + // - or in case g1 does an optional evacuation pass, g1 marks the cards in there + // as g1_scanned_card_val. If G1 only did an initial evacuation pass, the + // scanning already cleared these cards. In that case they are not in this set + // at the end of the collection. G1DirtyRegions* _all_dirty_regions; // The set of regions which card table needs to be scanned for new dirty cards // in the current evacuation pass. @@ -321,9 +330,8 @@ class G1RemSetScanState : public CHeapObj { } void prepare_for_merge_heap_roots() { - _all_dirty_regions->merge(_next_dirty_regions); + assert(_next_dirty_regions->size() == 0, "next dirty regions must be empty"); - _next_dirty_regions->reset(); for (size_t i = 0; i < _max_reserved_regions; i++) { _card_table_scan_state[i] = 0; } @@ -331,6 +339,13 @@ class G1RemSetScanState : public CHeapObj { ::memset(_region_scan_chunks, false, _num_total_scan_chunks * sizeof(*_region_scan_chunks)); } + void complete_evac_phase(bool merge_dirty_regions) { + if (merge_dirty_regions) { + _all_dirty_regions->merge(_next_dirty_regions); + } + _next_dirty_regions->reset(); + } + // Returns whether the given region contains cards we need to scan. The remembered // set and other sources may contain cards that // - are in uncommitted regions @@ -374,8 +389,6 @@ class G1RemSetScanState : public CHeapObj { } void cleanup(WorkGang* workers) { - _all_dirty_regions->merge(_next_dirty_regions); - clear_card_table(workers); delete _all_dirty_regions; @@ -448,7 +461,7 @@ class G1RemSetScanState : public CHeapObj { #ifdef ASSERT HeapRegion* hr = G1CollectedHeap::heap()->region_at(region); assert(hr->in_collection_set(), - "Only add young regions to all dirty regions directly but %u is %s", + "Only add collection set regions to all dirty regions directly but %u is %s", hr->hrm_index(), hr->get_short_type_str()); #endif _all_dirty_regions->add_dirty_region(region); @@ -641,6 +654,7 @@ class G1ScanHRForRegionClosure : public HeapRegionClosure { // The address to which this thread already scanned (walked the heap) up to during // card scanning (exclusive). HeapWord* _scanned_to; + G1CardTable::CardValue _scanned_card_value; HeapWord* scan_memregion(uint region_idx_for_card, MemRegion mr) { HeapRegion* const card_region = _g1h->region_at(region_idx_for_card); @@ -677,7 +691,7 @@ class G1ScanHRForRegionClosure : public HeapRegionClosure { } ALWAYSINLINE void do_card_block(uint const region_idx, size_t const first_card, size_t const num_cards) { - _ct->mark_as_scanned(first_card, num_cards); + _ct->change_dirty_cards_to(first_card, num_cards, _scanned_card_value); do_claimed_block(region_idx, first_card, num_cards); _blocks_scanned++; } @@ -727,7 +741,8 @@ class G1ScanHRForRegionClosure : public HeapRegionClosure { G1ScanHRForRegionClosure(G1RemSetScanState* scan_state, G1ParScanThreadState* pss, uint worker_id, - G1GCPhaseTimes::GCParPhases phase) : + G1GCPhaseTimes::GCParPhases phase, + bool remember_already_scanned_cards) : _g1h(G1CollectedHeap::heap()), _ct(_g1h->card_table()), _bot(_g1h->bot()), @@ -740,7 +755,9 @@ class G1ScanHRForRegionClosure : public HeapRegionClosure { _chunks_claimed(0), _rem_set_root_scan_time(), _rem_set_trim_partially_time(), - _scanned_to(NULL) { + _scanned_to(NULL), + _scanned_card_value(remember_already_scanned_cards ? G1CardTable::g1_scanned_card_val() + : G1CardTable::clean_card_val()) { } bool do_heap_region(HeapRegion* r) { @@ -765,10 +782,11 @@ class G1ScanHRForRegionClosure : public HeapRegionClosure { }; void G1RemSet::scan_heap_roots(G1ParScanThreadState* pss, - uint worker_id, - G1GCPhaseTimes::GCParPhases scan_phase, - G1GCPhaseTimes::GCParPhases objcopy_phase) { - G1ScanHRForRegionClosure cl(_scan_state, pss, worker_id, scan_phase); + uint worker_id, + G1GCPhaseTimes::GCParPhases scan_phase, + G1GCPhaseTimes::GCParPhases objcopy_phase, + bool remember_already_scanned_cards) { + G1ScanHRForRegionClosure cl(_scan_state, pss, worker_id, scan_phase, remember_already_scanned_cards); _scan_state->iterate_dirty_regions_from(&cl, worker_id); G1GCPhaseTimes* p = _g1p->phase_times(); @@ -891,18 +909,11 @@ void G1RemSet::scan_collection_set_regions(G1ParScanThreadState* pss, void G1RemSet::prepare_region_for_scan(HeapRegion* region) { uint hrm_index = region->hrm_index(); - if (region->in_collection_set()) { - // Young regions had their card table marked as young at their allocation; - // we need to make sure that these marks are cleared at the end of GC, *but* - // they should not be scanned for cards. - // So directly add them to the "all_dirty_regions". - // Same for regions in the (initial) collection set: they may contain cards from - // the log buffers, make sure they are cleaned. - _scan_state->add_all_dirty_region(hrm_index); - } else if (region->is_old_or_humongous_or_archive()) { + if (region->is_old_or_humongous_or_archive()) { _scan_state->set_scan_top(hrm_index, region->top()); } else { - assert(region->is_free(), "Should only be free region at this point %s", region->get_type_str()); + assert(region->in_collection_set() || region->is_free(), + "Should only be free or in the collection set at this point %s", region->get_type_str()); } } @@ -984,13 +995,34 @@ class G1MergeHeapRootsTask : public AbstractGangTask { } } - virtual bool do_heap_region(HeapRegion* r) { + // Helper to put the remembered set cards for these regions onto the card + // table. + // + // Called directly for humongous starts regions because we should not add + // humongous eager reclaim candidates to the "all" list of regions to + // clear the card table by default as we do not know yet whether this region + // will be reclaimed (and reused). + // If the humongous region contains dirty cards, g1 will scan them + // because dumping the remembered set entries onto the card table will add + // the humongous region to the "dirty" region list to scan. Then scanning + // either clears the card during scan (if there is only an initial evacuation + // pass) or the "dirty" list will be merged with the "all" list later otherwise. + // (And there is no problem either way if the region does not contain dirty + // cards). + void dump_rem_set_for_region(HeapRegion* r) { assert(r->in_collection_set() || r->is_starts_humongous(), "must be"); HeapRegionRemSet* rem_set = r->rem_set(); if (!rem_set->is_empty()) { rem_set->iterate_prts(*this); } + } + + virtual bool do_heap_region(HeapRegion* r) { + assert(r->in_collection_set(), "must be"); + + _scan_state->add_all_dirty_region(r->hrm_index()); + dump_rem_set_for_region(r); return false; } @@ -1022,7 +1054,7 @@ class G1MergeHeapRootsTask : public AbstractGangTask { guarantee(r->rem_set()->occupancy_less_or_equal_than(G1RSetSparseRegionEntries), "Found a not-small remembered set here. This is inconsistent with previous assumptions."); - _cl.do_heap_region(r); + _cl.dump_rem_set_for_region(r); // We should only clear the card based remembered set here as we will not // implicitly rebuild anything else during eager reclaim. Note that at the moment @@ -1239,6 +1271,10 @@ void G1RemSet::merge_heap_roots(bool initial_evacuation) { } } +void G1RemSet::complete_evac_phase(bool has_more_than_one_evacuation_phase) { + _scan_state->complete_evac_phase(has_more_than_one_evacuation_phase); +} + void G1RemSet::exclude_region_from_scan(uint region_idx) { _scan_state->clear_scan_top(region_idx); } diff --git a/src/hotspot/share/gc/g1/g1RemSet.hpp b/src/hotspot/share/gc/g1/g1RemSet.hpp index 534257ddacc18..c44ea34c739ad 100644 --- a/src/hotspot/share/gc/g1/g1RemSet.hpp +++ b/src/hotspot/share/gc/g1/g1RemSet.hpp @@ -84,13 +84,15 @@ class G1RemSet: public CHeapObj { void scan_heap_roots(G1ParScanThreadState* pss, uint worker_id, G1GCPhaseTimes::GCParPhases scan_phase, - G1GCPhaseTimes::GCParPhases objcopy_phase); + G1GCPhaseTimes::GCParPhases objcopy_phase, + bool remember_already_scanned_cards); // Merge cards from various sources (remembered sets, hot card cache, log buffers) // and calculate the cards that need to be scanned later (via scan_heap_roots()). // If initial_evacuation is set, this is called during the initial evacuation. void merge_heap_roots(bool initial_evacuation); + void complete_evac_phase(bool has_more_than_one_evacuation_phase); // Prepare for and cleanup after scanning the heap roots. Must be called // once before and after in sequential code. void prepare_for_scan_heap_roots();