Skip to content
Permalink
Browse files

Memory optimizations (#505)

* Remove LocatedBlock overhead in LBL map

* Add new space-efficient block map, with thourough testing

* Drop ordering property, add full insertion test

* Add licenses

* Fix mocked platform conflicts

* Disable full block map testing for faster builds

* Re-implement BlockMap with fastutil maps

* Re-write chunk batching to be memory efficient

* Make MultiStageReorder use BlockMap

* Increase LBL load factor, fix long-pack limit detection

* Fix infinite loop in chunk batching

* Save memory in history by cleaning up MSR

* Re-implement LocatedBlockList in BlockMap

* Fix data race with BlockType lazy fields

* Make IDs ALWAYS present, only runtime-consistent. Use for memory efficiency in BlockMap

* Remap inner structure of BlockMap for smaller maps

* Remove containedBlocks fields, not very efficient

* Fix minor de-optimizing bug in stage reorder

* Make long packed y signed

* Add extended Y limit configuration option

* Add licenses

* Store 3 ints for unoptimized BV list

* Add final to BitMath

* Correct int-cast for long-packing
  • Loading branch information...
kenzierocks authored and me4502 committed Aug 12, 2019
1 parent ec5bc5a commit f472c20bfbb125361834cfeb36650f493b17f0c0
Showing with 2,014 additions and 139 deletions.
  1. +3 −0 buildSrc/src/main/kotlin/PlatformConfig.kt
  2. +1 −0 buildSrc/src/main/kotlin/Versions.kt
  3. +1 −0 config/checkstyle/import-control.xml
  4. +3 −0 worldedit-bukkit/build.gradle.kts
  5. +1 −1 worldedit-core/build.gradle.kts
  6. +1 −0 worldedit-core/src/main/java/com/sk89q/worldedit/LocalConfiguration.java
  7. +16 −47 worldedit-core/src/main/java/com/sk89q/worldedit/extent/reorder/ChunkBatchingExtent.java
  8. +21 −17 worldedit-core/src/main/java/com/sk89q/worldedit/extent/reorder/MultiStageReorder.java
  9. +60 −0 worldedit-core/src/main/java/com/sk89q/worldedit/function/operation/SetBlockMap.java
  10. +23 −10 worldedit-core/src/main/java/com/sk89q/worldedit/internal/block/BlockStateIdAccess.java
  11. +52 −0 worldedit-core/src/main/java/com/sk89q/worldedit/math/BitMath.java
  12. +34 −0 worldedit-core/src/main/java/com/sk89q/worldedit/math/BlockVector3.java
  13. +41 −0 worldedit-core/src/main/java/com/sk89q/worldedit/math/RegionOptimizedChunkComparator.java
  14. +36 −0 worldedit-core/src/main/java/com/sk89q/worldedit/math/RegionOptimizedComparator.java
  15. +1 −0 worldedit-core/src/main/java/com/sk89q/worldedit/util/PropertiesConfiguration.java
  16. +1 −0 worldedit-core/src/main/java/com/sk89q/worldedit/util/YAMLConfiguration.java
  17. +427 −0 worldedit-core/src/main/java/com/sk89q/worldedit/util/collection/BlockMap.java
  18. +21 −21 worldedit-core/src/main/java/com/sk89q/worldedit/util/collection/LocatedBlockList.java
  19. +91 −0 worldedit-core/src/main/java/com/sk89q/worldedit/util/collection/LongPositionList.java
  20. +47 −0 worldedit-core/src/main/java/com/sk89q/worldedit/util/collection/PositionList.java
  21. +194 −0 worldedit-core/src/main/java/com/sk89q/worldedit/util/collection/SubBlockMap.java
  22. +98 −0 worldedit-core/src/main/java/com/sk89q/worldedit/util/collection/VectorPositionList.java
  23. +75 −0 worldedit-core/src/main/java/com/sk89q/worldedit/util/concurrency/LazyReference.java
  24. +0 −3 worldedit-core/src/main/java/com/sk89q/worldedit/world/block/BlockState.java
  25. +26 −37 worldedit-core/src/main/java/com/sk89q/worldedit/world/block/BlockType.java
  26. +12 −3 worldedit-core/src/test/java/com/sk89q/worldedit/internal/expression/ExpressionTest.java
  27. +133 −0 worldedit-core/src/test/java/com/sk89q/worldedit/util/VariedVectorsProvider.java
  28. +588 −0 worldedit-core/src/test/java/com/sk89q/worldedit/util/collection/BlockMapTest.java
  29. +5 −0 worldedit-core/src/test/resources/junit-platform.properties
  30. +2 −0 worldedit-sponge/src/main/java/com/sk89q/worldedit/sponge/config/ConfigurateConfiguration.java
@@ -46,6 +46,9 @@ fun Project.applyPlatformAndCoreConfiguration() {

dependencies {
"testImplementation"("org.junit.jupiter:junit-jupiter-api:${Versions.JUNIT}")
"testImplementation"("org.junit.jupiter:junit-jupiter-params:${Versions.JUNIT}")
"testImplementation"("org.mockito:mockito-core:${Versions.MOCKITO}")
"testImplementation"("org.mockito:mockito-junit-jupiter:${Versions.MOCKITO}")
"testRuntime"("org.junit.jupiter:junit-jupiter-engine:${Versions.JUNIT}")
}

@@ -4,4 +4,5 @@ object Versions {
const val PISTON = "0.4.3"
const val AUTO_VALUE = "1.6.5"
const val JUNIT = "5.5.0"
const val MOCKITO = "3.0.0"
}
@@ -40,6 +40,7 @@
<allow pkg="org.mozilla.javascript"/>
<allow pkg="de.schlichtherle"/>
<allow pkg="com.google.auto"/>
<allow pkg="it.unimi.dsi.fastutil"/>

<subpackage name="bukkit">
<allow pkg="org.bukkit"/>
@@ -63,6 +63,9 @@ tasks.named<ShadowJar>("shadowJar") {
relocate("io.papermc.lib", "com.sk89q.worldedit.bukkit.paperlib") {
include(dependency("io.papermc:paperlib:1.0.2"))
}
relocate("it.unimi.dsi.fastutil", "com.sk89q.worldedit.bukkit.fastutil") {
include(dependency("it.unimi.dsi:fastutil"))
}
}
}

@@ -21,14 +21,14 @@ dependencies {
"compile"("com.google.code.findbugs:jsr305:1.3.9")
"compile"("com.google.code.gson:gson:2.8.0")
"compile"("org.slf4j:slf4j-api:1.7.26")
"compile"("it.unimi.dsi:fastutil:8.2.1")

"compileOnly"(project(":worldedit-libs:core:ap"))
"annotationProcessor"(project(":worldedit-libs:core:ap"))
// ensure this is on the classpath for the AP
"annotationProcessor"("com.google.guava:guava:21.0")
"compileOnly"("com.google.auto.value:auto-value-annotations:${Versions.AUTO_VALUE}")
"annotationProcessor"("com.google.auto.value:auto-value:${Versions.AUTO_VALUE}")
"testCompile"("org.mockito:mockito-core:1.9.0-rc1")
}

tasks.withType<JavaCompile>().configureEach {
@@ -75,6 +75,7 @@
public int butcherMaxRadius = -1;
public boolean allowSymlinks = false;
public boolean serverSideCUI = true;
public boolean extendedYLimit = false;

protected String[] getDefaultDisallowedBlocks() {
List<BlockType> blockTypes = Lists.newArrayList(
@@ -19,25 +19,21 @@

package com.sk89q.worldedit.extent.reorder;

import com.google.common.collect.Table;
import com.google.common.collect.TreeBasedTable;
import com.google.common.collect.ImmutableSortedSet;
import com.sk89q.worldedit.WorldEditException;
import com.sk89q.worldedit.extent.AbstractBufferingExtent;
import com.sk89q.worldedit.extent.Extent;
import com.sk89q.worldedit.function.operation.Operation;
import com.sk89q.worldedit.function.operation.RunContext;
import com.sk89q.worldedit.math.BlockVector2;
import com.sk89q.worldedit.math.BlockVector3;
import com.sk89q.worldedit.math.RegionOptimizedComparator;
import com.sk89q.worldedit.util.collection.BlockMap;
import com.sk89q.worldedit.world.block.BaseBlock;
import com.sk89q.worldedit.world.block.BlockStateHolder;

import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;

/**
* A special extent that batches changes into Minecraft chunks. This helps
@@ -47,17 +43,7 @@
*/
public class ChunkBatchingExtent extends AbstractBufferingExtent {

/**
* Comparator optimized for sorting chunks by the region file they reside
* in. This allows for file caches to be used while loading the chunk.
*/
private static final Comparator<BlockVector2> REGION_OPTIMIZED_SORT =
Comparator.comparing((BlockVector2 vec) -> vec.shr(5), BlockVector2.COMPARING_GRID_ARRANGEMENT)
.thenComparing(BlockVector2.COMPARING_GRID_ARRANGEMENT);

private final Table<BlockVector2, BlockVector3, BaseBlock> batches =
TreeBasedTable.create(REGION_OPTIMIZED_SORT, BlockVector3.sortByCoordsYzx());
private final Set<BlockVector3> containedBlocks = new HashSet<>();
private final BlockMap blockMap = BlockMap.create();
private boolean enabled;

public ChunkBatchingExtent(Extent extent) {
@@ -81,32 +67,18 @@ public boolean commitRequired() {
return enabled;
}

private BlockVector2 getChunkPos(BlockVector3 location) {
return location.shr(4).toBlockVector2();
}

private BlockVector3 getInChunkPos(BlockVector3 location) {
return BlockVector3.at(location.getX() & 15, location.getY(), location.getZ() & 15);
}

@Override
public <B extends BlockStateHolder<B>> boolean setBlock(BlockVector3 location, B block) throws WorldEditException {
if (!enabled) {
return setDelegateBlock(location, block);
}
BlockVector2 chunkPos = getChunkPos(location);
BlockVector3 inChunkPos = getInChunkPos(location);
batches.put(chunkPos, inChunkPos, block.toBaseBlock());
containedBlocks.add(location);
blockMap.put(location, block.toBaseBlock());
return true;
}

@Override
protected Optional<BaseBlock> getBufferedBlock(BlockVector3 position) {
if (!containedBlocks.contains(position)) {
return Optional.empty();
}
return Optional.of(batches.get(getChunkPos(position), getInChunkPos(position)));
return Optional.ofNullable(blockMap.get(position));
}

@Override
@@ -117,24 +89,21 @@ protected Operation commitBefore() {
return new Operation() {

// we get modified between create/resume -- only create this on resume to prevent CME
private Iterator<Map.Entry<BlockVector2, Map<BlockVector3, BaseBlock>>> batchIterator;
private Iterator<BlockVector3> iterator;

@Override
public Operation resume(RunContext run) throws WorldEditException {
if (batchIterator == null) {
batchIterator = batches.rowMap().entrySet().iterator();
}
if (!batchIterator.hasNext()) {
return null;
if (iterator == null) {
iterator = ImmutableSortedSet.copyOf(RegionOptimizedComparator.INSTANCE,
blockMap.keySet()).iterator();
}
Map.Entry<BlockVector2, Map<BlockVector3, BaseBlock>> next = batchIterator.next();
BlockVector3 chunkOffset = next.getKey().toBlockVector3().shl(4);
for (Map.Entry<BlockVector3, BaseBlock> block : next.getValue().entrySet()) {
getExtent().setBlock(block.getKey().add(chunkOffset), block.getValue());
containedBlocks.remove(block.getKey());
while (iterator.hasNext()) {
BlockVector3 position = iterator.next();
BaseBlock block = blockMap.get(position);
getExtent().setBlock(position, block);
}
batchIterator.remove();
return this;
blockMap.clear();
return null;
}

@Override
@@ -24,9 +24,10 @@
import com.sk89q.worldedit.extent.Extent;
import com.sk89q.worldedit.function.operation.Operation;
import com.sk89q.worldedit.function.operation.OperationQueue;
import com.sk89q.worldedit.function.operation.SetLocatedBlocks;
import com.sk89q.worldedit.function.operation.RunContext;
import com.sk89q.worldedit.function.operation.SetBlockMap;
import com.sk89q.worldedit.math.BlockVector3;
import com.sk89q.worldedit.util.collection.LocatedBlockList;
import com.sk89q.worldedit.util.collection.BlockMap;
import com.sk89q.worldedit.world.block.BaseBlock;
import com.sk89q.worldedit.world.block.BlockCategories;
import com.sk89q.worldedit.world.block.BlockState;
@@ -36,12 +37,10 @@

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;

/**
* Re-orders blocks into several stages.
@@ -143,8 +142,7 @@
priorityMap.put(BlockTypes.MOVING_PISTON, PlacementPriority.FINAL);
}

private final Set<BlockVector3> containedBlocks = new HashSet<>();
private Map<PlacementPriority, LocatedBlockList> stages = new HashMap<>();
private Map<PlacementPriority, BlockMap> stages = new HashMap<>();

private boolean enabled;

@@ -178,7 +176,7 @@ public MultiStageReorder(Extent extent, boolean enabled) {
this.enabled = enabled;

for (PlacementPriority priority : PlacementPriority.values()) {
stages.put(priority, new LocatedBlockList());
stages.put(priority, BlockMap.create());
}
}

@@ -220,7 +218,7 @@ public boolean commitRequired() {
return setDelegateBlock(location, block);
}

BlockState existing = getBlock(location);
BlockState existing = getExtent().getBlock(location);
PlacementPriority priority = getPlacementPriority(block);
PlacementPriority srcPriority = getPlacementPriority(existing);

@@ -229,13 +227,13 @@ public boolean commitRequired() {

switch (srcPriority) {
case FINAL:
stages.get(PlacementPriority.CLEAR_FINAL).add(location, replacement);
stages.get(PlacementPriority.CLEAR_FINAL).put(location, replacement);
break;
case LATE:
stages.get(PlacementPriority.CLEAR_LATE).add(location, replacement);
stages.get(PlacementPriority.CLEAR_LATE).put(location, replacement);
break;
case LAST:
stages.get(PlacementPriority.CLEAR_LAST).add(location, replacement);
stages.get(PlacementPriority.CLEAR_LAST).put(location, replacement);
break;
}

@@ -244,16 +242,12 @@ public boolean commitRequired() {
}
}

stages.get(priority).add(location, block);
containedBlocks.add(location);
stages.get(priority).put(location, block.toBaseBlock());
return !existing.equalsFuzzy(block);
}

@Override
protected Optional<BaseBlock> getBufferedBlock(BlockVector3 position) {
if (!containedBlocks.contains(position)) {
return Optional.empty();
}
return stages.values().stream()
.map(blocks -> blocks.get(position))
.filter(Objects::nonNull)
@@ -267,7 +261,17 @@ public Operation commitBefore() {
}
List<Operation> operations = new ArrayList<>();
for (PlacementPriority priority : PlacementPriority.values()) {
operations.add(new SetLocatedBlocks(getExtent(), stages.get(priority)));
BlockMap blocks = stages.get(priority);
operations.add(new SetBlockMap(getExtent(), blocks) {
@Override
public Operation resume(RunContext run) throws WorldEditException {
Operation operation = super.resume(run);
if (operation == null) {
blocks.clear();
}
return operation;
}
});
}

return new OperationQueue(operations);
@@ -0,0 +1,60 @@
/*
* WorldEdit, a Minecraft world manipulation toolkit
* Copyright (C) sk89q <http://www.sk89q.com>
* Copyright (C) WorldEdit team and contributors
*
* This program is free software: you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published by the
* Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
* for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

package com.sk89q.worldedit.function.operation;

import com.sk89q.worldedit.WorldEditException;
import com.sk89q.worldedit.extent.Extent;
import com.sk89q.worldedit.math.BlockVector3;
import com.sk89q.worldedit.util.LocatedBlock;
import com.sk89q.worldedit.util.collection.BlockMap;
import com.sk89q.worldedit.world.block.BaseBlock;

import java.util.List;
import java.util.Map;

import static com.google.common.base.Preconditions.checkNotNull;

public class SetBlockMap implements Operation {

private final Extent extent;
private final BlockMap blocks;

public SetBlockMap(Extent extent, BlockMap blocks) {
this.extent = checkNotNull(extent);
this.blocks = checkNotNull(blocks);
}

@Override
public Operation resume(RunContext run) throws WorldEditException {
for (Map.Entry<BlockVector3, BaseBlock> entry : blocks.entrySet()) {
extent.setBlock(entry.getKey(), entry.getValue());
}
return null;
}

@Override
public void cancel() {
}

@Override
public void addStatusMessages(List<String> messages) {
}

}
@@ -22,10 +22,10 @@
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import com.sk89q.worldedit.world.block.BlockState;
import com.sk89q.worldedit.world.registry.BlockRegistry;

import javax.annotation.Nullable;
import java.util.Arrays;
import java.util.Map;
import java.util.BitSet;
import java.util.OptionalInt;

import static com.google.common.base.Preconditions.checkState;
@@ -43,19 +43,32 @@ public static OptionalInt getBlockStateId(BlockState holder) {
return ASSIGNED_IDS.inverse().get(id);
}

/**
* For platforms that don't have an internal ID system,
* {@link BlockRegistry#getInternalBlockStateId(BlockState)} will return
* {@link OptionalInt#empty()}. In those cases, we will use our own ID system,
* since it's useful for other entries as well.
* @return an unused ID in WorldEdit's ID tracker
*/
private static int provideUnusedWorldEditId() {
return usedIds.nextClearBit(0);
}

private static final BitSet usedIds = new BitSet();

public static void register(BlockState blockState, OptionalInt id) {
if (id.isPresent()) {
int i = id.getAsInt();
BlockState existing = ASSIGNED_IDS.inverse().get(i);
checkState(existing == null || existing == blockState,
"BlockState %s is using the same block ID (%s) as BlockState %s",
blockState, i, existing);
ASSIGNED_IDS.put(blockState, i);
}
int i = id.orElseGet(BlockStateIdAccess::provideUnusedWorldEditId);
BlockState existing = ASSIGNED_IDS.inverse().get(i);
checkState(existing == null || existing == blockState,
"BlockState %s is using the same block ID (%s) as BlockState %s",
blockState, i, existing);
ASSIGNED_IDS.put(blockState, i);
usedIds.set(i);
}

public static void clear() {
ASSIGNED_IDS.clear();
usedIds.clear();
}

private BlockStateIdAccess() {

0 comments on commit f472c20

Please sign in to comment.
You can’t perform that action at this time.