Skip to content
Permalink
Browse files

Re-write chunk batching to be memory efficient

  • Loading branch information...
kenzierocks committed Jul 16, 2019
1 parent 2dd5199 commit abc629091f292c872dffec98c0f2a684966f01bd
@@ -19,25 +19,22 @@

package com.sk89q.worldedit.extent.reorder;

import com.google.common.collect.Table;
import com.google.common.collect.TreeBasedTable;
import com.sk89q.worldedit.WorldEditException;
import com.sk89q.worldedit.extent.AbstractBufferingExtent;
import com.sk89q.worldedit.extent.Extent;
import com.sk89q.worldedit.function.operation.Operation;
import com.sk89q.worldedit.function.operation.RunContext;
import com.sk89q.worldedit.math.BlockVector2;
import com.sk89q.worldedit.math.BlockVector3;
import com.sk89q.worldedit.math.RegionOptimizedLongComparator;
import com.sk89q.worldedit.util.collection.BlockMap;
import com.sk89q.worldedit.world.block.BaseBlock;
import com.sk89q.worldedit.world.block.BlockStateHolder;
import it.unimi.dsi.fastutil.longs.LongAVLTreeSet;
import it.unimi.dsi.fastutil.longs.LongIterator;
import it.unimi.dsi.fastutil.longs.LongSet;

import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;

/**
* A special extent that batches changes into Minecraft chunks. This helps
@@ -47,17 +44,8 @@
*/
public class ChunkBatchingExtent extends AbstractBufferingExtent {

/**
* Comparator optimized for sorting chunks by the region file they reside
* in. This allows for file caches to be used while loading the chunk.
*/
private static final Comparator<BlockVector2> REGION_OPTIMIZED_SORT =
Comparator.comparing((BlockVector2 vec) -> vec.shr(5), BlockVector2.COMPARING_GRID_ARRANGEMENT)
.thenComparing(BlockVector2.COMPARING_GRID_ARRANGEMENT);

private final Table<BlockVector2, BlockVector3, BaseBlock> batches =
TreeBasedTable.create(REGION_OPTIMIZED_SORT, BlockVector3.sortByCoordsYzx());
private final Set<BlockVector3> containedBlocks = new HashSet<>();
private final BlockMap blockMap = BlockMap.create();
private final LongSet containedBlocks = new LongAVLTreeSet(RegionOptimizedLongComparator.INSTANCE);
private boolean enabled;

public ChunkBatchingExtent(Extent extent) {
@@ -81,32 +69,19 @@ public boolean commitRequired() {
return enabled;
}

private BlockVector2 getChunkPos(BlockVector3 location) {
return location.shr(4).toBlockVector2();
}

private BlockVector3 getInChunkPos(BlockVector3 location) {
return BlockVector3.at(location.getX() & 15, location.getY(), location.getZ() & 15);
}

@Override
public <B extends BlockStateHolder<B>> boolean setBlock(BlockVector3 location, B block) throws WorldEditException {
if (!enabled) {
return setDelegateBlock(location, block);
}
BlockVector2 chunkPos = getChunkPos(location);
BlockVector3 inChunkPos = getInChunkPos(location);
batches.put(chunkPos, inChunkPos, block.toBaseBlock());
containedBlocks.add(location);
blockMap.put(location, block.toBaseBlock());
containedBlocks.add(location.toLongPackedForm());
return true;
}

@Override
protected Optional<BaseBlock> getBufferedBlock(BlockVector3 position) {
if (!containedBlocks.contains(position)) {
return Optional.empty();
}
return Optional.of(batches.get(getChunkPos(position), getInChunkPos(position)));
return Optional.ofNullable(blockMap.get(position));
}

@Override
@@ -117,23 +92,19 @@ protected Operation commitBefore() {
return new Operation() {

// we get modified between create/resume -- only create this on resume to prevent CME
private Iterator<Map.Entry<BlockVector2, Map<BlockVector3, BaseBlock>>> batchIterator;
private LongIterator iterator;

@Override
public Operation resume(RunContext run) throws WorldEditException {
if (batchIterator == null) {
batchIterator = batches.rowMap().entrySet().iterator();
}
if (!batchIterator.hasNext()) {
return null;
if (iterator == null) {
iterator = containedBlocks.iterator();
}
Map.Entry<BlockVector2, Map<BlockVector3, BaseBlock>> next = batchIterator.next();
BlockVector3 chunkOffset = next.getKey().toBlockVector3().shl(4);
for (Map.Entry<BlockVector3, BaseBlock> block : next.getValue().entrySet()) {
getExtent().setBlock(block.getKey().add(chunkOffset), block.getValue());
containedBlocks.remove(block.getKey());
while (iterator.hasNext()) {
BlockVector3 position = BlockVector3.fromLongPackedForm(iterator.nextLong());
BaseBlock block = blockMap.get(position);
getExtent().setBlock(position, block);
iterator.remove();
}
batchIterator.remove();
return this;
}

@@ -0,0 +1,37 @@
package com.sk89q.worldedit.math;

public class BitMath {

public static final long BITS_26 = 0x3_FF_FF_FF;
public static final int BITS_14 = 0x3F_FF;
public static final int BITS_12 = 0x0F_FF;
public static final int BITS_8 = 0xFF;
public static final int BITS_4 = 0x0F;

private static final int FIX_SIGN_SHIFT = 32 - 26;

public static int unpackX(long packed) {
return fixSign26((int) (packed & BITS_26));
}

public static int unpackZ(long packed) {
return fixSign26((int) ((packed >> 26) & BITS_26));
}

public static int unpackY(long packed) {
return (int) ((packed >> (26 + 26)) & BITS_12);
}

/**
* Fix horizontal sign -- we have a 26-bit two's-complement int,
* we need it to be a 32-bit two's-complement int.
*/
public static int fixSign26(int h) {
// Using https://stackoverflow.com/a/29266331/436524
return (h << FIX_SIGN_SHIFT) >> FIX_SIGN_SHIFT;
}

private BitMath() {
}

}
@@ -24,6 +24,12 @@
import java.util.Comparator;

import static com.google.common.base.Preconditions.checkArgument;
import static com.sk89q.worldedit.math.BitMath.BITS_12;
import static com.sk89q.worldedit.math.BitMath.BITS_26;
import static com.sk89q.worldedit.math.BitMath.fixSign26;
import static com.sk89q.worldedit.math.BitMath.unpackX;
import static com.sk89q.worldedit.math.BitMath.unpackY;
import static com.sk89q.worldedit.math.BitMath.unpackZ;

/**
* An immutable 3-dimensional vector.
@@ -61,6 +67,29 @@ public static BlockVector3 at(int x, int y, int z) {
return new BlockVector3(x, y, z);
}

private static final int WORLD_XZ_MINMAX = 30_000_000;
private static final int WORLD_Y_MAX = 4095;

private static boolean isHorizontallyOOB(int h) {
return h < -WORLD_XZ_MINMAX || h > WORLD_XZ_MINMAX;
}

public static boolean isLongPackable(BlockVector3 location) {
return isHorizontallyOOB(location.getX()) ||
isHorizontallyOOB(location.getZ()) ||
location.getY() < 0 || location.getY() > WORLD_Y_MAX;
}

public static void checkLongPackable(BlockVector3 location) {
checkArgument(isLongPackable(location),
"Location exceeds long packing limits: %s", location);
}


public static BlockVector3 fromLongPackedForm(long packed) {
return at(unpackX(packed), unpackY(packed), unpackZ(packed));
}

// thread-safe initialization idiom
private static final class YzxOrderComparator {
private static final Comparator<BlockVector3> YZX_ORDER =
@@ -94,6 +123,11 @@ private BlockVector3(int x, int y, int z) {
this.z = z;
}

public long toLongPackedForm() {
checkLongPackable(this);
return (x & BITS_26) | ((z & BITS_26) << 26) | (((y & (long) BITS_12) << (26 + 26)));
}

/**
* Get the X coordinate.
*
@@ -0,0 +1,43 @@
package com.sk89q.worldedit.math;

import it.unimi.dsi.fastutil.longs.LongComparator;

import static com.sk89q.worldedit.math.BitMath.unpackX;
import static com.sk89q.worldedit.math.BitMath.unpackZ;

/**
* Sort packed block positions by region, then chunk.
*/
public class RegionOptimizedChunkLongComparator implements LongComparator {

public static final RegionOptimizedChunkLongComparator INSTANCE
= new RegionOptimizedChunkLongComparator();

private RegionOptimizedChunkLongComparator() {
}

@Override
public int compare(long a, long b) {
int acz = unpackZ(a) >> 4;
int bcz = unpackZ(b) >> 4;
// Region Z
int cmp = Integer.compare(acz >> 5, bcz >> 5);
if (cmp != 0) {
return cmp;
}
int acx = unpackX(a) >> 4;
int bcx = unpackX(b) >> 4;
// Region X
cmp = Integer.compare(acx >> 5, bcx >> 5);
if (cmp != 0) {
return cmp;
}
// Chunk Z
cmp = Integer.compare(acz, bcz);
if (cmp != 0) {
return cmp;
}
// Chunk X
return Integer.compare(acx, bcx);
}
}
@@ -0,0 +1,36 @@
package com.sk89q.worldedit.math;

import it.unimi.dsi.fastutil.longs.LongComparator;

import static com.sk89q.worldedit.math.BitMath.unpackX;
import static com.sk89q.worldedit.math.BitMath.unpackY;
import static com.sk89q.worldedit.math.BitMath.unpackZ;

/**
* Sort packed block positions by region, chunk, and finally Y-Z-X.
*/
public class RegionOptimizedLongComparator implements LongComparator {

public static final RegionOptimizedLongComparator INSTANCE
= new RegionOptimizedLongComparator();

private RegionOptimizedLongComparator() {
}

@Override
public int compare(long a, long b) {
int cmp = RegionOptimizedChunkLongComparator.INSTANCE.compare(a, b);
if (cmp != 0) {
return cmp;
}
cmp = Integer.compare(unpackY(a), unpackY(b));
if (cmp != 0) {
return cmp;
}
cmp = Integer.compare(unpackZ(a), unpackZ(b));
if (cmp != 0) {
return cmp;
}
return Integer.compare(unpackX(a), unpackX(b));
}
}
@@ -39,6 +39,12 @@
import java.util.function.BiFunction;
import java.util.function.Function;

import static com.sk89q.worldedit.math.BitMath.BITS_12;
import static com.sk89q.worldedit.math.BitMath.BITS_14;
import static com.sk89q.worldedit.math.BitMath.BITS_4;
import static com.sk89q.worldedit.math.BitMath.BITS_8;
import static com.sk89q.worldedit.math.BitMath.fixSign26;

/**
* A space-efficient map implementation for block locations.
*/
@@ -58,9 +64,6 @@ public static BlockMap copyOf(Map<? extends BlockVector3, ? extends BaseBlock> s
return new BlockMap(source);
}

private static final int WORLD_XZ_MINMAX = 30_000_000;
private static final int WORLD_Y_MAX = 4095;

/*
* Stores blocks by sub-dividing them into smaller groups.
* A block location is 26 bits long for x + z, and usually
@@ -74,21 +77,9 @@ public static BlockMap copyOf(Map<? extends BlockVector3, ? extends BaseBlock> s
* This means that each group has 12 bits of x + z, and 8 bits of y.
*/

private static final int BITS_14 = 0x3F_FF;
private static final int BITS_12 = 0x0F_FF;
private static final int BITS_8 = 0xFF;
private static final int BITS_4 = 0x0F;

private static boolean isHorizontallyOOB(int h) {
return h < -WORLD_XZ_MINMAX || h > WORLD_XZ_MINMAX;
}

private static int toGroupKey(BlockVector3 location) {
if (isHorizontallyOOB(location.getX()) ||
isHorizontallyOOB(location.getZ()) ||
location.getY() < 0 || location.getY() > WORLD_Y_MAX) {
throw new IllegalArgumentException("Location exceeds OrderedBlockMap limits: " + location);
}
BlockVector3.checkLongPackable(location);
return ((location.getX() >>> 12) & BITS_14)
| (((location.getZ() >>> 12) & BITS_14) << 14)
| (((location.getY() >>> 8) & BITS_4) << (14 + 14));
@@ -108,23 +99,12 @@ private static int toInnerKey(BlockVector3 location) {
private static final int INNER_Y = BITS_8 << (12 + 12);

private static BlockVector3 reconstructLocation(int group, int inner) {
int x = fixSign(((group & GROUP_X) << 12) | (inner & INNER_X));
int z = fixSign(((group & GROUP_Z) >>> (14 - 12)) | ((inner & INNER_Z) >>> 12));
int x = fixSign26(((group & GROUP_X) << 12) | (inner & INNER_X));
int z = fixSign26(((group & GROUP_Z) >>> (14 - 12)) | ((inner & INNER_Z) >>> 12));
int y = ((group & GROUP_Y) >>> (14 + 14 - 8)) | ((inner & INNER_Y) >>> (12 + 12));
return BlockVector3.at(x, y, z);
}

private static final int FIX_SIGN_SHIFT = 32 - 26;

/**
* Fix horizontal sign -- we have a 26-bit two's-complement int,
* we need it to be a 32-bit two's-complement int.
*/
private static int fixSign(int h) {
// Using https://stackoverflow.com/a/29266331/436524
return (h << FIX_SIGN_SHIFT) >> FIX_SIGN_SHIFT;
}

private final Int2ObjectMap<Int2ObjectMap<BaseBlock>> maps = new Int2ObjectOpenHashMap<>();
private Set<Entry<BlockVector3, BaseBlock>> entrySet;
private Collection<BaseBlock> values;

0 comments on commit abc6290

Please sign in to comment.
You can’t perform that action at this time.