Skip to content
Permalink
Browse files

Add basic implementation of render caching

  • Loading branch information...
ferrybig committed Mar 10, 2019
1 parent cf89255 commit 08d1bc1773d73446e909003f6049f18412d63399
@@ -13,9 +13,9 @@
import me.ferrybig.java.minecraft.overview.mapper.engine.AbstractProgressReporter;
import me.ferrybig.java.minecraft.overview.mapper.engine.RenderEngine;
import me.ferrybig.java.minecraft.overview.mapper.engine.RenderOptions;
import me.ferrybig.java.minecraft.overview.mapper.input.ArchieveInputSource;
import me.ferrybig.java.minecraft.overview.mapper.input.DirectoryInputSource;
import me.ferrybig.java.minecraft.overview.mapper.input.InputSource;
import me.ferrybig.java.minecraft.overview.mapper.input.ArchieveInputSource;
import me.ferrybig.java.minecraft.overview.mapper.input.WorldFile;
import me.ferrybig.java.minecraft.overview.mapper.render.BiomeMap;
import me.ferrybig.java.minecraft.overview.mapper.render.ComplexImageOutputRenderer;
@@ -0,0 +1,114 @@
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package me.ferrybig.java.minecraft.overview.mapper.engine;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.Closeable;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
import java.nio.file.StandardOpenOption;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;

public class RenderCache implements Closeable {

@Nonnull
private final Map<String, CacheEntry> entries;
@Nonnull
private final BufferedWriter writer;

public RenderCache(@Nonnull Path normalFile, @Nonnull Path backupFile) throws IOException {
boolean normalExists = Files.exists(normalFile);
boolean backupExists = Files.exists(backupFile);
if (normalExists && !backupExists) {
Files.move(normalFile, backupFile, ATOMIC_MOVE);
}
Map<String, CacheEntry> entries = new HashMap<>();
if (normalExists || backupExists) {
// Existing files exists, and have been moved to the backup file
try (BufferedReader reader = Files.newBufferedReader(backupFile, StandardCharsets.UTF_8)) {
String nextLine;
while ((nextLine = reader.readLine()) != null) {
CacheEntry entry = CacheEntry.readFromLine(nextLine);
if (entry != null) {
entries.put(entry.filename, entry);
}
}
}
// Recreate normal file
try (BufferedWriter writer = Files.newBufferedWriter(normalFile, StandardCharsets.UTF_8)) {
for (CacheEntry entry : entries.values()) {
writer.write(entry.toLine());
}
}
Files.delete(backupFile);
}
this.entries = new ConcurrentHashMap<>(entries);
this.writer = Files.newBufferedWriter(normalFile, StandardCharsets.UTF_8, StandardOpenOption.APPEND, StandardOpenOption.WRITE, StandardOpenOption.CREATE);
}

public int getLastModificationDate(@Nonnull String fileName) {
CacheEntry entry = this.entries.get(fileName);
if (entry == null) {
return Integer.MIN_VALUE;
}
return entry.lastModification;
}

public void storeLastModificationDate(@Nonnull String fileName, int lastModification) throws IOException {
final CacheEntry cacheEntry = new CacheEntry(fileName, lastModification);
this.entries.put(fileName, cacheEntry);
synchronized (this) {
this.writer.write(cacheEntry.toLine());
this.writer.flush();
}
}

@Override
public void close() throws IOException {
synchronized (this) {
this.writer.close();
}
}

private static class CacheEntry {

@Nonnull
private final String filename;
private final int lastModification;

public CacheEntry(@Nonnull String filename, int lastModification) {
this.filename = filename;
this.lastModification = lastModification;
}

@Nullable
public static CacheEntry readFromLine(@Nonnull String line) {
String[] split = line.split("\t", 3);
if (split.length != 3) {
return null; // Corrupt entry
}
int lastModification = Integer.parseInt(split[0]);
int fileNameLength = Integer.parseInt(split[1]);
if (split[2].length() != fileNameLength) {
return null; // Corrupt entry
}
return new CacheEntry(split[2], lastModification);
}

@Nonnull
public String toLine() {
return lastModification + "\t" + filename.length() + "\t" + filename + "\n";
}
}
}
@@ -6,7 +6,6 @@
package me.ferrybig.java.minecraft.overview.mapper.engine;

import com.google.common.util.concurrent.UncheckedExecutionException;
import java.awt.image.BufferedImage;
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
@@ -23,9 +22,6 @@
import java.util.concurrent.Future;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiConsumer;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -35,6 +31,7 @@
import me.ferrybig.java.minecraft.overview.mapper.input.InputSource;
import me.ferrybig.java.minecraft.overview.mapper.input.WorldFile;
import me.ferrybig.java.minecraft.overview.mapper.render.ImageWriter;
import me.ferrybig.java.minecraft.overview.mapper.render.RenderOutput;

public class RenderEngine implements Closeable {

@@ -125,7 +122,7 @@ public static RenderEngine parellel(@Nonnull RenderOptions options, int maxTasks
public void close() throws IOException {
ParallelOptions parallel = this.parallel;
if (parallel != null) {
if (parallel.isShouldShutdownPool()) {
if (parallel.shouldShutdownPool()) {
parallel.getPool().shutdown();
}
}
@@ -144,7 +141,14 @@ private boolean canConsume(WorldFile file) {
}

public void render() throws IOException {
try (InputInfo inputTask = this.files.generateFileListing()) {
RenderCache cacheInstance;
if (this.imageWriter.supportsCache()) {
cacheInstance = new RenderCache(this.imageWriter.cacheFile(), this.imageWriter.cacheBackupFile());
} else {
cacheInstance = null;
}

try (RenderCache cache = cacheInstance; InputInfo inputTask = this.files.generateFileListing()) {
System.out.println("Render start!");
imageWriter.startRender();

@@ -169,7 +173,7 @@ public void render() throws IOException {
}
}
}
final InputInfo.FileConsumer infoConsumer = makeInfoConsumer(hasSendFullFileList, processedFiles, totalFiles);
final InputInfo.FileConsumer infoConsumer = makeInfoConsumer(hasSendFullFileList, processedFiles, totalFiles, cache);
final ParallelOptions parallelOptions = this.parallel;

if (parallelOptions == null) {
@@ -262,7 +266,8 @@ public void run() {
public InputInfo.FileConsumer makeInfoConsumer(
final boolean hasSendFullFileList,
final Set<WorldFile> processedFiles,
final int totalFiles
final int totalFiles,
final RenderCache cache
) {
return new InputInfo.FileConsumer() {
private final AtomicInteger processedFilesCount = new AtomicInteger();
@@ -278,30 +283,39 @@ public boolean canConsume(WorldFile file) {
}

@Override
public void consume(PreparedFile file) throws IOException {
public void consume(PreparedFile prepared) throws IOException {
final WorldFile file = prepared.getFile();
if (!hasSendFullFileList) {
imageWriter.addKnownFile(file.getFile());
processedFiles.add(file.getFile());
imageWriter.addKnownFile(file);
processedFiles.add(file);
} else {
assert processedFiles.contains(file.getFile());
assert processedFiles.contains(file);
}
RenderEngine.this.reporter.onFileStart(file.getFile());
switch (file.getFile().getType()) {
RenderEngine.this.reporter.onFileStart(file);
switch (file.getType()) {
case REGION_MCA: {
BufferedImage render = RenderEngine.this.renderer.renderFile(file);
RenderEngine.this.imageWriter.addFile(file.getFile(), render);
final String orignalName = file.getOrignalName();
int lastModification = cache.getLastModificationDate(orignalName);
RenderOutput render = RenderEngine.this.renderer.renderFile(prepared, lastModification);
if (render.getOutput() != null) {
RenderEngine.this.imageWriter.addFile(file, render.getOutput());
cache.storeLastModificationDate(orignalName, render.getLastModification());
} else {
System.out.println("Using file from cache: " + orignalName);
RenderEngine.this.imageWriter.addCachedFile(file);
}
}
break;
default: {
RenderEngine.this.imageWriter.addFile(file.getFile(), file);
RenderEngine.this.imageWriter.addFile(file, prepared);
}
}
int processed = processedFilesCount.incrementAndGet();
if (hasSendFullFileList) {
double progress = processed * 100d / totalFiles;
RenderEngine.this.reporter.onProgress(progress, processed, totalFiles);
}
RenderEngine.this.reporter.onFileEnd(file.getFile());
RenderEngine.this.reporter.onFileEnd(file);

}
};
@@ -320,7 +334,7 @@ public ParallelOptions(boolean shouldShutdownPool, int maxTasks, ExecutorService
this.pool = pool;
}

public boolean isShouldShutdownPool() {
public boolean shouldShutdownPool() {
return shouldShutdownPool;
}

@@ -8,18 +8,13 @@
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
import java.util.Collections;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.zip.GZIPInputStream;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.ArchiveException;
import org.apache.commons.compress.archivers.ArchiveInputStream;
import org.apache.commons.compress.archivers.ArchiveStreamFactory;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;

public class ArchieveInputSource implements InputSource {

@@ -91,13 +86,13 @@ public InputInfo generateFileListing() throws IOException {
ArchiveInputStream inWrapper;
try {
String baseName = file.getName();
if(baseName.endsWith(".tar.gz")) {
if (baseName.endsWith(".tar.gz")) {
inWrapper = new ArchiveStreamFactory().createArchiveInputStream(ArchiveStreamFactory.TAR, new GZIPInputStream(fos));
} else if(baseName.endsWith(".tar")) {
} else if (baseName.endsWith(".tar")) {
inWrapper = new ArchiveStreamFactory().createArchiveInputStream(ArchiveStreamFactory.TAR, fos);
} else if(baseName.endsWith(".zip")) {
} else if (baseName.endsWith(".zip")) {
inWrapper = new ArchiveStreamFactory().createArchiveInputStream(ArchiveStreamFactory.ZIP, fos);
} else if(baseName.endsWith(".7z")) {
} else if (baseName.endsWith(".7z")) {
inWrapper = new ArchiveStreamFactory().createArchiveInputStream(ArchiveStreamFactory.SEVEN_Z, fos);
} else {
throw new IOException("Unsupported archieve type! " + baseName);
@@ -120,16 +115,16 @@ public void close() throws IOException {
@Override
public void forAllFiles(InputInfo.FileConsumer consumer) throws IOException {
ArchiveEntry nextEntry;
while((nextEntry = in.getNextEntry()) != null) {
while ((nextEntry = in.getNextEntry()) != null) {
String fileName = nextEntry.getName();
if(!subDirectory.isEmpty()) {
if(!fileName.startsWith(subDirectory)) {
if (!subDirectory.isEmpty()) {
if (!fileName.startsWith(subDirectory)) {
continue;
}
fileName = fileName.substring(subDirectory.length() + 1);
}
WorldFile file = WorldFile.of(fileName);
if(!consumer.canConsume(file)) {
if (!consumer.canConsume(file)) {
continue;
}
consumer.consume(PreparedFile.of(file, in));
@@ -0,0 +1,30 @@
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package me.ferrybig.java.minecraft.overview.mapper.render;

import java.io.IOException;
import java.nio.file.Path;

public abstract class CachedImageWriter implements ImageWriter {

protected abstract Path getOutputDirectory();

@Override
public Path cacheBackupFile() throws IOException {
return this.getOutputDirectory().resolve("render-cache.binlog");
}

@Override
public Path cacheFile() throws IOException {
return this.getOutputDirectory().resolve("render-cache.bak.binlog");
}

@Override
public boolean supportsCache() {
return true;
}

}
@@ -45,22 +45,36 @@
private final PriorityQueue<Integer> chunkIndexes;
private final ByteCountingDataInputStream in;
private byte[] dataCache = new byte[4 * KIBIBYTE];
private final int[] timeStamps = new int[SECTOR_INTS];
private final int lastModificationDate;

public ChunkReader(InputStream input) throws IOException {
this.in = new ByteCountingDataInputStream(wrapWithBuffer(input));
this.chunkIndexes = new PriorityQueue<>(Integer::compare);
for (int k = 0; k < SECTOR_INTS; ++k) {
for (int i = 0; i < SECTOR_INTS; ++i) {
int offset = in.readInt();
if (offset != 0) {
int sectorNumber = offset >> 8;
this.chunkIndexes.add(sectorNumber * SECTOR_BYTES);
}
}
assert this.in.getReadBytes() == SECTOR_BYTES;
this.skipFully(in, SECTOR_BYTES); // Skip timestamp sector
int lastModified = 0;
for (int i = 0; i < SECTOR_INTS; ++i) {
int timestamp = in.readInt();
timeStamps[i] = timestamp;
if (timestamp > lastModified) {
lastModified = timestamp;
}
}
this.lastModificationDate = lastModified;
assert this.in.getReadBytes() == SECTOR_BYTES * 2;
}

public int getLastModificationDate() {
return lastModificationDate;
}

@Override
public void close() throws IOException {
this.in.close();
Oops, something went wrong.

0 comments on commit 08d1bc1

Please sign in to comment.
You can’t perform that action at this time.
You signed in with another tab or window. Reload to refresh your session. You signed out in another tab or window. Reload to refresh your session.