Skip to content

Commit

Permalink
Made Cache compression level and lock timeout user configurable
Browse files Browse the repository at this point in the history
  • Loading branch information
@luccioman
luccioman committed Jun 14, 2017
1 parent a7394b4 commit 28b451a
Show file tree
Hide file tree
Showing 8 changed files with 149 additions and 18 deletions.
8 changes: 8 additions & 0 deletions defaults/yacy.init
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,14 @@ proxyCache = DATA/HTCACHE
# default: 4 Gigabyte
proxyCacheSize = 4096

# The compression level for cached content
# Supported values ranging from 0 - no compression (lower CPU, higher disk usage), to 9 - best compression (higher CPU, lower disk use)
proxyCache.compressionLevel = 9

# Timeout value (in milliseconds) for acquiring a synchronization lock on getContent/store Cache operations
# When timeout occurs, loader should fall back to regular remote resource loading
proxyCache.sync.lockTimeout = 2000

# you can use the proxy with fresh/stale rules or in a always-fresh mode
proxyAlwaysFresh = false

Expand Down
20 changes: 17 additions & 3 deletions htroot/ConfigHTCache_p.html
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<!DOCTYPE html>
<html lang="en">
<head>
<title>YaCy '#[clientname]#': Hypertext Cache Configuration</title>
#%env/templates/metas.template%#
Expand All @@ -18,10 +18,24 @@ <h2>Hypertext Cache Configuration</h2>
<dl>
<dt><label for="HTCachePath">The path where the cache is stored</label></dt>
<dd><input name="HTCachePath" id="HTCachePath" type="text" size="20" maxlength="300" value="#[HTCachePath]#" /></dd>
<dt><label for="actualCacheSize">The current size of the cache</label></dt>
<dt><label>The current size of the cache</label></dt>
<dd><span id="actualCacheSize">#[actualCacheSize]# MB for #[actualCacheDocCount]# files, #[docSizeAverage]# KB / file in average </span></dd>
<dt><label for="maxCacheSize">The maximum size of the cache</label></dt>
<dd><input name="maxCacheSize" id="maxCacheSize" type="text" size="8" maxlength="24" value="#[maxCacheSize]#" /> MB</dd>
<dt><label for="compressionLevel">Compression level</label></dt>
<dd><select id="compressionLevel" name="compressionLevel">
#{compressionLevels}#
<option value="#[value]#" #(selected)#::selected="selected"#(/selected)#>#[name]#</option>
#{/compressionLevels}#
</select></dd>
<dt><label for="lockTimeout" aria-describedby="timeoutInfo">Concurrent access timeout</label>
<span class="info">
<img src="env/grafics/i16.gif" width="16" height="16" alt="Concurrent access timeout info"/>
<span class="infobox" id="timeoutInfo">The maximum time to wait for acquiring a synchronization lock on concurrent get/store cache operations.
Beyond this limit, the crawler or proxy falls back to regular remote resource loading.</span>
</span>
</dt>
<dd><input name="lockTimeout" id="lockTimeout" type="number" min="10" max="600000" value="#[lockTimeout]#"/> milliseconds</dd>
<dt>&nbsp;</dt>
<dd><input type="submit" name="set" value="Set" class="btn btn-primary"/></dd>
</dl>
Expand Down
40 changes: 40 additions & 0 deletions htroot/ConfigHTCache_p.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@

import java.io.File;
import java.io.IOException;
import java.util.zip.Deflater;

import net.yacy.cora.protocol.RequestHeader;
import net.yacy.crawler.data.Cache;
Expand Down Expand Up @@ -61,6 +62,19 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea
final int newProxyCacheSize = Math.max(post.getInt("maxCacheSize", 64), 0);
env.setConfig(SwitchboardConstants.PROXY_CACHE_SIZE, newProxyCacheSize);
Cache.setMaxCacheSize(newProxyCacheSize * 1024L * 1024L);

/* Compression level*/
/* Ensure a value within the range supported by the Deflater class */
final int newCompressionLevel = Math.max(Deflater.NO_COMPRESSION, Math.min(Deflater.BEST_COMPRESSION,
post.getInt("compressionLevel", SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL_DEFAULT)));
env.setConfig(SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL, newCompressionLevel);
Cache.setCompressionLevel(newCompressionLevel);

/* Synchronization lock timeout */
final long newLockTimeout = Math.max(10, Math.min(60000,
post.getLong("lockTimeout", SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT_DEFAULT)));
env.setConfig(SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT, newLockTimeout);
Cache.setLockTimeout(newLockTimeout);
}

if (post != null && post.containsKey("deletecomplete")) {
Expand All @@ -73,6 +87,32 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea
}

prop.put("HTCachePath", env.getConfig(SwitchboardConstants.HTCACHE_PATH, SwitchboardConstants.HTCACHE_PATH_DEFAULT));

/* Compression levels */
final int configuredCompressionLevel = env.getConfigInt(SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL,
SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL_DEFAULT);
int levelsCount = 0;
for(int level = Deflater.NO_COMPRESSION; level <= Deflater.BEST_COMPRESSION; level++) {
if(level == configuredCompressionLevel) {
prop.put("compressionLevels_" + levelsCount + "_selected", "1");
} else {
prop.put("compressionLevels_" + levelsCount + "_selected", "0");
}
prop.put("compressionLevels_" + levelsCount + "_value", level);
prop.put("compressionLevels_" + levelsCount + "_name", level);
if(level == Deflater.NO_COMPRESSION) {
prop.put("compressionLevels_" + levelsCount + "_name", "0 - No compression");
} else if(level == Deflater.BEST_SPEED) {
prop.put("compressionLevels_" + levelsCount + "_name", Deflater.BEST_SPEED + " - Best speed");
} else if(level == Deflater.BEST_COMPRESSION) {
prop.put("compressionLevels_" + levelsCount + "_name", Deflater.BEST_COMPRESSION + " - Best compression");
}
levelsCount++;
}
prop.put("compressionLevels", levelsCount);

prop.put("lockTimeout", env.getConfigLong(SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT,
SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT_DEFAULT));
prop.put("actualCacheSize", Cache.getActualCacheSize() / 1024 / 1024);
prop.put("actualCacheDocCount", Cache.getActualCacheDocCount());
prop.put("docSizeAverage", Cache.getActualCacheDocCount() == 0 ? 0 : Cache.getActualCacheSize() / Cache.getActualCacheDocCount() / 1024);
Expand Down
23 changes: 20 additions & 3 deletions source/net/yacy/crawler/data/Cache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,9 @@ public final class Cache {
* @param peerSalt peer identifier
* @param cacheSizeMax maximum cache size in bytes
* @param lockTimeout maximum time (in milliseconds) to acquire a synchronization lock on store() and getContent()
* @param compressionLevel the compression level : supported values ranging from 0 - no compression, to 9 - best compression
*/
public static void init(final File htCachePath, final String peerSalt, final long cacheSizeMax, final long lockTimeout) {
public static void init(final File htCachePath, final String peerSalt, final long cacheSizeMax, final long lockTimeout, final int compressionLevel) {

cachePath = htCachePath;
maxCacheSize = cacheSizeMax;
Expand Down Expand Up @@ -116,7 +117,7 @@ public static void init(final File htCachePath, final String peerSalt, final lon
try {
fileDBunbuffered = new ArrayStack(new File(cachePath, FILE_DB_NAME), prefix, Base64Order.enhancedCoder, 12, DEFAULT_BACKEND_BUFFER_SIZE, false, true);
fileDBunbuffered.setMaxSize(maxCacheSize);
fileDB = new Compressor(fileDBunbuffered, DEFAULT_COMPRESSOR_BUFFER_SIZE, lockTimeout);
fileDB = new Compressor(fileDBunbuffered, DEFAULT_COMPRESSOR_BUFFER_SIZE, lockTimeout, compressionLevel);
} catch (final IOException e) {
ConcurrentLog.logException(e);
// try a healing
Expand All @@ -125,7 +126,7 @@ public static void init(final File htCachePath, final String peerSalt, final lon
try {
fileDBunbuffered = new ArrayStack(new File(cachePath, FILE_DB_NAME), prefix, Base64Order.enhancedCoder, 12, DEFAULT_BACKEND_BUFFER_SIZE, false, true);
fileDBunbuffered.setMaxSize(maxCacheSize);
fileDB = new Compressor(fileDBunbuffered, DEFAULT_COMPRESSOR_BUFFER_SIZE, lockTimeout);
fileDB = new Compressor(fileDBunbuffered, DEFAULT_COMPRESSOR_BUFFER_SIZE, lockTimeout, compressionLevel);
} catch (final IOException ee) {
ConcurrentLog.logException(e);
}
Expand Down Expand Up @@ -226,6 +227,22 @@ public static long getActualCacheSize() {
public static long getActualCacheDocCount() {
return fileDBunbuffered.size();
}

/**
* Set the new content compression level
* @param newCompressionLevel the new compression level. Supported values between 0 (no compression) and 9 (best compression)
*/
public static void setCompressionLevel(final int newCompressionLevel) {
fileDB.setCompressionLevel(newCompressionLevel);
}

/**
* Set the new synchronization lock timeout.
* @param lockTimeout the new synchronization lock timeout (in milliseconds).
*/
public static void setLockTimeout(final long lockTimeout) {
fileDB.setLockTimeout(lockTimeout);
}

/**
* close the databases
Expand Down
48 changes: 39 additions & 9 deletions source/net/yacy/kelondro/blob/Compressor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,19 +61,32 @@ public class Compressor implements BLOB, Iterable<byte[]> {

/** Total size (in bytes) of uncompressed entries in buffer */
private long bufferlength;

/** Maximum {@link #bufferlength} value before compressing and flushing to the backend */
private final long maxbufferlength;

/** Maximum time (in milliseconds) to acquire a synchronization lock on get() and insert() */
private final long lockTimeout;
private long lockTimeout;

/** Synchronization lock */
private final ReentrantLock lock;

public Compressor(final BLOB backend, final long buffersize, final long lockTimeout) {

/** The compression level */
private int compressionLevel;

/**
* @param backend the backend storage
* @param buffersize the maximum total size (in bytes) of uncompressed in-memory entries before compressing and flushing to the backend
* @param lockTimeout maximum time to acquire a synchronization lock on get() and insert() operations
* @param compressionLevel the compression level : supported values ranging from 0 - no compression, to 9 - best compression
*/
public Compressor(final BLOB backend, final long buffersize, final long lockTimeout, final int compressionLevel) {
this.backend = backend;
this.maxbufferlength = buffersize;
this.lockTimeout = lockTimeout;
this.lock = new ReentrantLock();
/* Ensure a value within the range supported by the Deflater class */
this.compressionLevel = Math.max(Deflater.NO_COMPRESSION, Math.min(Deflater.BEST_COMPRESSION, compressionLevel));
initBuffer();
}

Expand Down Expand Up @@ -125,21 +138,21 @@ public void close(final boolean writeIDX) {
}
}

private static byte[] compress(final byte[] b) {
private static byte[] compress(final byte[] b, final int compressionLevel) {
final int l = b.length;
if (l < 100) return markWithPlainMagic(b);
final byte[] bb = compressAddMagic(b);
final byte[] bb = compressAddMagic(b, compressionLevel);
if (bb.length >= l) return markWithPlainMagic(b);
return bb;
}

private static byte[] compressAddMagic(final byte[] b) {
private static byte[] compressAddMagic(final byte[] b, final int compressionLevel) {
// compress a byte array and add a leading magic for the compression
try {
//System.out.print("/(" + cdr + ")"); // DEBUG
final ByteArrayOutputStream baos = new ByteArrayOutputStream(b.length / 5);
baos.write(gzipMagic);
final OutputStream os = new GZIPOutputStream(baos, 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}};
final OutputStream os = new GZIPOutputStream(baos, 65536){{def.setLevel(compressionLevel);}};
os.write(b);
os.close();
baos.close();
Expand Down Expand Up @@ -213,7 +226,7 @@ public byte[] get(final byte[] key) throws IOException, SpaceExceededException {
b = this.buffer.remove(key);
if (b != null) {
this.bufferlength = this.bufferlength - b.length;
this.backend.insert(key, compress(b));
this.backend.insert(key, compress(b, this.compressionLevel));
return b;
}
} finally {
Expand Down Expand Up @@ -412,7 +425,7 @@ private boolean flushOne() {
final Map.Entry<byte[], byte[]> entry = this.buffer.entrySet().iterator().next();
this.buffer.remove(entry.getKey());
try {
this.backend.insert(entry.getKey(), compress(entry.getValue()));
this.backend.insert(entry.getKey(), compress(entry.getValue(), this.compressionLevel));
this.bufferlength -= entry.getValue().length;
return true;
} catch (final IOException e) {
Expand Down Expand Up @@ -457,6 +470,23 @@ public int reduce(final byte[] key, final Reducer reducer) throws IOException, S
insert(key, c);
return reduction;
}

/**
* Set the new content compression level.
* @param compressionLevel the new compression level. Supported values between 0 (no compression) and 9 (best compression).
*/
public void setCompressionLevel(final int compressionLevel) {
/* Ensure a value within the range supported by the Deflater class */
this.compressionLevel = Math.max(Deflater.NO_COMPRESSION, Math.min(Deflater.BEST_COMPRESSION, compressionLevel));
}

/**
* Set the new synchronization lock timeout.
* @param lockTimeout the new synchronization lock timeout (in milliseconds).
*/
public void setLockTimeout(final long lockTimeout) {
this.lockTimeout = lockTimeout;
}


}
6 changes: 5 additions & 1 deletion source/net/yacy/search/Switchboard.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -723,7 +723,11 @@ public void run() {
this.log.info("HTCACHE Path = " + this.htCachePath.getAbsolutePath());
final long maxCacheSize =
1024L * 1024L * Long.parseLong(getConfig(SwitchboardConstants.PROXY_CACHE_SIZE, "2")); // this is megabyte
Cache.init(this.htCachePath, this.peers.mySeed().hash, maxCacheSize, 2000);
Cache.init(this.htCachePath, this.peers.mySeed().hash, maxCacheSize,
getConfigLong(SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT,
SwitchboardConstants.HTCACHE_SYNC_LOCK_TIMEOUT_DEFAULT),
getConfigInt(SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL,
SwitchboardConstants.HTCACHE_COMPRESSION_LEVEL_DEFAULT));
final File transactiondir = new File(this.htCachePath, "snapshots");
Transactions.init(transactiondir);

Expand Down
15 changes: 15 additions & 0 deletions source/net/yacy/search/SwitchboardConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@

package net.yacy.search;

import java.util.zip.Deflater;

import net.yacy.cora.order.Digest;
import net.yacy.server.http.RobotsTxtConfig;

Expand Down Expand Up @@ -390,6 +392,19 @@ public final class SwitchboardConstants {
*/
public static final String HTCACHE_PATH = "proxyCache";
public static final String HTCACHE_PATH_DEFAULT = "DATA/HTCACHE";

/** Key of the setting configuring the cache synchronization */
public static final String HTCACHE_COMPRESSION_LEVEL = "proxyCache.compressionLevel";

/** Default compression level for cached content */
public static final int HTCACHE_COMPRESSION_LEVEL_DEFAULT = Deflater.BEST_COMPRESSION;

/** Key of the setting configuring Cache synchronization lock timeout on getContent/store operations*/
public static final String HTCACHE_SYNC_LOCK_TIMEOUT = "proxyCache.sync.lockTimeout";

/** Default timeout value (in milliseconds) for acquiring a synchronization lock on getContent/store Cache operations */
public static final long HTCACHE_SYNC_LOCK_TIMEOUT_DEFAULT = 2000;

public static final String RELEASE_PATH = "releases";
public static final String RELEASE_PATH_DEFAULT = "DATA/RELEASE";

Expand Down
7 changes: 5 additions & 2 deletions test/java/net/yacy/crawler/data/CacheTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import java.util.Random;
import java.util.concurrent.TimeUnit;
import java.util.logging.LogManager;
import java.util.zip.Deflater;

import org.apache.http.HttpStatus;
import org.junit.After;
Expand Down Expand Up @@ -68,7 +69,7 @@ public void setUp() {
Cache.init(new File(System.getProperty("java.io.tmpdir") + File.separator + "testCache"), "peerSalt",
Math.max(Math.max(TEXT_CONTENT.getBytes(StandardCharsets.UTF_8).length * 10,
Cache.DEFAULT_COMPRESSOR_BUFFER_SIZE * 2), Cache.DEFAULT_BACKEND_BUFFER_SIZE * 2),
2000);
2000, Deflater.BEST_COMPRESSION);
Cache.clear();
}

Expand Down Expand Up @@ -497,9 +498,11 @@ public static void main(final String args[]) throws IOException {
final long sleepTime = 0;
/* Maximum waiting time (in ms) for acquiring a synchronization lock */
final long lockTimeout = 2000;
/* The backend compression level */
final int compressionLevel = Deflater.BEST_COMPRESSION;

Cache.init(new File(System.getProperty("java.io.tmpdir") + File.separator + "yacyTestCache"), "peerSalt",
cacheMaxSize, lockTimeout);
cacheMaxSize, lockTimeout, compressionLevel);
Cache.clear();
System.out.println("Cache initialized with a maximum size of " + cacheMaxSize + " bytes.");

Expand Down

0 comments on commit 28b451a

Please sign in to comment.