Permalink
Browse files

updates for ByteArray pool

  • Loading branch information...
opendedup committed Apr 8, 2018
1 parent ac96229 commit 7eb57a1aee82411e5cc16efa11c2572746b39f45
@@ -99,7 +99,7 @@ Section "SDFS Setup" SecMain
File /r bin\*
SetOutPath "$INSTDIR\lib"
File ..\..\target\sdfs-${VERSION}-jar-with-dependencies.jar
File ..\..\lib\b2-2.0.3.jar
File ..\..\target\lib\b2-2.0.3.jar
SetOutPath "$INSTDIR\etc"
File etc\*
;Store installation folder
@@ -157,7 +157,7 @@ FunctionEnd
;Language strings
LangString DESC_SecMain ${LANG_ENGLISH} "SDFS Volume Binaries Setup."
LangString DESC_SecDokan ${LANG_ENGLISH} "Dokan Windows FileSystem Driver."
;Assign language strings to sections
;Assign language strings to sections
!insertmacro MUI_FUNCTION_DESCRIPTION_BEGIN
!insertmacro MUI_DESCRIPTION_TEXT ${SecMain} $(DESC_SecMain)
!insertmacro MUI_DESCRIPTION_TEXT ${SecDokan} $(DESC_SecDokan)
View
@@ -111,7 +111,7 @@
<dependency>
<groupId>org.opendedupe</groupId>
<artifactId>rabinwindow</artifactId>
<version>1.0.1</version>
<version>1.0.2</version>
</dependency>
<dependency>
<groupId>com.datastax.cassandra</groupId>
@@ -166,7 +166,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>18.0</version>
<version>24.1-jre</version>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
@@ -537,7 +537,7 @@ public int readlink(String path, CharBuffer link) throws FuseException {
try {
String lpath = Files.readSymbolicLink(p).toString();
if (lpath.startsWith(this.mountedVolume))
if (new File(lpath).getPath().startsWith(this.mountedVolume))
lpath = this.mountPoint + lpath.substring(this.mountedVolume.length());
// SDFSLogger.getLog().info("path=" + path + " lpath=" + lpath);
link.put(lpath);
@@ -85,7 +85,7 @@
private transient ThreadPoolExecutor executor = null;
private List<String> colFamily = new ArrayList<String>();
FlushOptions flo = null;
private ConcurrentHashMap<ByteArrayWrapper, ByteBuffer> tempHt = new ConcurrentHashMap<ByteArrayWrapper, ByteBuffer>();
private ConcurrentHashMap<ByteArrayWrapper, ByteBuffer> tempHt = new ConcurrentHashMap<ByteArrayWrapper, ByteBuffer>(1024,0.75f,Main.writeThreads);
static boolean windowsLegacy = false;
static {
if (org.apache.commons.lang3.SystemUtils.IS_OS_WINDOWS) {
@@ -671,7 +671,7 @@ public InsertRecord put(ChunkData cm, boolean persist) throws IOException, Hasht
else
bf.putLong(cm.references);
this.tempHt.put(new ByteArrayWrapper(cm.getHash()), bf);
this.rmdb.delete(cm.getHash());
//this.rmdb.delete(cm.getHash());
return new InsertRecord(true, cm.getcPos());
} else {
// SDFSLogger.getLog().info("Hash Found");
@@ -28,6 +28,9 @@
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.BitSet;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -86,6 +89,7 @@ public String getPath() {
}
private ReentrantLock iterlock = new ReentrantLock();
private Iterator<Entry<ByteArrayWrapper, Long>> iter = null;
/*
* (non-Javadoc)
@@ -97,6 +101,9 @@ public void iterInit() {
this.iterlock.lock();
this.iterPos = 0;
this.iterlock.unlock();
if(bw != null && bw.size() > 0) {
iter = bw.entrySet().iterator();
}
}
/*
@@ -122,6 +129,14 @@ public int getCurrentSize() {
*/
@Override
public KeyValuePair next() throws IOException, MapClosedException {
if(iter != null) {
KeyValuePair kv = null;
if(iter.hasNext()) {
Entry<ByteArrayWrapper, Long> e = iter.next();
kv = new KeyValuePair(e.getKey().getData(),e.getValue());
}
return kv;
}
while (iterPos < this.kFC.size()) {
Lock l = this.hashlock.writeLock();
l.lock();
@@ -436,6 +451,7 @@ private int insertKeyRehash(byte[] key, int index, int hash, byte[] cur) throws
}
ByteBuffer vb = null;
HashMap<ByteArrayWrapper,Long> bw = new HashMap<ByteArrayWrapper,Long>();
/*
* (non-Javadoc)
@@ -464,6 +480,9 @@ public boolean put(byte[] key, long value) throws MapClosedException {
} else {
vb.putLong(value);
}
if(bw != null) {
bw.put(new ByteArrayWrapper(key), value);
}
vb.position(0);
this.kFC.write(vb, pos + offset);
vb.position(0);
@@ -548,6 +567,9 @@ public void close() {
} catch (Exception e) {
}
if(bw != null)
bw.clear();
bw = null;
this.mapped = null;
} finally {
l.unlock();
@@ -31,7 +31,7 @@ public void execute(JobExecutionContext arg0) throws JobExecutionException {
try {
new SyncFS();
} catch (Exception e) {
SDFSLogger.getLog().error("SyncFS Job Failed", e);
SDFSLogger.getLog().info("SyncFS Job Failed", e);
}
} catch (Exception e) {
SDFSLogger.getLog().warn("unable to finish executing fdisk", e);
@@ -19,6 +19,7 @@
package org.opendedup.hashing;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
public interface AbstractHashEngine {
@@ -35,5 +36,5 @@
public int getMinLen();
public List<Finger> getChunks(byte[] data,String lookupFilter,String uuid) throws IOException;
public List<Finger> getChunks(ByteBuffer byteBuffer,String lookupFilter,String uuid) throws IOException;
}
@@ -19,6 +19,7 @@
package org.opendedup.hashing;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
import org.opendedup.sdfs.Main;
@@ -78,7 +79,7 @@ public void setSeed(int seed) {
}
@Override
public List<Finger> getChunks(byte[] data,String fp,String uuid) throws IOException {
public List<Finger> getChunks(ByteBuffer b,String fp,String uuid) throws IOException {
throw new IOException("not supported");
}
}
@@ -19,7 +19,7 @@
package org.opendedup.hashing;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.security.NoSuchAlgorithmException;
import java.util.List;
@@ -80,7 +80,7 @@ public void setSeed(int seed) {
}
@Override
public List<Finger> getChunks(byte[] data,String lp,String uuid) throws IOException {
public List<Finger> getChunks(ByteBuffer b,String lp,String uuid) throws IOException {
throw new IOException("not supported");
}
}
@@ -19,17 +19,17 @@
package org.opendedup.hashing;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
import org.opendedup.logging.SDFSLogger;
import org.opendedup.sdfs.Main;
import org.rabinfingerprint.handprint.BoundaryDetectors;
import org.rabinfingerprint.handprint.BuffEnhancedFingerFactory;
import org.rabinfingerprint.handprint.FingerFactory.ChunkBoundaryDetector;
import org.rabinfingerprint.handprint.EnhancedFingerFactory;
import org.rabinfingerprint.handprint.EnhancedFingerFactory.EnhancedChunkVisitor;
import org.rabinfingerprint.handprint.BuffEnhancedFingerFactory.EnhancedChunkVisitor;
import org.rabinfingerprint.polynomial.Polynomial;
public class VariableHashEngine implements AbstractHashEngine {
@@ -38,13 +38,13 @@
static Polynomial p = Polynomial.createFromLong(10923124345206883L);
ChunkBoundaryDetector boundaryDetector = BoundaryDetectors.DEFAULT_BOUNDARY_DETECTOR;
private EnhancedFingerFactory ff = null;
private BuffEnhancedFingerFactory ff = null;
public VariableHashEngine() throws NoSuchAlgorithmException {
this.seed = Main.hashSeed;
while (ff == null) {
SDFSLogger.getLog().info("Variable minLen=" +HashFunctionPool.minLen + " maxlen=" + HashFunctionPool.maxLen + " windowSize=" + HashFunctionPool.bytesPerWindow);
ff = new EnhancedFingerFactory(p, HashFunctionPool.bytesPerWindow, boundaryDetector,
ff = new BuffEnhancedFingerFactory(p, HashFunctionPool.bytesPerWindow, boundaryDetector,
HashFunctionPool.minLen, HashFunctionPool.maxLen);
}
@@ -58,9 +58,9 @@ public VariableHashEngine() throws NoSuchAlgorithmException {
public List<Finger> getChunks(byte[] data,String lookupFilter,String uuid) throws IOException {
public List<Finger> getChunks(ByteBuffer b,String lookupFilter,String uuid) throws IOException {
final ArrayList<Finger> al = new ArrayList<Finger>();
ff.getChunkFingerprints(data, new EnhancedChunkVisitor() {
ff.getChunkFingerprints(b, new EnhancedChunkVisitor() {
public void visit(long fingerprint, long chunkStart, long chunkEnd,
byte[] chunk) {
byte[] hash = getHash(chunk);
@@ -19,6 +19,7 @@
package org.opendedup.hashing;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
@@ -28,8 +29,8 @@
import org.opendedup.sdfs.Main;
import org.rabinfingerprint.handprint.BoundaryDetectors;
import org.rabinfingerprint.handprint.FingerFactory.ChunkBoundaryDetector;
import org.rabinfingerprint.handprint.EnhancedFingerFactory;
import org.rabinfingerprint.handprint.EnhancedFingerFactory.EnhancedChunkVisitor;
import org.rabinfingerprint.handprint.BuffEnhancedFingerFactory;
import org.rabinfingerprint.handprint.BuffEnhancedFingerFactory.EnhancedChunkVisitor;
import org.rabinfingerprint.polynomial.Polynomial;
import org.opendedup.rabin.hashing.HighwayHash;
@@ -39,15 +40,15 @@
static Polynomial p = Polynomial.createFromLong(10923124345206883L);
ChunkBoundaryDetector boundaryDetector = BoundaryDetectors.DEFAULT_BOUNDARY_DETECTOR;
private EnhancedFingerFactory ff = null;
private BuffEnhancedFingerFactory ff = null;
public static enum HASHTYPE {HASH128,HASH160,HASH256};
private HASHTYPE ht;
public VariableHighwayHashEngine(HASHTYPE ht) throws NoSuchAlgorithmException {
this.ht=ht;
while (ff == null) {
SDFSLogger.getLog().info("Variable minLen=" +HashFunctionPool.minLen + " maxlen=" + HashFunctionPool.maxLen + " windowSize=" + HashFunctionPool.bytesPerWindow);
ff = new EnhancedFingerFactory(p, HashFunctionPool.bytesPerWindow, boundaryDetector,
ff = new BuffEnhancedFingerFactory(p, HashFunctionPool.bytesPerWindow, boundaryDetector,
HashFunctionPool.minLen, HashFunctionPool.maxLen);
}
@@ -78,7 +79,7 @@ public VariableHighwayHashEngine(HASHTYPE ht) throws NoSuchAlgorithmException {
public List<Finger> getChunks(byte[] data,String lookupFilter,String uuid) throws IOException {
public List<Finger> getChunks(ByteBuffer data,String lookupFilter,String uuid) throws IOException {
final ArrayList<Finger> al = new ArrayList<Finger>();
ff.getChunkFingerprints(data, new EnhancedChunkVisitor() {
public void visit(long fingerprint, long chunkStart, long chunkEnd,
@@ -19,8 +19,7 @@
package org.opendedup.hashing;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
@@ -29,8 +28,8 @@
import org.opendedup.sdfs.Main;
import org.rabinfingerprint.handprint.BoundaryDetectors;
import org.rabinfingerprint.handprint.FingerFactory.ChunkBoundaryDetector;
import org.rabinfingerprint.handprint.EnhancedFingerFactory;
import org.rabinfingerprint.handprint.EnhancedFingerFactory.EnhancedChunkVisitor;
import org.rabinfingerprint.handprint.BuffEnhancedFingerFactory;
import org.rabinfingerprint.handprint.BuffEnhancedFingerFactory.EnhancedChunkVisitor;
import org.rabinfingerprint.polynomial.Polynomial;
import com.google.common.hash.HashFunction;
@@ -41,13 +40,13 @@
static Polynomial p = Polynomial.createFromLong(10923124345206883L);
ChunkBoundaryDetector boundaryDetector = BoundaryDetectors.DEFAULT_BOUNDARY_DETECTOR;
private EnhancedFingerFactory ff = null;
private BuffEnhancedFingerFactory ff = null;
HashFunction hf = Hashing.md5();
public VariableMD5HashEngine() throws NoSuchAlgorithmException {
while (ff == null) {
SDFSLogger.getLog().info("Variable minLen=" +HashFunctionPool.minLen + " maxlen=" + HashFunctionPool.maxLen + " windowSize=" + HashFunctionPool.bytesPerWindow);
ff = new EnhancedFingerFactory(p, HashFunctionPool.bytesPerWindow, boundaryDetector,
ff = new BuffEnhancedFingerFactory(p, HashFunctionPool.bytesPerWindow, boundaryDetector,
HashFunctionPool.minLen, HashFunctionPool.maxLen);
}
@@ -61,7 +60,7 @@ public VariableMD5HashEngine() throws NoSuchAlgorithmException {
public List<Finger> getChunks(byte[] data,String lookupFilter,String uuid) throws IOException {
public List<Finger> getChunks(ByteBuffer data,String lookupFilter,String uuid) throws IOException {
final ArrayList<Finger> al = new ArrayList<Finger>();
ff.getChunkFingerprints(data, new EnhancedChunkVisitor() {
public void visit(long fingerprint, long chunkStart, long chunkEnd,
@@ -19,6 +19,7 @@
package org.opendedup.hashing;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
@@ -28,8 +29,8 @@
import org.opendedup.sdfs.Main;
import org.rabinfingerprint.handprint.BoundaryDetectors;
import org.rabinfingerprint.handprint.FingerFactory.ChunkBoundaryDetector;
import org.rabinfingerprint.handprint.EnhancedFingerFactory;
import org.rabinfingerprint.handprint.EnhancedFingerFactory.EnhancedChunkVisitor;
import org.rabinfingerprint.handprint.BuffEnhancedFingerFactory;
import org.rabinfingerprint.handprint.BuffEnhancedFingerFactory.EnhancedChunkVisitor;
import org.rabinfingerprint.polynomial.Polynomial;
import com.google.common.hash.HashFunction;
@@ -40,14 +41,14 @@
static Polynomial p = Polynomial.createFromLong(10923124345206883L);
ChunkBoundaryDetector boundaryDetector = BoundaryDetectors.DEFAULT_BOUNDARY_DETECTOR;
public static enum HASHTYPE {HASH160,HASH256};
private EnhancedFingerFactory ff = null;
private BuffEnhancedFingerFactory ff = null;
HashFunction hf = Hashing.sha256();
private final HASHTYPE ht;
public VariableSha256HashEngine(HASHTYPE ht) throws NoSuchAlgorithmException {
this.ht = ht;
while (ff == null) {
SDFSLogger.getLog().info("Variable minLen=" +HashFunctionPool.minLen + " maxlen=" + HashFunctionPool.maxLen + " windowSize=" + HashFunctionPool.bytesPerWindow);
ff = new EnhancedFingerFactory(p, HashFunctionPool.bytesPerWindow, boundaryDetector,
ff = new BuffEnhancedFingerFactory(p, HashFunctionPool.bytesPerWindow, boundaryDetector,
HashFunctionPool.minLen, HashFunctionPool.maxLen);
}
@@ -67,7 +68,7 @@ public VariableSha256HashEngine(HASHTYPE ht) throws NoSuchAlgorithmException {
public List<Finger> getChunks(byte[] data,String lookupFilter,String uuid) throws IOException {
public List<Finger> getChunks(ByteBuffer data,String lookupFilter,String uuid) throws IOException {
final ArrayList<Finger> al = new ArrayList<Finger>();
ff.getChunkFingerprints(data, new EnhancedChunkVisitor() {
public void visit(long fingerprint, long chunkStart, long chunkEnd,
Oops, something went wrong.

0 comments on commit 7eb57a1

Please sign in to comment.