/
VariableHashEngine.java
106 lines (88 loc) · 3.38 KB
/
VariableHashEngine.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
/*******************************************************************************
* Copyright (C) 2016 Sam Silverberg sam.silverberg@gmail.com
*
* This file is part of OpenDedupe SDFS.
*
* OpenDedupe SDFS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 2 of the License, or
* (at your option) any later version.
*
* OpenDedupe SDFS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Foobar. If not, see <http://www.gnu.org/licenses/>.
*******************************************************************************/
package org.opendedup.hashing;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
import org.opendedup.logging.SDFSLogger;
import org.opendedup.sdfs.Main;
import org.rabinfingerprint.handprint.BoundaryDetectors;
import org.rabinfingerprint.handprint.BuffEnhancedFingerFactory;
import org.rabinfingerprint.handprint.FingerFactory.ChunkBoundaryDetector;
import org.rabinfingerprint.handprint.BuffEnhancedFingerFactory.EnhancedChunkVisitor;
import org.rabinfingerprint.polynomial.Polynomial;
public class VariableHashEngine implements AbstractHashEngine {
private int seed;
static Polynomial p = Polynomial.createFromLong(10923124345206883L);
ChunkBoundaryDetector boundaryDetector = BoundaryDetectors.DEFAULT_BOUNDARY_DETECTOR;
private BuffEnhancedFingerFactory ff = null;
public VariableHashEngine() throws NoSuchAlgorithmException {
this.seed = Main.hashSeed;
while (ff == null) {
SDFSLogger.getLog().info("Variable minLen=" +HashFunctionPool.minLen + " maxlen=" + HashFunctionPool.maxLen + " windowSize=" + HashFunctionPool.bytesPerWindow);
ff = new BuffEnhancedFingerFactory(p, HashFunctionPool.bytesPerWindow, boundaryDetector,
HashFunctionPool.minLen, HashFunctionPool.maxLen);
}
}
@Override
public byte[] getHash(byte[] data) {
byte[] hash = MurMurHash3.murmurhash3_x64_128(data, seed);
return hash;
}
public List<Finger> getChunks(ByteBuffer b,String lookupFilter,String uuid) throws IOException {
final ArrayList<Finger> al = new ArrayList<Finger>();
ff.getChunkFingerprints(b, new EnhancedChunkVisitor() {
public void visit(long fingerprint, long chunkStart, long chunkEnd,
byte[] chunk) {
byte[] hash = getHash(chunk);
Finger f = new Finger(lookupFilter,uuid);
f.chunk = chunk;
f.hash = hash;
f.len = (int) (chunkEnd - chunkStart);
f.start = (int) chunkStart;
al.add(f);
}
});
return al;
}
@Override
public void destroy() {
}
@Override
public boolean isVariableLength() {
// TODO Auto-generated method stub
return true;
}
@Override
public int getMaxLen() {
// TODO Auto-generated method stub
return Main.CHUNK_LENGTH;
}
@Override
public int getMinLen() {
// TODO Auto-generated method stub
return HashFunctionPool.minLen;
}
@Override
public void setSeed(int seed) {
this.seed = seed;
}
}