Skip to content

Commit

Permalink
feat: optimize degree + minor refactorings + update as-pect (#12)
Browse files Browse the repository at this point in the history
* optimize degree + minor refactorings + update as-pect

* var -> let

* updates deps + refactor tests

* simplify

* remove unnecessary info
  • Loading branch information
MaxGraey committed Apr 22, 2020
1 parent 8451abf commit 574afad
Show file tree
Hide file tree
Showing 6 changed files with 4,863 additions and 4,073 deletions.
33 changes: 17 additions & 16 deletions assembly/__tests__/example.spec.ts
Original file line number Diff line number Diff line change
@@ -1,30 +1,31 @@
import { degree, mod, Rabin } from '../index'
import { degree, mod, Rabin } from '../index';

NativeMath.seedRandom(2777792873);

NativeMath.seedRandom(2777792873)
function getRandomArbitrary(min: f64, max: f64): f64 {
return NativeMath.random() * (max - min) + min;
}
describe("rabin degree", (): void => {
it("degree should be truthy", (): void => {
expect<u64>(degree(0)).toBe(-1, "42 is the meaning of life.");

describe("rabin degree", () => {
it("degree should be truthy", () => {
expect(degree(0)).toBe(-1);
});

it("mod should be truthy", (): void => {
expect<u64>(mod((<u64>0) << 0x3DA3358B4DC173, 0x3DA3358B4DC173)).toBe(0, "42 is the meaning of life.");
it("mod should be truthy", () => {
expect(mod((<u64>0) << 0x3DA3358B4DC173, 0x3DA3358B4DC173)).toBe(0);
});

it("fingerprint", (): void => {
let r = new Rabin(124, 1 * 8, 2 * 8, 64)
it("fingerprint", () => {
let r = new Rabin(124, 1 * 8, 2 * 8, 64);
let file = new Uint8Array(1024);
for (let i = 0; i < file.length; i++) {
file[i] = <u32>getRandomArbitrary(128, 8888)
file[i] = <u32>getRandomArbitrary(128, 8888);
}
const out = r.fingerprint(file, new Int32Array(file.length/128))
const out = r.fingerprint(file, new Int32Array(file.length / 128));

log(out)
const expected = new Int32Array(1)
expected[0] = 16
expect(out[0]).toBe(expected[0])
log(out);
const expected = new Int32Array(1);
expected[0] = 16;
expect(out[0]).toBe(expected[0]);
});

});
105 changes: 48 additions & 57 deletions assembly/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,28 +5,20 @@ export const Uint8Array_ID = idof<Uint8Array>();
const POLYNOMIAL_DEGREE = 53;
const POLYNOMIAL_SHIFT = POLYNOMIAL_DEGREE - 8;

let tables_initialized = false
let tables_initialized = false;

const modTable = new Uint64Array(256)
const outTable = new Uint64Array(256)
const modTable = new Uint64Array(256);
const outTable = new Uint64Array(256);


@inline
export function degree(polynom: u64): i32 {
var mask: u64 = 0x8000000000000000;

for (let i = 0; i < 64; i++) {
if ((mask & polynom) > 0) {
return 63 - i;
}
mask >>= 1;
}
return -1;
return 63 - <i32>clz(polynom);
}

@inline
export function mod(x: u64, p: u64): u64 {
var shift: i32;
let shift: i32;
let dp = degree(p);
while ((shift = degree(x) - dp) >= 0) {
x ^= p << shift;
Expand All @@ -45,42 +37,42 @@ function append_byte(hash: u64, b: u8, pol: u64): u64 {
@inline
function calc_tables(h: Rabin): void {
for (let b = 0; b < 256; b++) {
let hash: u64 = 0;
let hash: u64 = 0;

hash = append_byte(hash, <u8>b, h.polynomial);
for (let i = 0; i < h.window_size-1; i++) {
hash = append_byte(hash, 0, h.polynomial);
}
unchecked(outTable[b] = hash);
hash = append_byte(hash, <u8>b, h.polynomial);
for (let i = 0; i < h.window_size-1; i++) {
hash = append_byte(hash, 0, h.polynomial);
}
unchecked(outTable[b] = hash);
}

var k = <u64>degree(h.polynomial);
for (var b = 0; b < 256; b++) {
const bk = (<u64>b) << k;
unchecked(modTable[b] = mod(bk, h.polynomial) | bk);
let k = <u64>degree(h.polynomial);
for (let b = 0; b < 256; b++) {
const bk = (<u64>b) << k;
unchecked(modTable[b] = mod(bk, h.polynomial) | bk);
}
}

@inline
function rabin_append(h: Rabin, b: usize): void {
var digest = h.digest
var index = <u8>(digest >> POLYNOMIAL_SHIFT)
function rabin_append(h: Rabin, b: u8): void {
let digest = h.digest;
let index = i32(digest >> POLYNOMIAL_SHIFT);

h.digest = ((digest << 8) | <u64>b) ^ unchecked(modTable[index]);
}

@inline
function rabin_slide(h: Rabin, b: usize): void {
var out: u8 = h.window[h.wpos];
h.window[h.wpos] = b;
function rabin_slide(h: Rabin, b: u8): void {
let out = h.window[h.wpos];
unchecked(h.window[h.wpos] = b);
h.digest ^= unchecked(outTable[out]);
h.wpos = (h.wpos + 1) % h.window_size;
rabin_append(h, b);
}

@inline
function rabin_reset(h: Rabin): void {
for (let i = 0; i < h.window_size; i++){
for (let i = 0; i < h.window_size; i++) {
h.window[i] = 0;
}
h.digest = 0;
Expand All @@ -94,20 +86,20 @@ function rabin_reset(h: Rabin): void {
@inline
function rabin_next_chunk(h: Rabin, buf: usize, len: i32): i32 {
for (let i = 0; i < len; i++) {
let b = load<u8>(buf + i)
let b = load<u8>(buf + i);

rabin_slide(h, b);
rabin_slide(h, b);

h.count++;
h.pos++;
if ((h.count >= h.minsize && ((h.digest & h.mask) == 0)) || h.count >= h.maxsize) {
h.chunk_start = h.start;
h.chunk_length = h.count;
h.chunk_cut_fingerprint = h.digest;
h.count++;
h.pos++;
if ((h.count >= h.minsize && ((h.digest & h.mask) == 0)) || h.count >= h.maxsize) {
h.chunk_start = h.start;
h.chunk_length = h.count;
h.chunk_cut_fingerprint = h.digest;

rabin_reset(h);
return i+1;
}
rabin_reset(h);
return i + 1;
}
}

return -1;
Expand All @@ -116,8 +108,8 @@ function rabin_next_chunk(h: Rabin, buf: usize, len: i32): i32 {
@inline
function rabin_init(h: Rabin): Rabin {
if (!tables_initialized) {
calc_tables(h);
tables_initialized = true;
calc_tables(h);
tables_initialized = true;
}

h.pos = 0;
Expand All @@ -144,32 +136,31 @@ export class Rabin {
mask: u64

constructor(average_bits: u32, minsize: u32, maxsize: u32, window_size: i32) {
this.minsize = <u64>minsize
this.maxsize = <u64>maxsize
this.window = new Uint8Array(window_size)
this.window_size = window_size
this.mask = (1 << average_bits) - 1
this.polynomial = 0x3DA3358B4DC173

rabin_init(this)
this.minsize = <u64>minsize;
this.maxsize = <u64>maxsize;
this.window = new Uint8Array(window_size);
this.window_size = window_size;
this.mask = (1 << average_bits) - 1;
this.polynomial = 0x3DA3358B4DC173;

rabin_init(this);
}

fingerprint(buf: Uint8Array, lengths: Int32Array): Int32Array {
let idx = 0;
let len = buf.length;
let ptr = buf.dataStart
let ptr = buf.dataStart;
while (true) {
var remaining = rabin_next_chunk(this, ptr, len);
let remaining = rabin_next_chunk(this, ptr, len);
if (remaining < 0) {
break;
}

len -= remaining;
ptr += remaining;
let c = idx++
let c = idx++;
// lengths.push(<i32>this.chunk_length)
unchecked(lengths[c] = <i32>this.chunk_length)
unchecked(lengths[c] = <i32>this.chunk_length);
}
return lengths
return lengths;
}
}
Loading

0 comments on commit 574afad

Please sign in to comment.