Permalink
Browse files

add/use mmapf

  • Loading branch information...
ryancdotorg committed Oct 17, 2015
1 parent c9806ce commit 5aade72d8c1f38193c7fd48674447bc325835a62
Showing with 228 additions and 88 deletions.
  1. +5 −5 Makefile
  2. +10 −2 blfchk.c
  3. +1 −50 bloom.c
  4. +0 −2 bloom.h
  5. +14 −2 brainflayer.c
  6. +15 −27 ec_pubkey_fast.c
  7. +138 −0 mmapf.c
  8. +45 −0 mmapf.h
View
@@ -1,5 +1,5 @@
HEADERS = bloom.h crack.h hash160.h warpwallet.h
OBJECTS = brainflayer.o bloom.o hex2blf.o warpwallet.o hex.o
OBJECTS = brainflayer.o bloom.o hex2blf.o warpwallet.o hex.o mmapf.o
BINARIES = brainflayer hex2blf blfchk ecmtabgen
LIBS = -lssl -lrt -lcrypto -lz -ldl -lgmp
CFLAGS = -O3 -flto -pedantic -std=gnu99 -Wall -Wextra -funsigned-char -Wno-pointer-sign -Wno-sign-compare
@@ -36,16 +36,16 @@ ec_pubkey_fast.o: ec_pubkey_fast.c secp256k1/include/secp256k1.h
%.o: %.c
$(COMPILE) -c $< -o $@
blfchk: blfchk.o hex.o bloom.o
blfchk: blfchk.o hex.o bloom.o mmapf.o
$(COMPILE) -static $^ $(LIBS) -o $@
hex2blf: hex2blf.o hex.o bloom.o
hex2blf: hex2blf.o hex.o bloom.o mmapf.o
$(COMPILE) -static $^ $(LIBS) -o $@
ecmtabgen: ecmtabgen.o ec_pubkey_fast.o secp256k1/.libs/libsecp256k1.a
ecmtabgen: ecmtabgen.o mmapf.o ec_pubkey_fast.o secp256k1/.libs/libsecp256k1.a
$(COMPILE) -static $^ $(LIBS) -o $@
brainflayer: brainflayer.o hex.o bloom.o warpwallet.o brainwalletio.o brainv2.o ec_pubkey_fast.o secp256k1/.libs/libsecp256k1.a scrypt-jane/scrypt-jane.o
brainflayer: brainflayer.o hex.o bloom.o mmapf.o warpwallet.o brainwalletio.o brainv2.o ec_pubkey_fast.o secp256k1/.libs/libsecp256k1.a scrypt-jane/scrypt-jane.o
$(COMPILE) -static $^ $(LIBS) -o $@
clean:
View
@@ -18,15 +18,18 @@
#include "hex.h"
#include "bloom.h"
#include "mmapf.h"
#include "hash160.h"
int main(int argc, char **argv) {
int ret;
hash160_t hash;
char *line = NULL;
size_t line_sz = 0;
unsigned char buf[128];
unsigned char *bloom, *bloomfile;
FILE *ifile = stdin, *ofile = stdout;
mmapf_ctx bloom_mmapf;
if (argc != 2) {
fprintf(stderr, "Usage: %s BLOOM_FILTER_FILE\n", argv[0]);
@@ -35,11 +38,16 @@ int main(int argc, char **argv) {
bloomfile = argv[1];
if ((bloom = bloom_open(bloomfile)) == NULL) {
fprintf(stderr, "failed to open bloom filter.\n");
if ((ret = mmapf(&bloom_mmapf, bloomfile, BLOOM_SIZE, MMAPF_RNDRD)) != MMAPF_OKAY) {
fprintf(stderr, "failed to open bloom filter '%s': %s\n", bloomfile, mmapf_strerror(ret));
return 1;
} else if (bloom_mmapf.mem == NULL) {
fprintf(stderr, "got NULL pointer trying to set up bloom filter\n");
return 1;
}
bloom = bloom_mmapf.mem;
while (getline(&line, &line_sz, ifile) > 0) {
unhex(line, strlen(line), hash.uc, sizeof(hash.uc));
if (bloom_chk_hash160(bloom, hash.ul)) {
View
51 bloom.c
@@ -10,6 +10,7 @@
#include <sys/mman.h>
#include "bloom.h"
#include "mmapf.h"
void bloom_set_hash160(unsigned char *bloom, uint32_t *h) {
unsigned int t;
@@ -35,56 +36,6 @@ void bloom_set_hash160(unsigned char *bloom, uint32_t *h) {
t = BH19(h); BLOOM_SET_BIT(t);
}
int _bloom_mmap(unsigned char **bloom, unsigned char *filename) {
int ret, fd, i;
struct stat sb;
unsigned char z[1024*1024];
if (stat(filename, &sb) == 0) {
if (!S_ISREG(sb.st_mode) || sb.st_size != BLOOM_SIZE)
return -100;
if ((fd = open(filename, O_RDWR)) < 0)
return fd;
} else {
/* Assume the file didn't exist */
if ((fd = open(filename, O_RDWR | O_CREAT | O_EXCL, 0660)) < 0)
return fd;
/* Make sure the buffer is zeroed */
memset(z, 0, sizeof(z));
i = BLOOM_SIZE;
/* pre-write an empty bloom filter */
while (i > 0) {
if (i > sizeof(z)) {
ret = write(fd, z, sizeof(z));
} else {
ret = write(fd, z, i);
}
//fprintf(stderr, "bloom init write %2d %9d %9d\n", fd, ret, i);
/* We shouldn't be getting short writes */
if (ret != sizeof(z))
return -101;
i -= ret;
}
}
if ((ret = posix_fadvise(fd, 0, BLOOM_SIZE, POSIX_FADV_WILLNEED|POSIX_FADV_RANDOM)) < 0)
return ret;
/* We should now have a file of the right size open. */
*bloom = mmap(NULL, BLOOM_SIZE, PROT_READ, MAP_SHARED|MAP_NORESERVE|MAP_POPULATE, fd, 0);
return fd;
}
unsigned char * bloom_open(unsigned char *filename) {
int fd;
unsigned char *bloom;
bloom = malloc(sizeof(void *));
if ((fd = _bloom_mmap(&bloom, filename) < 0))
return NULL;
return bloom;
}
/*
int bloom_save(unsigned char *filename, unsigned char *bloom);
*/
View
@@ -60,8 +60,6 @@ inline unsigned int bloom_chk_hash160(unsigned char *bloom, uint32_t *h) {
return 1;
}
unsigned char * bloom_open(unsigned char *);
//#define bloom_new() malloc(536870912)
//unsigned char * bloom_load(unsigned char *);
//int bloom_save(unsigned char *, unsigned char *);
View
@@ -27,6 +27,7 @@
#include "hex.h"
#include "bloom.h"
#include "mmapf.h"
#include "hash160.h"
#include "brainv2.h"
@@ -42,6 +43,7 @@ static hash160_t hash160_compr;
static hash160_t hash160_uncmp;
static unsigned char *mem;
static mmapf_ctx bloom_mmapf;
static unsigned char *bloom = NULL;
static unsigned char hexed[4096], unhexed[4096];
@@ -273,6 +275,8 @@ int main(int argc, char **argv) {
FILE *ifile = stdin;
FILE *ofile = stdout;
int ret;
float alpha, ilines_rate, ilines_rate_avg;
uint64_t report_mask = 0;
uint64_t time_last, time_curr, time_delta;
@@ -413,8 +417,16 @@ int main(int argc, char **argv) {
}
}
if (bopt && (bloom = bloom_open(bopt)) == NULL) {
bail(1, "failed to open bloom filter.\n");
if (bopt) {
if (Lopt) {
bail(1, "The '-L' option cannot be used with a bloom filter\n");
}
if ((ret = mmapf(&bloom_mmapf, bopt, BLOOM_SIZE, MMAPF_RNDRD)) != MMAPF_OKAY) {
bail(1, "failed to open bloom filter '%s': %s\n", bopt, mmapf_strerror(ret));
} else if (bloom_mmapf.mem == NULL) {
bail(1, "got NULL pointer trying to set up bloom filter\n");
}
bloom = bloom_mmapf.mem;
}
if (iopt && (ifile = fopen(iopt, "r")) == NULL) {
View
@@ -23,6 +23,8 @@
#include "secp256k1/src/ecmult_gen_impl.h"
#include "secp256k1/src/ecmult.h"
#include "mmapf.h"
#undef ASSERT
#define READBIT(A, B) ((A >> (B & 7)) & 1)
@@ -35,6 +37,7 @@ secp256k1_ge_t *prec;
int remmining = 0;
int WINDOW_SIZE = 0;
size_t MMAP_SIZE;
mmapf_ctx prec_mmapf;
int secp256k1_ec_pubkey_precomp_table_save(int window_size, unsigned char *filename) {
int fd, ret;
@@ -56,9 +59,9 @@ int secp256k1_ec_pubkey_precomp_table_save(int window_size, unsigned char *filen
}
int secp256k1_ec_pubkey_precomp_table(int window_size, unsigned char *filename) {
int ret, fd;
int ret;
struct stat sb;
size_t prec_sz, page_sz, mmap_sz;
size_t prec_sz;
secp256k1_gej_t gj; // base point in jacobian coordinates
secp256k1_gej_t *table;
@@ -71,6 +74,7 @@ int secp256k1_ec_pubkey_precomp_table(int window_size, unsigned char *filename)
}
}
// try to find a window size that matched the file size
for (;;) {
WINDOW_SIZE = window_size;
n_values = 1 << window_size;
@@ -86,32 +90,16 @@ int secp256k1_ec_pubkey_precomp_table(int window_size, unsigned char *filename)
++window_size;
}
page_sz = sysconf(_SC_PAGE_SIZE);
// round up to the next multiple of the page size
mmap_sz = prec_sz % page_sz ? (prec_sz/page_sz+1)*page_sz : prec_sz;
//prec = malloc(prec_sz);
if (prec != NULL) munmap(prec, MMAP_SIZE); // work correctly if called again
if (filename) {
if (stat(filename, &sb) == 0) {
if (!S_ISREG(sb.st_mode))
return -100;
if (sb.st_size != prec_sz)
return -102;
if ((fd = open(filename, O_RDONLY)) < 0)
return fd;
if ((ret = posix_fadvise(fd, 0, 0, POSIX_FADV_WILLNEED|POSIX_FADV_RANDOM)) < 0)
return ret;
if ((prec = mmap(NULL, MMAP_SIZE = mmap_sz, PROT_READ, MAP_SHARED|MAP_NORESERVE|MAP_POPULATE, fd, 0)) == NULL)
return -103;
} else {
return -101;
}
return 0;
} else {
prec = mmap(NULL, MMAP_SIZE = mmap_sz, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
if ((ret = mmapf(&prec_mmapf, filename, prec_sz, MMAPF_RNDRD)) != MMAPF_OKAY) {
fprintf(stderr, "failed to open ecmult table '%s': %s\n", filename, mmapf_strerror(ret));
exit(1);
} else if (prec_mmapf.mem == NULL) {
fprintf(stderr, "got NULL pointer from mmapf\n");
exit(1);
}
prec = prec_mmapf.mem;
if (filename) { return 0; }
table = malloc(n_windows*n_values*sizeof(secp256k1_gej_t));
Oops, something went wrong.

0 comments on commit 5aade72

Please sign in to comment.