@@ -6,14 +6,21 @@ typedef unsigned char byte; -int read_cmd(byte *buf) +int read_cmd(byte** buf, int& buffer_size) { int len; - if (read_exact(buf, 2) != 2) + if (read_exact(*buf, 2) != 2) return(-1); - len = (buf[0] << 8) | buf[1]; - return read_exact(buf, len); + len = ((*buf)[0] << 8) | (*buf)[1]; + + if(len > buffer_size){ + buffer_size = len; + delete[] *buf; + *buf = new byte[buffer_size]; + } + + return read_exact(*buf, len); } int write_cmd(byte *buf, int len) googleurl/src/erl_comm.c @@ -4,7 +4,7 @@ typedef unsigned char byte; -int read_cmd(byte *buf); +int read_cmd(byte **buf, int& buffer_size); int write_cmd(byte *buf, int len); googleurl/src/erl_comm.h @@ -22,7 +22,7 @@ int main() { erl_init(NULL, 0); //Required line to initialize erl_interface - while (read_cmd(buf) > 0) { + while (read_cmd(&buf, buffer_size) > 0) { tuplep = erl_decode(buf); tuple1 = erl_element(1, tuplep); googleurl/src/port.cc @@ -1,3 +1,5 @@ #!/bin/sh +make + erl -pa ebin -eval 'inets:start().' -eval 'googleurl:start("googleurl/bin/port").' -eval 'application:start(spider).' spider_run.sh @@ -158,9 +158,9 @@ extract_document_links(Html, URL) -> StringLinks = lists:map(fun(X) -> binary_to_list(X) end, BinaryLinks), CleanedLinks = clean_links(StringLinks, URL), - - lists:filter(fun(dud) -> false; - (_X) -> true +% io:format("DEBUG: cleaned links: ~p~n", [CleanedLinks]), + lists:filter(fun([$h,$t,$t,$p,$:,$/,$/|_]) -> true; + (_X) -> false end, CleanedLinks). src/fetcher.erl ac778d0b0700bb5296ba1eeaa28ecbdce2a5bd0c xanados bjterry@Benjamin-Terrys-MacBook-Pro-15.local http://github.com/formido/spider/commit/ade2440311d3b33a8b9bd00d38e778d9b65c75ae ade2440311d3b33a8b9bd00d38e778d9b65c75ae 2008-09-20T15:06:52-07:00 2008-09-20T15:06:52-07:00 Fixed the bug for too long of lists to canonicalize. Script to run the application. c87f15a20a0f02e4882183795d9908354bd21b30 xanados bjterry@Benjamin-Terrys-MacBook-Pro-15.local