@@ -1,2 +1,2 @@ --record(task, {url}). +-record(task, {url,sandboxRegex="",depth=-1}). include/task.hrl @@ -109,17 +109,21 @@ code_change(_OldVsn, State, _Extra) -> process_task(Task) -> Url = Task#task.url, + io:format("~p Processing ~p~n", [self(), Url]), - {http, Host, _Port, _File} = parse(Url), + {http, Host, Port, File} = parse(Url), case http:request(get, {Url, ?HEADERS(Host)}, [], [{body_format, string}]) of {ok, {{_Version, 200, _Reason}, _Headers, Body}} -> Parsed = mochiweb_html:parse(Body), - Links = extract_document_links(Parsed), + % Extracts all links from the document and ensures they + % are within the sandbox + Links = filter_regex(extract_document_links(Parsed, {http, Host, Port, File}), Task#task.sandboxRegex), + %DocumentText = clean_document(Parsed), %TermFrequencies = clustering:term_frequencies(DocumentText), @@ -134,11 +138,11 @@ process_task(Task) -> #result{status=failure, code=Other} end. -extract_document_links(Html) -> +extract_document_links(Html, {http, Host, Port, File}) -> BinaryLinks = lists:flatten(extract_links(Html)), StringLinks = lists:map(fun(X) -> binary_to_list(X) end, BinaryLinks), - CleanedLinks = clean_links(StringLinks), + CleanedLinks = clean_links(StringLinks, {http, Host, Port, File}), lists:filter(fun(dud) -> false; (_X) -> true @@ -164,16 +168,19 @@ extract_links([_Head|Tail]) -> extract_links(Tail); extract_links(X) when is_binary(X) -> []; extract_links([]) -> []; extract_links(X) -> - io:format("DEBUG: extract_links(~p)~n", [X]), +% io:format("DEBUG: extract_links(~p)~n", [X]), []. -clean_links(Links) -> +clean_links(Links, {http, Host, Port, File}) -> lists:map(fun("") -> dud; % Probably an AJAX link. ("javascript:" ++ _Tail) -> dud; % Don't care about JS ("http:") -> dud; % These appear on Google pages sometimes... ("#") -> dud; % Ignore page-local links -% ("/" ++ Tail) -> % Site-relative URL, convert to absolute -% "http://" ++ Host ++ "/" ++ Tail; + ("/" ++ Tail) -> % Site-relative URL, convert to absolute + case Port of + 80 ->"http://" ++ Host ++ "/" ++ Tail; + _ ->"http://" ++Host++":"+ integer_to_list(Port) ++ "/" ++ Tail + end; ("mailto:" ++ _Tail) -> dud; % Mail links... don't care! ("http://" ++ Tail) -> "http://" ++ Tail; (Other) -> @@ -235,3 +242,16 @@ scan_for_slash(X) -> $/ -> lists:reverse(Rest); _ -> scan_for_slash(Rest) end. + + +% Filters a list for all items that match a given regular expression. +% Items with no match are discarded +filter_regex(ItemList, Regex) -> + lists:filter(fun(Item) -> + case regexp:first_match(Item, Regex) of + {match, _, _} -> true; + _ -> false + end + end, + ItemList). + src/fetcher.erl @@ -102,5 +102,5 @@ report_stats() -> io:format("~nSTATS: l(WorkerQueue) = ~p, l(TaskQueue) = ~p", [WorkerQueueLength, TaskQueueLength]), - timer:apply_after(1000, ?MODULE, report_stats, []), + timer:apply_after(10000, ?MODULE, report_stats, []), ok. src/stats_collector.erl @@ -100,9 +100,12 @@ handle_cast({post_result, Task, Result}, State) -> lists:foreach(fun(T) -> case get(T) of undefined -> - insert_task(T), - put(T, new); - + % Insert a new task with each URL, retaining other params + if Task#task.depth /= 1 -> + insert_task(Task#task{url=T, depth=Task#task.depth - 1}), + put(T, new); + true -> ok + end; _Other -> % Already got this task ok @@ -118,18 +121,19 @@ handle_cast({post_result, Task, Result}, State) -> handle_cast({insert_task, Task}, State) -> case queue:is_empty(State#state.worker_queue) of - true -> - {noreply, - State#state{task_queue=queue:in_r(Task, - State#state.task_queue)}}; - false -> - {{value, Worker}, NewWorkerQueue} = - queue:out(State#state.worker_queue), - - gen_server:cast(Worker, {task, Task}), - {noreply, State#state{worker_queue=NewWorkerQueue}} + true -> + {noreply, + State#state{task_queue=queue:in_r(Task, + State#state.task_queue)}}; + false -> + {{value, Worker}, NewWorkerQueue} = + queue:out(State#state.worker_queue), + + gen_server:cast(Worker, {task, Task}), + {noreply, State#state{worker_queue=NewWorkerQueue}} end. - + + %%-------------------------------------------------------------------- %% Function: handle_info(Info, State) -> {noreply, State} | src/task_master.erl 9c7cfc7c9495f63912e2f506305b139056ca3d51 xanados bjterry@Benjamin-Terrys-MacBook-Pro-15.local http://github.com/michaelmelanson/spider/commit/83c8512d2a8d8295401e1e218545479fff6b5c16 83c8512d2a8d8295401e1e218545479fff6b5c16 2008-09-06T17:36:34-07:00 2008-09-06T17:36:34-07:00 Added support for control of search depth and sandboxing based on regular expression. To test the features use the following command: rr("include/tasks.hrl"). %You need this to access task records from the erl shell task_master:insert_task(#task{url="http://www.example.com/", sandboxRegex="example.com", depth=5}). By default, no sandbox is specified (sandboxRegex="") and the search is unbounded (depth=-1). A depth of 1 means only the initial page is grabbed. Also half-implemented support for relative links of the form "/link". That is, host relative links rather than directory relative links. Directory relative links will require full parsing of the URL, probably. I also decreased the update on the status output to 10 seconds instead of 1. X ea59502a9e4829cde2835960802f7813093db509 xanados bjterry@Benjamin-Terrys-MacBook-Pro-15.local