Browse files

reduce the depth of the crawler

  • Loading branch information...
1 parent 14b7159 commit 60ca35e7869bbfc9311065c83ed41167a04c5be3 @ptek committed Mar 16, 2012
Showing with 3 additions and 3 deletions.
  1. +2 −2 src/Service/CrawlerService.hs
  2. +1 −1 tests/Unit/Service/CrawlerServiceTest.hs
View
4 src/Service/CrawlerService.hs
@@ -11,7 +11,7 @@ import Model.Crawler
runCrawler :: (Text -> IO [Tag Text]) -> Text -> [Text] -> IO [(Text, Bool)]
runCrawler fetcher domainToSearchFor domainsToSearchOn = mapM search domainsToSearchOn where
search d = do
- found <- crawl fetcher domainToSearchFor d [d] [] 3
+ found <- crawl fetcher domainToSearchFor d [d] [] 2
return (d, found)
crawl :: (Text -> IO [Tag Text]) -> Text -> Text -> [Text] -> [Text] -> Int -> IO Bool
@@ -21,5 +21,5 @@ crawl fetcher target domain sources acc depth = do
if (findBackLink target tags)
then return True
else do
- let newLinks = nub ((\\) (nub (findLocalLinks domain tags)) acc)
+ let newLinks = (nub (findLocalLinks domain tags)) \\ acc
crawl fetcher target domain newLinks (acc ++ newLinks) (depth-1)
View
2 tests/Unit/Service/CrawlerServiceTest.hs
@@ -11,7 +11,7 @@ tests = [
"runs crawler" ~: test [
testIO (runCrawler fetcherMock1 "git.io" ["http://k.net"]) (@?= [("http://k.net",True)])
,testIO (runCrawler fetcherMock2 "git.io" ["http://k.net"]) (@?= [("http://k.net",True)])
- ,testIO (runCrawler fetcherMock3 "git.io" ["http://k.net"]) (@?= [("http://k.net",True)])
+ ,testIO (runCrawler fetcherMock3 "git.io" ["http://k.net"]) (@?= [("http://k.net",False)])
,testIO (runCrawler fetcherMock4 "git.io" ["http://k.net"]) (@?= [("http://k.net",False)])
]
]

0 comments on commit 60ca35e

Please sign in to comment.