Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Loading…

Adding new event: discoverycomplete #10

Merged
merged 1 commit into from

2 participants

@mmoulton

Adding event 'discoverycomplete' to emit the discovered resources of a queued item.

@cgiffard
Owner

Thanks, looks good! I'll have to update the documentation to include it. :)

@cgiffard cgiffard merged commit 5610ff6 into cgiffard:master
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Dec 27, 2012
  1. @mmoulton
This page is out of date. Refresh to see the latest.
Showing with 8 additions and 2 deletions.
  1. +8 −2 index.js
View
10 index.js
@@ -279,7 +279,7 @@ var Crawler = function(host,initialPath,initialPort,interval) {
// This might be a bit of a gamble... but get hard-coded strings out of javacript: URLs
// They're often popup-image or preview windows, which would otherwise be unavailable to us
cleanAndQueue(resourceText.match(/^javascript\:[a-z0-9]+\(['"][^'"\s]+/ig));
-
+
return resources;
}
@@ -348,7 +348,13 @@ var Crawler = function(host,initialPath,initialPort,interval) {
// Input some text/html and this function will delegate resource discovery, check link validity
// and queue up resources for downloading!
function queueLinkedItems(resourceData,queueItem) {
- discoverResources(resourceData,queueItem).forEach(function(url){ queueURL(url,queueItem); });
+ var resources = discoverResources(resourceData,queueItem);
+
+ // Emit discovered resources. ie: might be useful in building a graph of
+ // page relationships.
+ crawler.emit("discoverycomplete",queueItem,resources);
+
+ resources.forEach(function(url){ queueURL(url,queueItem); });
}
// Clean and queue a single URL...
Something went wrong with that request. Please try again.