Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Comparing changes

Choose two branches to see what's changed or to start a new pull request. If you need to, you can also compare across forks.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also compare across forks.
...
  • 6 commits
  • 3 files changed
  • 0 commit comments
  • 3 contributors
Commits on Sep 01, 2011
@gregory80 gregory80 update docs as per npm docs for linking projects
cd ~/projects/node-bloggy  # go into the dir of your main project
npm link ../node-redis     # link the dir of your dependency
2cedca0
@mikeal mikeal Merge pull request #10 from gregory80/patch-1
update docs as per npm docs for linking projects
669c10a
Commits on Mar 10, 2012
@mikeal mikeal Fixing for new node and jsdom. ce70f1f
@mikeal mikeal Merge branch 'master' of github.com:mikeal/spider 268df09
Commits on Mar 11, 2012
@twleung Merge remote-tracking branch 'upstream/master' c900aa1
@twleung remove leftover test code 844ca05
Showing with 15 additions and 37 deletions.
  1. +1 −1  README.md
  2. +11 −33 main.js
  3. +3 −3 package.json
View
2  README.md
@@ -7,7 +7,7 @@ From source:
<pre>
git clone git://github.com/mikeal/spider.git
cd spider
- npm link .
+ npm link ../spider
</pre>
## (How to use the) API
View
44 main.js
@@ -2,6 +2,7 @@ var request = require('request')
, fs = require('fs')
, sys = require('sys')
, path = require('path')
+ , vm = require('vm')
, jsdom = require('jsdom')
, util = require('util')
, urlParse = require('url').parse
@@ -40,33 +41,6 @@ jsdom.defaultDocumentFeatures =
, MutationEvents : false
, QuerySelector : false
}
-
-var Context = process.binding('evals').Context,
- Script = process.binding('evals').Script;
-jqueryify = function(window, document) {
- var filename = jqueryFilename
- , document = window.document
- ;
- if (window) {
- var ctx = window.__scriptContext;
- if (!ctx) {
- window.__scriptContext = ctx = new Context();
- ctx.__proto__ = window;
- }
- var tracelimitbak = Error.stackTraceLimit;
- Error.stackTraceLimit = 100;
- try {
- Script.runInContext(jquery, ctx, filename);
- }
- catch(e) {
- document.trigger(
- 'error', 'Running ' + filename + ' failed.',
- {error: e, filename: filename}
- );
- }
- Error.stackTraceLimit = tracelimitbak;
- }
-};
var debug = 1
, info = 50
@@ -167,7 +141,8 @@ Spider.prototype.get = function (url, referer) {
return;
}
if (resp.headers['set-cookie']) {
- self.jar.setCookies(resp.headers['set-cookie'])
+ try { self.jar.setCookies(resp.headers['set-cookie']) }
+ catch(e) {}
}
self.cache.set(url, resp.headers, body);
self._handler(url, referer, {fromCache:false, headers:resp.headers, body:body});
@@ -195,8 +170,11 @@ Spider.prototype._handler = function (url, referer, response) {
r.spider = this;
r.response = response
r.url = u;
- var window = jsdom.jsdom(response.body).createWindow();
- jqueryify(window);
+
+ var document = jsdom.jsdom(response.body, null, {})
+ var window = document.createWindow()
+ window.run(jquery, jqueryFilename)
+
window.$.fn.spider = function () {
this.each(function () {
var h = window.$(this).attr('href');
@@ -206,15 +184,15 @@ Spider.prototype._handler = function (url, referer, response) {
self.get(h, url);
})
}
-
+
this.currentUrl = url;
if (jsdom.defaultDocumentFeatures.ProcessExternalResources) {
$(function () { r.fn.call(r, window, window.$); })
} else {
r.fn.call(r, window, window.$);
}
- this.currentUrl = null;
- }
+ this.currentUrl = null;
+ }
}
Spider.prototype.log = function (level) {
if (typeof level === 'string') level = logLevels[level];
View
6 package.json
@@ -1,7 +1,7 @@
{ "name" : "spider"
, "description" : "Programmable spidering of web sites with node.js and jQuery"
, "tags" : ["dom","javascript","crawling","jquery", "spider", "spidering"]
-, "version" : "0.0.2"
+, "version" : "0.1.0"
, "author" : "Mikeal Rogers <mikeal.rogers@gmail.com>"
, "repository" :
{ "type" : "git"
@@ -9,11 +9,11 @@
}
, "bugs" :
{ "web" : "http://github.com/mikeal/spider/issues" }
-, "engines" : ["node >= 0.4.4"]
+, "engines" : ["node >= 0.6.4"]
, "main" : "./main"
, "dependencies":
{ "request" : ">= 1.9.3"
- , "jsdom" : ">= 0.2.0"
+ , "jsdom" : ">= 0.2.13"
, "routes" : ">= 0.1.0"
, "cookiejar": ">= 1.3.0"
}

No commit comments for this range

Something went wrong with that request. Please try again.