Starting to use httpbin to test node-crawler and update dependencies

bda-research · May 13, 2014 · 3914a1e · 3914a1e
1 parent 069514b
commit 3914a1e
Show file tree

Hide file tree

Showing 4 changed files with 83 additions and 91 deletions.
diff --git a/package.json b/package.json
@@ -1,64 +1,66 @@
 {
-    "name": "crawler",
-    "version": "0.2.5",
-    "description": "Crawler is a web spider written with Nodejs. It gives you the full power of jQuery on the server to parse a big number of pages as they are downloaded, asynchronously. Scraping should be simple and fun!",
-    "keywords": [
-        "dom",
-        "javascript",
-        "crawling",
-        "spider",
-        "scraper",
-        "scraping",
-        "jquery"
-    ],
-    "maintainers": [
-        {
-            "name": "Sylvain Zimmer",
-            "email": "sylvain@sylvainzimmer.com",
-            "url": "http://sylvinus.org/"
-        }
-    ],
-    "bugs": {
-        "mail": "sylvain@sylvainzimmer.com",
-        "url": "http://github.com/sylvinus/node-crawler/issues"
-    },
-    "licenses": [
-        {
-            "type": "MIT",
-            "url": "http://github.com/sylvinus/node-crawler/blob/master/LICENSE.txt"
-        }
-    ],
-    "repository":
-        {
-            "type": "git",
-            "url": "https://github.com/sylvinus/node-crawler.git"
-        }
-    ,
-    "dependencies": {
-       "htmlparser": "1.7.6",
-       "request": "2.21.0",
-       "jsdom": "0.8.2",
-       "generic-pool": "2.0.3",
-       "underscore": "1.4.4",
-       "jschardet": "1.0.2",
-       "iconv-lite": "0.2.8"
-    },
-    "optionalDependencies":{
-        "iconv": "2.0.6"
-    },
-    "devDependencies": {
-       "qunit": "0.5.16",
-       "express": "2.5.x",
-       "memwatch": "0.2.2"
-    },
-    "scripts": {
-        "test": "node test/testrunner.js"
-    },
-    "engines": [
-        "node >=0.6.x"
-    ],
-    "directories": {
-        "lib": "lib"
-    },
-    "main": "./lib/crawler"
+  "name": "crawler",
+  "version": "0.2.5",
+  "description": "Crawler is a web spider written with Nodejs. It gives you the full power of jQuery on the server to parse a big number of pages as they are downloaded, asynchronously. Scraping should be simple and fun!",
+  "keywords": [
+    "dom",
+    "javascript",
+    "crawling",
+    "spider",
+    "scraper",
+    "scraping",
+    "jquery"
+  ],
+  "maintainers": [
+    {
+      "name": "Sylvain Zimmer",
+      "email": "sylvain@sylvainzimmer.com",
+      "url": "http://sylvinus.org/"
+    }
+  ],
+  "bugs": {
+    "mail": "sylvain@sylvainzimmer.com",
+    "url": "http://github.com/sylvinus/node-crawler/issues"
+  },
+  "licenses": [
+    {
+      "type": "MIT",
+      "url": "http://github.com/sylvinus/node-crawler/blob/master/LICENSE.txt"
+    }
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/sylvinus/node-crawler.git"
+  },
+  "dependencies": {
+    "htmlparser": "^1.7.7",
+    "request": "^2.34.0",
+    "jsdom": "^0.10.5",
+    "generic-pool": "^2.0.4",
+    "underscore": "^1.6.0",
+    "jschardet": "^1.1.0",
+    "iconv-lite": "^0.4.0-pre3",
+    "express": "^4.2.0",
+    "qunit": "^0.6.3",
+    "iconv": "^2.1.0",
+    "compression": "^1.0.2"
+  },
+  "optionalDependencies": {
+    "iconv": "*"
+  },
+  "devDependencies": {
+    "qunit": "*",
+    "express": "*",
+    "memwatch": "*"
+  },
+  "scripts": {
+    "test": "node test/testrunner.js"
+  },
+  "engines": [
+    "node >=0.8.x"
+  ],
+  "directories": {
+    "lib": "lib"
+  },
+  "main": "./lib/crawler"
 }
diff --git a/test/mockserver.js b/test/mockserver.js
@@ -1,6 +1,5 @@
-var express = require('express'),
-    path = require("path");
-var app = express.createServer();
+var express = require('express');
+var app = express();
 
 app.get('/timeout', function(req, res){
   setTimeout(function() {
@@ -37,13 +36,6 @@ app.get('/bigpage', function(req, res){
 });
 
 
-app.use("/mockfiles/gzipped/",express.compress());
-
-app.use('/mockfiles/', express["static"](path.resolve(__dirname, 'mockfiles')));
-
-
-
-
 exports.app = app;
 
 if (require.main === module) {

diff --git a/test/testrunner.js b/test/testrunner.js
@@ -1,34 +1,32 @@
 #!/usr/bin/env node
 
-var testrunner = require( "qunit" ),
+var testrunner = require('qunit'),
+
+path = require('path').normalize( __dirname + '/..');
 
-path = require( "path" ).normalize( __dirname + "/.." );
-
 testrunner.setup({
 
 });
 
-var mockserver = require("./mockserver").app;
-
-mockserver.listen(30045);
+var mockserver = require('./mockserver').app.listen(30045);
 
 testrunner.run([
         {
-            code: path + "/lib/crawler.js",
+            code: path + '/lib/crawler.js',
             tests: [
-             path + "/test/units/simple.js",
-              path + "/test/units/links.js",
-             
-              path + "/test/units/forceutf8.js",
-              
-              path + "/test/units/errors.js",
-             
-              path + "/test/units/leaks.js"
-  
+             path + '/test/units/simple.js',
+              path + '/test/units/links.js',
+
+              path + '/test/units/forceutf8.js',
+
+              path + '/test/units/errors.js',
+
+              path + '/test/units/leaks.js'
+
             ]
         }
 ],function(err, report) {
-  console.log("Stopping mockserver...");
+  console.log('Stopping mockserver...');
   mockserver.close();
   process.exit((err || report.failed !== 0) ? 1 : 0);
 });
diff --git a/test/units/simple.js b/test/units/simple.js
@@ -76,13 +76,13 @@ test("one request gzipped", function() {
         "debug":DEBUG,
         "callback":function(error,result,$) {
             equal(error,null);
-            ok(result.body.indexOf("gzipped okay.")>0);
+            ok(result.body.indexOf("User-Agent")>0);
             ok(result.headers["content-encoding"]=="gzip");
             start();
         }
     });
 
-    c.queue(["http://127.0.0.1:"+MOCKPORT+"/mockfiles/gzipped/test-gzip.html"]);
+    c.queue(['http://httpbin.org/gzip']);
 
 });