A Node.js Web Spider
Switch branches/tags
Nothing to show
Clone or download
Fetching latest commit…
Cannot retrieve the latest commit at this time.
Permalink
Failed to load latest commit information.
examples
lib
test
.gitignore
.jshintrc
.travis.yml
Gruntfile.js
LICENSE
README.md
index.js
package.json

README.md

nspider

Build Status

A simple Web Spider/Crawler Node.js package.

var nspider = require('nspider')();

/* create spider object */
var spider = new nspider.Spider();

/* fetch the content of 'http://nodejs.org' */
spider.fetch('http://nodejs.org', function (err, data) {
  if (err) {
    /* fetch error */
    console.log('fetchError: %s', err.toString());
    return;
  }

  if (data.contentType.type !== 'text' ||
      data.contentType.subtype !== 'html') {
    /* Not a HTML page */
    console.log('Invalid contentType: %j', data.contentType);
    return;
  }

  /* Print all <a> href attr */
  data.$('a').each(function (idx,  item) {
    console.log('Link[%d]: %s', idx, data.$(item).attr('href'));
  });
});

Installation

npm install nspider

Examples

  • examples/list_all_links.js - List all links.
  • examples/save_all_img.js - Save all images.

Build

git clone https://github.com/xiongjia/nspider.git
npm run build