written, needs to be tested better.

Author: matthewmueller

.gitignore

@@ -1,2 +1,5 @@
 node_modules
 npm-debug.log
+.DS_Store
+examples/_*
+examples/*.json

Makefile

@@ -2,6 +2,7 @@
 test:
 	@./node_modules/.bin/mocha \
 		--require should \
-		--reporter spec
+		--reporter spec \
+		--timeout 20s
 
-.PHONY: test
+.PHONY: test

Readme.md

@@ -3,6 +3,8 @@
 
   structure any website
 
+##
+
 ## License
 
 (The MIT License)

examples/dribbble-search.js

@@ -0,0 +1,65 @@
+/**
+ * Module Dependencies
+ */
+
+var xray = require('..');
+var write = require('fs').createWriteStream(__dirname + '/dribbble-search.json');
+
+/**
+ * Base URL
+ */
+
+var url = 'https://dribbble.com/search?q=form';
+
+/**
+ * x-ray
+ */
+
+xray(url)
+  .selector('.dribbbles > li')
+  .property('url', '.dribbble-img > .dribbble-link', 'href')
+  .property('name', '.dribbble-img img', 'alt')
+  .property('image', '.dribbble-img img', 'src')
+  .paginate('.next_page')
+  // .format(format)
+  .run(function(err, json) {
+    if (err) throw err;
+    // console.log(json);
+    // console.log(json);
+    // console.log('called');
+    // var str = JSON.stringify(arr, true, 2);
+    // write(__dirname + '/dribble.json', str, 'utf8');
+    // console.log('crawled!');
+  })
+  .end(function(err, json) {
+    console.log(err, json);
+  })
+  .stream(write);
+  // .end(function(err) {
+  //   if (err) throw err;
+  //   console.log('crawled');
+  // });
+
+/**
+ * Images
+ */
+
+function images($el, $) {
+  var imgs = [];
+
+  $el.find('a').each(function(i, el) {
+    imgs.push($(el).attr('data-src'));
+  })
+
+  return imgs;
+}
+
+/**
+ * Format
+ */
+
+function format(json) {
+  json.tags = json.tags.split(',');
+  json.shots = +json.shots.replace(/\D+/g, '');
+  json.followers = +json.followers.replace(/\D+/g, '');
+}

examples/github-stars.js

@@ -1,13 +1,71 @@
+/**
+ * Module Dependencies
+ */
 
-Structural
-  .url('https://github.com/stars/matthewmueller?direction=asc&language=javascript&sort=created')
-  .url('https://github.com/stars/matthewmueller?direction=desc&language=javascript&sort=created')
-  .key('title', 'div > ul > li.repo-list-item.public.source > h3.repo-list-name > a')
+var xray = require('..');
+var array = require('array');
+var fmt = require('util').format;
+
+/**
+ * Urls
+ */
+
+var languages = [
+  'JavaScript',
+  'CSS',
+  'CoffeeScript',
+  'Ruby',
+  'Python',
+  'Shell',
+  'Go',
+  'C',
+  'C++',
+  'Objective-C',
+  'PHP',
+  'VimL',
+  'Java',
+  'Swift',
+  'Scala',
+  'TeX',
+  'Perl',
+  'Lua',
+  'Clojure',
+  'IDL',
+  'Objective-C++',
+  'Processing',
+  'R',
+  'Vala',
+  'Lisp',
+  'XSLT',
+  'LiveScript',
+  'TypeScript'
+];
+
+/**
+ * Get the urls
+ */
+
+var base = 'https://github.com/stars/matthewmueller?direction=%s&language=%s&sort=created';
+var urls = [];
+languages.forEach(function(language) {
+  language = decodeURIComponent(language);
+  urls.push(fmt(base, 'asc', language));
+  urls.push(fmt(base, 'desc', language));
+})
+
+/**
+ * Use
+ */
+
+xray(urls)
+  .key('repo', 'div > ul > li.repo-list-item.public.source > h3.repo-list-name > a')
   .key('url', 'div > ul > li.repo-list-item.public.source > h3.repo-list-name > a', 'href')
   .key('author', 'ul > li.repo-list-item.public.source > h3.repo-list-name > a > span.prefix')
   .key('description', 'div > div > ul > li > p.repo-list-description')
+  .key('time', '.repo-list-meta time', 'datetime')
   .paginate('div.pagination > a:last-child')
   .json(function(err, json) {
     if (err) throw err;
-    console.log(json);
+    json = array(json).unique('repo').toArray();
+    console.log(JSON.stringify(json, true, 2));
   })