Skip to content

Commit

Permalink
Add template for morph.io scraper
Browse files Browse the repository at this point in the history
  • Loading branch information
cellomango committed Feb 29, 2016
0 parents commit f3ed0cb
Show file tree
Hide file tree
Showing 4 changed files with 69 additions and 0 deletions.
2 changes: 2 additions & 0 deletions .gitignore
@@ -0,0 +1,2 @@
# Ignore output of scraper
data.sqlite
1 change: 1 addition & 0 deletions README.md
@@ -0,0 +1 @@
This is a scraper that runs on [Morph](https://morph.io). To get started [see the documentation](https://morph.io/documentation)
7 changes: 7 additions & 0 deletions package.json
@@ -0,0 +1,7 @@
{
"dependencies": {
"cheerio": "latest",
"request": "latest",
"sqlite3": "latest"
}
}
59 changes: 59 additions & 0 deletions scraper.js
@@ -0,0 +1,59 @@
// This is a template for a Node.js scraper on morph.io (https://morph.io)

var cheerio = require("cheerio");
var request = require("request");
var sqlite3 = require("sqlite3").verbose();

function initDatabase(callback) {
// Set up sqlite database.
var db = new sqlite3.Database("data.sqlite");
db.serialize(function() {
db.run("CREATE TABLE IF NOT EXISTS data (name TEXT)");
callback(db);
});
}

function updateRow(db, value) {
// Insert some data.
var statement = db.prepare("INSERT INTO data VALUES (?)");
statement.run(value);
statement.finalize();
}

function readRows(db) {
// Read some data.
db.each("SELECT rowid AS id, name FROM data", function(err, row) {
console.log(row.id + ": " + row.name);
});
}

function fetchPage(url, callback) {
// Use request to read in pages.
request(url, function (error, response, body) {
if (error) {
console.log("Error requesting page: " + error);
return;
}

callback(body);
});
}

function run(db) {
// Use request to read in pages.
fetchPage("https://morph.io", function (body) {
// Use cheerio to find things in the page with css selectors.
var $ = cheerio.load(body);

var elements = $("div.media-body span.p-name").each(function () {
var value = $(this).text().trim();
updateRow(db, value);
});

readRows(db);

db.close();
});
}

initDatabase(run);

0 comments on commit f3ed0cb

Please sign in to comment.