First commit

drzax · Jun 11, 2016 · 8898269 · 8898269
1 parent c77e008
commit 8898269
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 27 deletions.
diff --git a/README.md b/README.md
@@ -1,5 +1,5 @@
-Australian federal election candidates 2016
-===========================================
+Australian federal election electorates 2016
+============================================
 
 According to the ABC's website.
 

diff --git a/package.json b/package.json
@@ -4,8 +4,8 @@
     "request": "latest",
     "sqlite3": "latest"
   },
-  "name": "morph-australian-federal-election-candidates-2016",
-  "description": "Scrapes the ABC's website for Australian federal election candidates in 2016 election",
+  "name": "morph-australian-federal-election-electorates-2016",
+  "description": "Scrapes the ABC's website for Australian federal election electorates in 2016",
   "version": "1.0.0",
   "main": "scraper.js",
   "devDependencies": {},
@@ -14,14 +14,14 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/drzax/morph-australian-federal-election-candidates-2016.git"
+    "url": "git+https://github.com/drzax/morph-australian-federal-election-electorates-2016.git"
   },
   "keywords": [
     "scraper",
     "data",
     "politics",
     "australia",
-    "candidates"
+    "electorates"
   ],
   "engines": {
     "node": "5.8.0",
@@ -30,7 +30,7 @@
   "author": "Simon Elvery",
   "license": "ISC",
   "bugs": {
-    "url": "https://github.com/drzax/morph-australian-federal-election-candidates-2016/issues"
+    "url": "https://github.com/drzax/morph-australian-federal-election-electorates-2016/issues"
   },
-  "homepage": "https://github.com/drzax/morph-australian-federal-election-candidates-2016#readme"
+  "homepage": "https://github.com/drzax/morph-australian-federal-election-electorates-2016#readme"
 }
diff --git a/scraper.js b/scraper.js
@@ -19,45 +19,34 @@ db = new Promise((resolve, reject) => {
 		conn.run(`CREATE TABLE IF NOT EXISTS data
 				(
 					partyCode TEXT,
-					partyName TEXT,
-					name TEXT,
-					familyName TEXT,
-					givenName TEXT,
-					sitting INT,
 					electorateCode TEXT,
 					electorateState TEXT,
 					electorateName TEXT,
-					electorateChamber TEXT
+					margin REAL
 				)`, (err) => err ? reject(err) : resolve(conn));
 	});
 });
 
-url('http://www.abc.net.au/news/federal-election-2016/guide/candidates/')
+url('http://www.abc.net.au/news/federal-election-2016/guide/electorates/')
 	.then((html) => {
 		var $;
 		$ = cheerio.load(html);
-		$('#candidatestable tbody tr').filter(function(){
-			return $(this).find('.candidate').text().indexOf('Retiring MP') === -1;
-		}).each(function() {
+		$('#electoratestable tbody tr').each(function() {
 			var $tr, data, electorateCodeMatch;
 			$tr = $(this);
 			data = {};
 
 			electorateCodeMatch = $tr.find('.electorate a').attr('href').match(/([a-z]+)\/$/);
 
-			data.$partyCode = $tr.find('.party span').text().trim();
-			data.$partyName = $tr.find('.party span').attr('title').trim();
-			data.$name = $tr.find('.candidate').text().replace(/\(.*$/,'').trim();
-			data.$familyName = $tr.find('.candidate .familyname').text().trim();
-			data.$givenName = data.$name.replace(data.$familyName,'').trim();
-			data.$sitting = $tr.find('.candidate').text().indexOf('(Sitting MP)') > -1;
+			data.$partyCode = $tr.find('.party span').text().trim().split(' ')[0].toLowerCase();
+			// data.$partyName = $tr.find('.party span').attr('title').trim();
+			data.$electorateName = $tr.find('.electorate a').text().replace('(*)','').trim();
 			data.$electorateCode = (electorateCodeMatch) ? electorateCodeMatch[1] : null;
 			data.$electorateState = $tr.find('.electorate').text().replace('(*)','').match(/\((.+)\)/)[1];
-			data.$electorateName = $tr.find('.electorate a').text().replace('Senate - ','').replace('(*)','').trim();
-			data.$electorateChamber = ($tr.find('.electorate a').text().indexOf('Senate') >= 0) ? 'senate' : 'house';
+			data.$margin = $tr.find('.margin').text().trim();
 
 			db.then(function(db) {
-				db.run("INSERT INTO data (partyCode, partyName, name, familyName, givenName, sitting, electorateCode, electorateState, electorateName, electorateChamber) VALUES ($partyCode, $partyName, $name, $familyName, $givenName, $sitting, $electorateCode, $electorateState, $electorateName, $electorateChamber)", data, (global.gc) ? global.gc : null);
+				db.run("INSERT INTO data (partyCode, electorateCode, electorateState, electorateName, margin) VALUES ($partyCode, $electorateCode, $electorateState, $electorateName, $margin)", data, (global.gc) ? global.gc : null);
 			}, handleErr);
 
 		});