Skip to content

Commit

Permalink
Add initial roadster data update utility
Browse files Browse the repository at this point in the history
  • Loading branch information
jakewmeyer committed Jul 24, 2018
1 parent 6bd52c6 commit 409cb43
Show file tree
Hide file tree
Showing 3 changed files with 157 additions and 7 deletions.
60 changes: 60 additions & 0 deletions bin/roadster.js
@@ -0,0 +1,60 @@
#!/usr/bin/env node

/**
* This updater scrapes calculated roadster orbital data, strips everthing
* but numbers, and updates a mongo document with the results
*/

const MongoClient = require('mongodb');
const puppeteer = require('puppeteer');

let client;

(async () => {
try {
client = await MongoClient.connect(process.env.MONGO_URL, { useNewUrlParser: true });
const db = client.db('spacex-api');
const col = db.collection('info');

const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto('https://where-is-tesla-roadster.space/live');

let parsed = {};
let document;

// Create object for mongo update using HTML id's for each table element
// Speed + Distance values don't have an inner span element, hence the textContent swap
const data = await page.evaluate(() => {
parsed = {
apoapsis_au: parseFloat((document.querySelector('#data_ra_au').innerText).replace(/[a-z]/gi, '')),
periapsis_au: parseFloat((document.querySelector('#data_rp_au').innerText).replace(/[a-z]/gi, '')),
semi_major_axis_au: parseFloat((document.querySelector('#data_a_au').innerText).replace(/[a-z]/gi, '')),
eccentricity: parseFloat((document.querySelector('#data_e').innerText).replace(/[a-z]/gi, '')),
inclination: parseFloat((document.querySelector('#data_i').innerText).replace(/[a-z]/gi, '')),
longitude: parseFloat((document.querySelector('#data_O').innerText).replace(/[a-z]/gi, '')),
periapsis_arg: parseFloat((document.querySelector('#data_w').innerText).replace(/[a-z]/gi, '')),
period_days: parseFloat((document.querySelector('#data_T_d').innerText).replace(/[a-z]/gi, '')),
speed_kph: parseFloat((document.querySelector('#data_os_kmh').textContent).replace(/[a-z]/gi, '')),
speed_mph: parseFloat((document.querySelector('#data_os_mph').textContent).replace(/[a-z]/gi, '')),
earth_distance_km: parseFloat((document.querySelector('#data_de_km').textContent).replace(/[a-z]/gi, '')),
earth_distance_mi: parseFloat((document.querySelector('#data_de_mile').textContent).replace(/[a-z]/gi, '')),
mars_distance_km: parseFloat((document.querySelector('#data_dm_km').textContent).replace(/[a-z]/gi, '')),
mars_distance_mi: parseFloat((document.querySelector('#data_dm_mile').textContent).replace(/[a-z]/gi, '')),
};
return parsed;
});
console.log(data);
await browser.close();

await col.updateOne({ name: 'Elon Musk\'s Tesla Roadster' }, { $set: data });
console.log('Updated!');
} catch (err) {
console.log(err.stack);
process.exit(1);
}

if (client) {
client.close();
}
})();
8 changes: 5 additions & 3 deletions package.json
Expand Up @@ -6,7 +6,8 @@
"scripts": {
"test": "yarn lint && NODE_ENV=test PORT=0 jest --silent",
"start": "node src/app.js",
"lint": "eslint \"**/*.js\""
"lint": "eslint \"**/*.js\"",
"roadster": "node bin/roadster.js"
},
"repository": {
"type": "git",
Expand Down Expand Up @@ -36,7 +37,8 @@
"koa2-cors": "^2.0.6",
"lower-case": "^1.1.4",
"mongodb": "^3.1.1",
"path": "^0.12.7"
"path": "^0.12.7",
"puppeteer": "^1.6.0"
},
"devDependencies": {
"eslint": "^5.2.0",
Expand All @@ -61,4 +63,4 @@
"engines": {
"node": ">=8.x.x"
}
}
}
96 changes: 92 additions & 4 deletions yarn.lock
Expand Up @@ -47,6 +47,12 @@ acorn@^5.0.0, acorn@^5.0.3, acorn@^5.3.0, acorn@^5.6.0:
version "5.7.1"
resolved "https://registry.yarnpkg.com/acorn/-/acorn-5.7.1.tgz#f095829297706a7c9776958c0afc8930a9b9d9d8"

agent-base@^4.1.0:
version "4.2.1"
resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-4.2.1.tgz#d89e5999f797875674c07d87f260fc41e83e8ca9"
dependencies:
es6-promisify "^5.0.0"

ajv-keywords@^3.0.0:
version "3.2.0"
resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.2.0.tgz#e86b819c602cf8821ad637413698f1dec021847a"
Expand Down Expand Up @@ -680,6 +686,15 @@ concat-map@0.0.1:
version "0.0.1"
resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"

concat-stream@1.6.2:
version "1.6.2"
resolved "https://registry.yarnpkg.com/concat-stream/-/concat-stream-1.6.2.tgz#904bdf194cd3122fc675c77fc4ac3d4ff0fd1a34"
dependencies:
buffer-from "^1.0.0"
inherits "^2.0.3"
readable-stream "^2.2.2"
typedarray "^0.0.6"

console-control-strings@^1.0.0, console-control-strings@~1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e"
Expand Down Expand Up @@ -781,7 +796,7 @@ debug@0.7.4:
version "0.7.4"
resolved "https://registry.yarnpkg.com/debug/-/debug-0.7.4.tgz#06e1ea8082c2cb14e39806e22e2f6f757f92af39"

debug@^2.1.2, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8, debug@^2.6.9:
debug@2.6.9, debug@^2.1.2, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8, debug@^2.6.9:
version "2.6.9"
resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
dependencies:
Expand Down Expand Up @@ -968,12 +983,22 @@ es6-promise@^3.2.1:
version "3.3.1"
resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-3.3.1.tgz#a08cdde84ccdbf34d027a1451bc91d4bcd28a613"

es6-promise@^4.0.3:
version "4.2.4"
resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.2.4.tgz#dc4221c2b16518760bd8c39a52d8f356fc00ed29"

es6-promisify@^4.1.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/es6-promisify/-/es6-promisify-4.1.0.tgz#dbcc1a3e93fdb8e9e709744182b7d2080c03fdb3"
dependencies:
es6-promise "^3.2.1"

es6-promisify@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/es6-promisify/-/es6-promisify-5.0.0.tgz#5109d62f3e56ea967c4b63505aef08291c8a5203"
dependencies:
es6-promise "^4.0.3"

escape-html@^1.0.3:
version "1.0.3"
resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988"
Expand Down Expand Up @@ -1263,6 +1288,15 @@ extglob@^2.0.4:
snapdragon "^0.8.1"
to-regex "^3.0.1"

extract-zip@^1.6.6:
version "1.6.7"
resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-1.6.7.tgz#a840b4b8af6403264c8db57f4f1a74333ef81fe9"
dependencies:
concat-stream "1.6.2"
debug "2.6.9"
mkdirp "0.5.1"
yauzl "2.4.1"

extsprintf@1.3.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05"
Expand Down Expand Up @@ -1293,6 +1327,12 @@ fb-watchman@^2.0.0:
dependencies:
bser "^2.0.0"

fd-slicer@~1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.0.1.tgz#8b5bcbd9ec327c5041bf9ab023fd6750f1177e65"
dependencies:
pend "~1.2.0"

figures@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/figures/-/figures-2.0.0.tgz#3ab1a2d2a62c8bfb431a0c94cb797a2fce27c962"
Expand Down Expand Up @@ -1665,6 +1705,13 @@ http-signature@~1.2.0:
jsprim "^1.2.2"
sshpk "^1.7.0"

https-proxy-agent@^2.2.1:
version "2.2.1"
resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz#51552970fa04d723e04c56d04178c3f92592bbc0"
dependencies:
agent-base "^4.1.0"
debug "^3.1.0"

humanize-number@0.0.2:
version "0.0.2"
resolved "https://registry.yarnpkg.com/humanize-number/-/humanize-number-0.0.2.tgz#11c0af6a471643633588588048f1799541489c18"
Expand Down Expand Up @@ -1711,7 +1758,7 @@ inflight@^1.0.4:
once "^1.3.0"
wrappy "1"

inherits@2, inherits@2.0.3, inherits@~2.0.3:
inherits@2, inherits@2.0.3, inherits@^2.0.3, inherits@~2.0.3:
version "2.0.3"
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de"

Expand Down Expand Up @@ -2724,6 +2771,10 @@ mime@^1.4.1:
version "1.6.0"
resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1"

mime@^2.0.3:
version "2.3.1"
resolved "https://registry.yarnpkg.com/mime/-/mime-2.3.1.tgz#b1621c54d63b97c47d3cfe7f7215f7d64517c369"

mimic-fn@^1.0.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-1.2.0.tgz#820c86a39334640e99516928bd03fca88057d022"
Expand Down Expand Up @@ -2766,7 +2817,7 @@ mixin-deep@^1.2.0:
for-in "^1.0.2"
is-extendable "^1.0.1"

mkdirp@^0.5.0, mkdirp@^0.5.1:
mkdirp@0.5.1, mkdirp@^0.5.0, mkdirp@^0.5.1:
version "0.5.1"
resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903"
dependencies:
Expand Down Expand Up @@ -3158,6 +3209,10 @@ path@^0.12.7:
process "^0.11.1"
util "^0.10.3"

pend@~1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50"

performance-now@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"
Expand Down Expand Up @@ -3249,6 +3304,10 @@ prop-types@^15.6.2:
loose-envify "^1.3.1"
object-assign "^4.1.1"

proxy-from-env@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.0.0.tgz#33c50398f70ea7eb96d21f7b817630a55791c7ee"

pseudomap@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"
Expand All @@ -3265,6 +3324,19 @@ punycode@^2.1.0:
version "2.1.1"
resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec"

puppeteer@^1.6.0:
version "1.6.0"
resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-1.6.0.tgz#b6b6bd6a063a0b7591258613ec89a135f2cf8f2e"
dependencies:
debug "^3.1.0"
extract-zip "^1.6.6"
https-proxy-agent "^2.2.1"
mime "^2.0.3"
progress "^2.0.0"
proxy-from-env "^1.0.0"
rimraf "^2.6.1"
ws "^5.1.1"

qs@^6.5.1, qs@~6.5.1:
version "6.5.2"
resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36"
Expand Down Expand Up @@ -3316,7 +3388,7 @@ read-pkg@^2.0.0:
normalize-package-data "^2.3.2"
path-type "^2.0.0"

readable-stream@^2.0.1, readable-stream@^2.0.5, readable-stream@^2.0.6:
readable-stream@^2.0.1, readable-stream@^2.0.5, readable-stream@^2.0.6, readable-stream@^2.2.2:
version "2.3.6"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf"
dependencies:
Expand Down Expand Up @@ -4011,6 +4083,10 @@ type-is@^1.6.16:
media-typer "0.3.0"
mime-types "~2.1.18"

typedarray@^0.0.6:
version "0.0.6"
resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777"

uglify-js@^2.6:
version "2.8.29"
resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.29.tgz#29c5733148057bb4e1f75df35b7a9cb72e6a59dd"
Expand Down Expand Up @@ -4205,6 +4281,12 @@ ws@^4.0.0:
async-limiter "~1.0.0"
safe-buffer "~5.1.0"

ws@^5.1.1:
version "5.2.2"
resolved "https://registry.yarnpkg.com/ws/-/ws-5.2.2.tgz#dffef14866b8e8dc9133582514d1befaf96e980f"
dependencies:
async-limiter "~1.0.0"

x-xss-protection@1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/x-xss-protection/-/x-xss-protection-1.1.0.tgz#4f1898c332deb1e7f2be1280efb3e2c53d69c1a7"
Expand Down Expand Up @@ -4257,6 +4339,12 @@ yargs@~3.10.0:
decamelize "^1.0.0"
window-size "0.1.0"

yauzl@2.4.1:
version "2.4.1"
resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.4.1.tgz#9528f442dab1b2284e58b4379bb194e22e0c4005"
dependencies:
fd-slicer "~1.0.1"

ylru@^1.2.0:
version "1.2.1"
resolved "https://registry.yarnpkg.com/ylru/-/ylru-1.2.1.tgz#f576b63341547989c1de7ba288760923b27fe84f"

0 comments on commit 409cb43

Please sign in to comment.