From a0868aaaf307152b067ece204afd180698590eb1 Mon Sep 17 00:00:00 2001 From: thigg Date: Wed, 20 Mar 2024 11:24:09 +0100 Subject: [PATCH] cleaned up repo --- .editorconfig | 14 ------- .eslintrc.json | 17 --------- dbrenderer.js | 35 ------------------ package.json | 52 -------------------------- prices.js | 76 -------------------------------------- readme.md | 32 +++++----------- render.js | 36 ------------------ service/fahrpreise.service | 9 ----- service/fahrpreise.timer | 12 ------ testconnections.json | 5 --- where.js | 51 ------------------------- 11 files changed, 9 insertions(+), 330 deletions(-) delete mode 100644 .editorconfig delete mode 100644 .eslintrc.json delete mode 100644 dbrenderer.js delete mode 100644 package.json delete mode 100755 prices.js delete mode 100644 render.js delete mode 100644 service/fahrpreise.service delete mode 100644 service/fahrpreise.timer delete mode 100644 testconnections.json delete mode 100644 where.js diff --git a/.editorconfig b/.editorconfig deleted file mode 100644 index 5b16fcc..0000000 --- a/.editorconfig +++ /dev/null @@ -1,14 +0,0 @@ -# editorconfig.org - -root = true - -[*] -end_of_line = lf -charset = utf-8 -trim_trailing_whitespace = true -insert_final_newline = true - -# Use tabs in JavaScript. -[**.{js}] -indent_style = tab -indent_size = 4 diff --git a/.eslintrc.json b/.eslintrc.json deleted file mode 100644 index 1d44c69..0000000 --- a/.eslintrc.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "extends": "eslint:recommended", - "env": { - "commonjs": true, - "es6": true, - "node": true - }, - "parserOptions": { - "ecmaVersion": 2018 - }, - "ignorePatterns": [ - "node_modules" - ], - "rules": { - "no-unused-vars": "off" - } -} diff --git a/dbrenderer.js b/dbrenderer.js deleted file mode 100644 index b43f7b1..0000000 --- a/dbrenderer.js +++ /dev/null @@ -1,35 +0,0 @@ -'use strict' - -const fs = require('fs'); -const zlib = require('zlib'); -const { Readable } = require("stream") - - -function delete_unwanted_data(days) { - for (let day of days) { - for (let journey of day) { - const price = journey.price; - delete price.description; - } - } -} - -const days = (days, from, to, destfolder) => { - const brotli = zlib.createBrotliCompress( { chunkSize: 32 * 1024, - params: { - [zlib.constants.BROTLI_PARAM_MODE]: zlib.constants.BROTLI_MODE_TEXT, - [zlib.constants.BROTLI_PARAM_QUALITY]: 8, - [zlib.constants.BROTLI_PARAM_SIZE_HINT]: 3461903 - }}); - delete_unwanted_data(days); - process.stdout.write(`writing ${days.length} days to db`) - const data = JSON.stringify({queried_at: new Date(), data: days}) - const readable = Readable.from([data]) - let filename = destfolder + Date.now() + "-" + from + "-" + to + '.json.brotli'; - - const out = fs.createWriteStream(filename); - readable.pipe(brotli).pipe(out); - -} - -module.exports = days diff --git a/package.json b/package.json deleted file mode 100644 index 9628dd3..0000000 --- a/package.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "name": "fahrpreise", - "description": "Collect and plot collection prices over time", - "version": "1.0.0", - "files": [ - "render.js", - "where.js", - "prices.js" - ], - "bin": { - "db-prices": "prices.js" - }, - "author": "thigg", - "homepage": "https://github.com/thigg/fahrpreis-plotter", - "repository": "thigg/fahrpreis-plotter", - "bugs": "https://github.com/thigg/fahrpreis-plotter/issues", - "license": "ISC", - "preferGlobal": true, - "engines": { - "node": ">=14" - }, - "keywords": [ - "prices", - "db", - "deutsche", - "bahn", - "german", - "railways", - "fahrpreise" - ], - "dependencies": { - "chalk": "^4.1.2", - "cli-autocomplete": "^0.4", - "cli-table2": "^0.2", - "db-prices": "^3.0.3", - "db-stations": "^4.1.0", - "db-stations-autocomplete": "^3.0.1", - "moment": "^2.14.1", - "mri": "^1.1.0", - "normalize-for-search": "^2.0.1" - }, - "devDependencies": { - "eslint": "^8.0.1", - "@vercel/ncc": "^0.34.0" - }, - "scripts": { - "lint": "eslint .", - "test": "./prices.js 8011160 8000261 --days 1", - "prepublishOnly": "npm run lint && npm test", - "prodbuild": "ncc build prices.js -o dist --target es2021" - } -} diff --git a/prices.js b/prices.js deleted file mode 100755 index 95c4752..0000000 --- a/prices.js +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env node -'use strict' - - -const fsPromises = require('fs/promises') -const mri = require('mri') -const prices = require('db-prices') - -const where = require('./where') -const render = require('./render') -const dbrender = require('./dbrenderer') - - -const argv = mri(process.argv.slice(2), { - boolean: ['help', 'h'] -}) - - -if (argv.help || argv.h) { - process.stdout.write(`\ -Usage: db-prices [from] [to] [options] - -Arguments: - from Station number (e.g. 8011160). - to Station number (e.g. 8000261). - -Options: - --days -d The number of days to show. Default: 7`) - process.exit(0) -} - - -(async () => { - - let connections = [] - if (!argv.connectionfile) { - const from = /[0-9]+/.test(argv._[0]) - ? +argv._[0] - : await where('From where?') - - const to = /[0-9]+/.test(argv._[1]) - ? +argv._[1] - : await where('To where?') - connections[0] = [from, to]; - } else { - const data = await fsPromises.readFile(argv.connectionfile); - connections = JSON.parse(data); - if (connections.length === 0) throw ("input connection file has no connections") - if (connections.find(value => value.length !== 2)) throw("input file connections do not all have exactly 2 elements") - } - const renderer = argv.renderer || "console" - - const now = new Date() - const days = new Array(argv.days || argv.d || 7) - .fill(null, 0, argv.days || argv.d || 7) - .map((_, i) => new Date(now.getFullYear(), now.getMonth(), now.getDate() + i + 1)) - - for (let connection of connections) { - const from = connection[0]; - const to = connection[1]; - const byDay = await Promise.all(days.map(async (when) => { - const res = await prices(from, to, when) - return res.sort((a, b) => a.price.amount - b.price.amount) - })) - if (renderer === "console") - process.stdout.write(render(byDay) + '\n') - else if (renderer === "db") { - dbrender(byDay, from, to, argv.destfolder || "./"); - process.stdout.write(`wrote ${from} to ${to} to db\n`) - } else throw 'unknown renderer ' + renderer - } -})() - .catch((err) => { - console.error(err) - process.exit(1) - }) diff --git a/readme.md b/readme.md index c5c014f..68c62f8 100644 --- a/readme.md +++ b/readme.md @@ -19,29 +19,13 @@ Findings: ![fahpreise plot](images/img.png) ## How to run -I run the service on a raspberry and do the plots from my laptop. I mount the datafolder of the raspberry -to my laptop with `mkdir /tmp/fahrpreise && sshfs raspberry:/var/fahrpreise /tmp/fahrpreise`. -### run service - 1. build this tool with `npm run prodbuild` - 5. create `/var/fahrpreise`, `/opt/fahrpreise` and `/etc/fahrpreise/connections.json` and grant access to the fahrpreise user - 2. move the files from dist to `/opt/fahrpreise/` - 3. create a user fahrpreise - 4. install the files from [service](service) to `/etc/systemd/system/` - 6. create `/etc/fahrpreise/connections.json` with the connections you want to monitor. - Format: - ```json -[ - [start, destination],... -] -``` - - 7. `sudo systemctl daemon-reload && sudo systemctl enable --now fahrpreise` to enable the service - 8. This collects now every few hours (see timer definition file) the connection prices - 9. In /var/fahrpreise are for each capture run the files with date of capture and stations compressed with brotli for minimal filesize +I run the service on a server and create the plots from my laptop. I sync the database file to my laptop for plotting. +### install gatherer service +You can install the gatherer via ansible using this [role](./gatherer_role). +This sets the [node app](./gatherer_hafas) up as a separate user and adds a systemd service and timer ### plot - 1. mount the datafoler (see above) - 2. run `plotter/main.py` (install dependencies before?) - 3. for development of the plotter you can set the accufile parameter so you dont need to accumulate data for every test + 1. get the file + 2. run `plotter/main.py` (install dependencies before?) with required arguments # Contributing For questions/issues use the issue tracker. Feel free to create pull requests with your modifications. @@ -50,7 +34,9 @@ If you created interesting results with this tool and published them, let me kno # License -This is based on [db-prices-cli](https://github.com/derhuerst/db-prices-cli) +This was based on [db-prices-cli](https://github.com/derhuerst/db-prices-cli) and by Jannis R. Thanks go to him for the great base. +Reworked this to work with a fork of [hafas-client](https://github.com/public-transport/hafas-client). thanks to the authos + See the [license file](license.md) for further information. diff --git a/render.js b/render.js deleted file mode 100644 index dae8892..0000000 --- a/render.js +++ /dev/null @@ -1,36 +0,0 @@ -'use strict' - -const chalk = require('chalk') -const moment = require('moment') -const Table = require('cli-table2') - -const table = () => new Table({ - chars: { - top: '', 'top-mid': '', 'top-left': '', 'top-right': '', - bottom: '', 'bottom-mid': '', 'bottom-left': '', 'bottom-right': '', - left: '', 'left-mid': '', mid: '', 'mid-mid': '', - right: '', 'right-mid': '', middle: ' ' - }, - style: {'padding-left': 1, 'padding-right': 0} -}) - -const day = (day) => { - if (!day) return null - return [ - moment(day.legs[0].departure).format('ddd DD'), - day.price.amount + '€', - day.legs - .map((leg) => moment(leg.departure).format('hh:mm')) - .join(' ') - ] -} - -const days = (days) => { - const t = table() - days.flat(1) - .map(day).filter((line) => !!line) - .forEach((line) => t.push(line)) - return t.toString() -} - -module.exports = days diff --git a/service/fahrpreise.service b/service/fahrpreise.service deleted file mode 100644 index cc4bc6a..0000000 --- a/service/fahrpreise.service +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=get fahrpreise - -[Service] -Type=simple -ExecStart=/usr/bin/node /opt/fahrpreise/dist/index.js --connectionfile /etc/fahrpreise/connections.json --renderer db --days 90 --destfolder /var/fahrpreise/ -User=fahrpreise -[Install] -WantedBy=default.target diff --git a/service/fahrpreise.timer b/service/fahrpreise.timer deleted file mode 100644 index c5fcefc..0000000 --- a/service/fahrpreise.timer +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=query fahrpreise -RefuseManualStart=no # Allow manual starts - -[Timer] -Persistent=true -OnBootSec=195 -OnUnitActiveSec=2hr -Unit=fahrpreise.service - -[Install] -WantedBy=timers.target diff --git a/testconnections.json b/testconnections.json deleted file mode 100644 index cdc989a..0000000 --- a/testconnections.json +++ /dev/null @@ -1,5 +0,0 @@ -[ - [8000068,8000050 ], - [8000050,8000068 ] - -] diff --git a/where.js b/where.js deleted file mode 100644 index 05db6da..0000000 --- a/where.js +++ /dev/null @@ -1,51 +0,0 @@ -'use strict' - -const readStations = require('db-stations') -const autocompleteDbStations = require('db-stations-autocomplete') -const promptAutocomplete = require('cli-autocomplete') - -const pStations = new Promise((resolve, reject) => { - const res = Object.create(null) - readStations() - .on('data', (s) => { - res[s.id] = s - }) - .once('end', () => { - resolve(res) - }) - .once('error', reject) -}) - -const suggest = (input) => { - return pStations - .then((stationsById) => { - const results = autocompleteDbStations(input, 5) - const choices = [] - - for (let result of results) { - const station = stationsById[result.id] - if (!station) continue - - choices.push({ - title: [ - station.name, - '–', - 'score:', result.score.toFixed(3), - 'relevance:', result.relevance.toFixed(3), - 'stationcode:', station.id - - ].join(' '), - value: station.id - }) - } - - return choices - }) -} - -const where = (msg) => new Promise((yay, nay) => - promptAutocomplete(msg, suggest) - .once('error', nay) - .once('submit', yay)) - -module.exports = where