Cleanup of daemon package:

* remove unuused imports other `pyflakes` fixes
* better logging: streamhandler, syslog
* neater handling of "global" data. no more global `DIR_PATH`
* don't bail out on non-200 status. log and return empty results
* use config directory `~/.coincharts` storing api auth and history symbols

coincharts/daemon/base.py

@@ -5,29 +5,33 @@
import os
import sys

# FIXME
# beautiful HACK until I get some stuff figured out.
sys.path.insert(0, '../mutils')

import datetime
import pytz
from dateutil.parser import parse as parse_dt
import json
import urllib
import requests
import logging
import logging.handlers
import time

import yaml
from sqlalchemy import Integer, String, Float
import daemon
import daemon.pidfile

from mutil import simple_alchemy
from mutils import simple_alchemy

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

DIR_PATH = None
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
logger.addHandler(ch)


CONFIG_DIR = os.path.expanduser('~/.coincharts')
API_KEY_FILE = os.path.join(CONFIG_DIR, 'API_KEY')
CONFIG_FILE = os.path.join(CONFIG_DIR, 'config.yaml')


class PriceSeries(object):
@@ -40,7 +44,7 @@ class PriceSeries(object):
    )
    date_format_template = '%Y-%m-%dT%H:%M:%S.%f0Z'  # magic
    headers = {'X-CoinAPI-Key':
               open('API_KEY').read().strip()}
               open(API_KEY_FILE).read().strip()}
    schema = [
        ('time_period_start', 'TEXT'),
        ('time_period_end', 'TEXT'),
@@ -63,11 +67,12 @@ class PriceSeries(object):

    _session = None

    # the sqlalchemy is a class singleton, so many symbols can share a connection.
    @classmethod
    def get_db_session(cls):
    def get_db_session(cls, dir_path):
        if cls._session is not None:
            return cls._session
        db_path = os.path.join(os.path.abspath(DIR_PATH), 'db.sqlite3')
        db_path = os.path.join(os.path.abspath(dir_path), 'db.sqlite3')
        cls._session = simple_alchemy.get_session(db_path)
        return cls._session

@@ -92,9 +97,10 @@ def round_up_hour(cls, dt):
        dt = dt.replace(minute=0, second=0, microsecond=0)
        return dt

    def __init__(self, symbol_id):
    def __init__(self, symbol_id, dir_path):
        logger.debug('creating PriceSeries object for {}'.format(symbol_id))
        self.symbol_id = symbol_id
        self.dir_path = dir_path
        schema = [
            ('time_period_start', String),
            ('time_period_end', String),
@@ -118,11 +124,11 @@ def get_prices_since(self, start_dt):
            start_dt = parse_dt(start_dt)
        except TypeError:
            pass
        start_dt = self.get_normalized_datetime(self.round_up(start_dt))
        start_dt = self.get_normalized_datetime(self.round_up_hour(start_dt))
        kwargs = {self.TIME: start_dt}
        session = self.get_db_session()
        session = self.get_db_session(self.dir_path)
        first = session.query(self.data).filter_by(**kwargs).first()
        results = session.query(self.data).filter(id >= start_dt).first()
        results = session.query(self.data).filter(id >= first).first()
        return results

    def get_url(self, query_data):
@@ -162,12 +168,15 @@ def fetch(self):
        url = self.get_url(query_data)
        logger.debug('getting url {}'.format(url))
        response = requests.get(url, headers=self.headers)
        if response.status_code != 200:
            logger.error('request {} failed: {}'.format(url, response.reason))
            return {}
        logger.info('account has {} more API requests for this time period'.format(
            response.headers['X-RateLimit-Remaining']))
        return response.json()

    def get_last_date_from_store(self):
        session = self.get_db_session()
        session = self.get_db_session(self.dir_path)
        obj = session.query(self.data).order_by(self.data.id.desc()).first()
        if obj is None:
            return parse_dt(self.first_date)
@@ -179,7 +188,7 @@ def insert(self, data):
        insertions = []
        for row in data:
            insertions.append(self.data(**row))
        session = self.get_db_session()
        session = self.get_db_session(self.dir_path)
        session.add_all(insertions)
        session.commit()

@@ -188,29 +197,24 @@ def update(self):
        self.insert(data)


def main(dir_path, daemon=True):

    global DIR_PATH
    DIR_PATH = dir_path
def worker(dir_path, daemonize=True):

    fh = logging.FileHandler(os.path.join(dir_path, 'logs'))
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    fh.setFormatter(formatter)
    fh.setLevel(logging.DEBUG)
    logger.addHandler(fh)

    symbols = [
        'BITSTAMP_SPOT_BTC_USD',
        'BITSTAMP_SPOT_XRP_USD',
        'BITSTAMP_SPOT_ETH_USD',
        'BITSTAMP_SPOT_LTC_USD',
        'BITSTAMP_SPOT_EUR_USD',
        'BITSTAMP_SPOT_BCH_USD'
    ]
    sh = logging.handlers.SysLogHandler(address='/var/run/syslog')
    sh.setLevel(logging.DEBUG)
    logger.addHandler(sh)

    with open(CONFIG_FILE) as f:
        config = yaml.load(f)

    series = []
    for symbol_id in symbols:
        series.append(PriceSeries(symbol_id))
    for symbol_id in config['history_symbols']:
        series.append(PriceSeries(symbol_id, dir_path))

    while True:
        for ps in series:
@@ -219,8 +223,50 @@ def main(dir_path, daemon=True):
        time.sleep(3600)


if __name__ == '__main__':
def main():

    script_basename, _ = os.path.splitext(os.path.basename(__file__))

    # no need to involve argparse for something this simple
    if len(sys.argv) == 1:
        print('usage: {} [--daemon] <directory>'.format(script_basename))
        sys.exit(1)

    from mutils.simple_daemon import daemonize
    daemonize = True
    if '--daemon' in sys.argv:
        script_name, _, dir_path = sys.argv
    else:
        script_name, dir_path = sys.argv
        daemonize = False

    dir_path = os.path.abspath(dir_path)

    if daemonize:
        # when I'm a daemon, log all exceptions
        def exception_handler(type_, value, tb):
            logger.exception('uncaught exception on line {}; {}: {}'.format(
                tb.tb_lineno,
                type_.__name__,
                value,
            ))
            sys.__excepthook__(type_, value, tb)
        sys.excepthook = exception_handler

    logger.debug('starting daemon {} using path {}'.format(script_name, dir_path))

    if daemonize:
        pid_file = os.path.join(dir_path, script_basename + '.pid')

        with daemon.DaemonContext(
                working_directory=dir_path,
                pidfile=daemon.pidfile.PIDLockFile(pid_file),

        ):
            worker(dir_path, daemonize=daemonize)
    else:
        worker(dir_path, daemonize=daemonize)


if __name__ == '__main__':

    daemonize(main)
    main()