Skip to content

Commit

Permalink
Merge pull request #138 from fau-fablab/improve-kassenbuch-speed
Browse files Browse the repository at this point in the history 
Improve kassenbuch speed
  • Loading branch information
@patkan
patkan committed Jun 21, 2016
2 parents 3c8a7cf + 321cb9d commit ca065a8
Show file tree
Hide file tree
Showing 2 changed files with 115 additions and 16 deletions.
75 changes: 60 additions & 15 deletions FabLabKasse/kassenbuch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,52 @@ def __init__(self, sqlite_file=':memory:'):
cur.execute(
"CREATE TABLE IF NOT EXISTS statistik(id INTEGER PRIMARY KEY AUTOINCREMENT, datum, gruppe, user, rechnung INT, betrag)")

# search indexes for faster execution
cur.execute("CREATE INDEX IF NOT EXISTS buchungDateIndex ON buchung(datum)")
cur.execute("CREATE INDEX IF NOT EXISTS buchungRechnungIndex ON buchung(rechnung)")
cur.execute("CREATE INDEX IF NOT EXISTS rechnungDateIndex ON rechnung(datum)")
cur.execute("CREATE INDEX IF NOT EXISTS positionRechnungIndex ON position(rechnung)")
cur.execute("CREATE INDEX IF NOT EXISTS bargeldDateIndex ON bargeld(datum)")
cur.execute("CREATE INDEX IF NOT EXISTS kundenbuchungDateIndex ON kundenbuchung(datum)")
cur.execute("CREATE INDEX IF NOT EXISTS kundenbuchungKundeIndex ON kundenbuchung(kunde)")
cur.execute("CREATE INDEX IF NOT EXISTS statistikDateIndex ON statistik(datum)")
cur.execute("CREATE INDEX IF NOT EXISTS statistikRechnungIndex ON statistik(rechnung)")

@staticmethod
def _date_query_generator(from_table=None, from_date=None, until_date=None):
"""
returns a string for a SQL query to rechnung or buchung
:param from_table: which table should be queried
:param from_date: datetime start date (included)
:param until_date: datetime end date (not included)
:type from_date: datetime.datetime | None
:type until_date: datetime.datetime | None
:return: query string
"""
# TODO comparing against these strings might make problems with py3
# --> best import unicode_literals from future
# --> check whole file if this import is problematic
known_tables = ["buchung", "rechnung"]
if from_table not in known_tables:
raise NotImplementedError("unimplemented table {0}".format(from_table))

query = "SELECT id FROM {0}".format(from_table)
if from_date and until_date:
query = query + " WHERE datum >= Datetime('{from_date}') AND datum < Datetime('{until_date}')".format(
from_date=from_date, until_date=until_date
)
elif from_date:
query = query + " WHERE datum >= Datetime('{from_date}')".format(
from_date=from_date
)
elif until_date:
query = query + " WHERE datum < Datetime('{until_date}')".format(
until_date=until_date
)

return query

@property
def buchungen(self):
return self.get_buchungen()
Expand All @@ -355,34 +401,33 @@ def get_buchungen(self, from_date=None, until_date=None):
"""
buchungen = []

self.cur.execute("SELECT id FROM buchung")
query = Kasse._date_query_generator(from_table="buchung", from_date=from_date, until_date=until_date)
self.cur.execute(query)
for row in self.cur.fetchall():
buchungen.append(Buchung.load_from_id(row[0], self.cur))

# TODO move filters to SQL query
if from_date:
buchungen = [b for b in buchungen if b.datum >= from_date]
if until_date:
buchungen = [b for b in buchungen if b.datum < until_date]

return buchungen

@property
def rechnungen(self):
return self.get_rechnungen()

def get_rechnungen(self, von=None, bis=None):
def get_rechnungen(self, from_date=None, until_date=None):
"""
get invoices between the given dates. If a date is ``None``, no filter will be applied.
:param from_date: start datetime (included)
:param until_date: end datetime (not included)
:type from_date: datetime.datetime | None
:type until_date: datetime.datetime | None
"""
rechnungen = []

self.cur.execute("SELECT id FROM rechnung")
query = Kasse._date_query_generator(from_table="rechnung", from_date=from_date, until_date=until_date)
self.cur.execute(query)
for row in self.cur.fetchall():
rechnungen.append(Rechnung.load_from_id(row[0], self.cur))

if von:
rechnungen = [r for r in rechnungen if r.datum >= von]
if bis:
rechnungen = [r for r in rechnungen if r.datum < bis]

return rechnungen

@property
Expand Down Expand Up @@ -1387,7 +1432,7 @@ def check_number_greater_zero_or_minus_one(n):

for k in k.kunden:
letzte_zahlung = sorted(
[b.datum for b in [b for b in k.buchungen if b.betrag > 0]]
(b.datum for b in (b for b in k.buchungen if b.betrag > 0))
)[:1]

if not letzte_zahlung:
Expand Down
56 changes: 55 additions & 1 deletion FabLabKasse/test_kassenbuch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,11 @@
"""unittests for kassenbuch.py"""

import unittest
from FabLabKasse.kassenbuch import Kasse, Kunde, NoDataFound, parse_args
from FabLabKasse.kassenbuch import Kasse, Kunde, Buchung, Rechnung, NoDataFound, parse_args
from FabLabKasse.kassenbuch import argparse_parse_date, argparse_parse_currency
from hypothesis import given
from hypothesis.strategies import text
import hypothesis.extra.datetime as hypothesis_datetime
import dateutil
from datetime import datetime, timedelta
from decimal import Decimal
Expand Down Expand Up @@ -95,3 +96,56 @@ def test_accounting_database_client_creation(self, clientname):
self.fail("client entry in database has not been created")
# TODO test integrity checking (no double creation of same ID)
# TODO code crashes when reading Kunde with "None" in e.g. schuldengrenze

@given(from_date=hypothesis_datetime.datetimes(), until_date=hypothesis_datetime.datetimes())
def test_datestring_generator(self, from_date, until_date):
"""test the datestring_generator in Kasse"""
query = Kasse._date_query_generator('buchung', from_date=from_date, until_date=until_date)
pristine_query = "SELECT id FROM buchung WHERE datum >= Datetime('{from_date}') AND " \
"datum < Datetime('{until_date}')".format(from_date=from_date, until_date=until_date)
self.assertEqual(query, pristine_query)
query = Kasse._date_query_generator('buchung', until_date=until_date)
pristine_query = "SELECT id FROM buchung WHERE datum < Datetime('{until_date}')".format(until_date=until_date)
self.assertEqual(query, pristine_query)
query = Kasse._date_query_generator('buchung', from_date=from_date)
pristine_query = "SELECT id FROM buchung WHERE datum >= Datetime('{from_date}')".format(from_date=from_date)
self.assertEqual(query, pristine_query)

@given(rechnung_date=hypothesis_datetime.datetimes(min_year=1900, timezones=[]),
from_date=hypothesis_datetime.datetimes(min_year=1900, timezones=[]),
until_date=hypothesis_datetime.datetimes(min_year=1900, timezones=[]))
def test_get_rechnungen(self, rechnung_date, from_date, until_date):
"""test the get_rechnungen function"""
kasse = Kasse(sqlite_file=':memory:')
rechnung = Rechnung(datum=rechnung_date.strftime('%Y-%m-%d %H:%M:%S.%f'))
rechnung.store(kasse.cur)
kasse.con.commit()

query = kasse.get_rechnungen(from_date, until_date)
if from_date <= rechnung_date < until_date:
self.assertTrue(query)
else:
self.assertFalse(query)

@given(buchung_date=hypothesis_datetime.datetimes(min_year=1900, timezones=[]),
from_date=hypothesis_datetime.datetimes(min_year=1900, timezones=[]),
until_date=hypothesis_datetime.datetimes(min_year=1900, timezones=[]))
def test_get_buchungen(self, buchung_date, from_date, until_date):
"""test the get_buchungen function"""
kasse = Kasse(sqlite_file=':memory:')
rechnung = Rechnung(datum=buchung_date.strftime('%Y-%m-%d %H:%M:%S.%f'))
rechnung.store(kasse.cur)
buchung = Buchung(konto='somewhere',
betrag='0',
rechnung=rechnung.id,
kommentar="Passing By And Thought I'd Drop In",
datum=buchung_date.strftime('%Y-%m-%d %H:%M:%S.%f'))
buchung._store(kasse.cur)
kasse.con.commit()

#TODO load_from_row ist sehr anfällig gegen kaputte datetimes, das sollte am besten schon sauber in die Datenbank
query = kasse.get_buchungen(from_date, until_date)
if from_date <= buchung_date < until_date:
self.assertTrue(query)
else:
self.assertFalse(query)

0 comments on commit ca065a8

Please sign in to comment.