* Layout change

 * Working on Transactions listing and history averages

src/app.yaml

@@ -9,6 +9,8 @@ libraries:
   version: latest
 - name: markupsafe
   version: latest
+- name: numpy
+  version: latest
 
 derived_file_type:
 - python_precompiled

src/index.yaml

@@ -15,3 +15,18 @@ indexes:
   - name: character_key
   - name: td
     direction: desc
+
+- kind: WalletTransaction
+  properties:
+  - name: character_key
+  - name: tt
+  - name: td
+    direction: desc
+
+- kind: WalletTransaction
+  properties:
+  - name: character_key
+  - name: tt
+  - name: tyi
+  - name: td
+    direction: desc

src/maeve/character_view.py

@@ -3,7 +3,7 @@
 from maeve.web import BaseHandler, profile_required
 from maeve.settings import webapp2_config
 from maeve.utils import is_prod_environment
-from maeve.models import Account, Character
+from maeve.models import Account, Character, WalletTransaction
 from google.appengine.ext.ndb import toplevel
 from google.appengine.api import users
 import webapp2
@@ -18,7 +18,8 @@ def get(self, char_id):
 
     if char:
       env.update(dict(character=char,
-                      account=char.account))
+                      account=char.account,
+                      current='dashboard'))
 
       if char.active:
         pass
@@ -47,9 +48,30 @@ def post(self, char_id, action):
       self.session.add_flash('No character with that id found', key='top_messages', level='warning')
       self.redirect('/profile')
 
+
+class CharacterTransactionsHandler(BaseHandler):
+
+  def get(self, char_id):
+    env = {}
+    char = Character.by_char_id(char_id)
+    if char:
+      env.update(dict(character=char,
+                      account=char.account,
+                      current='transactions',
+                      WalletTransaction=WalletTransaction))
+
+      if char.active:
+        pass
+
+      self.render_response('character/transactions.html', env)
+    else:
+      self.session.add_flash('No character with that id found', key='top_messages')
+      self.redirect('/profile')
+
 app = webapp2.WSGIApplication([
                                 (r'/character/(\d+)/?$', CharacterHandler),
                                 (r'/character/(\d+)/(activate)/?$', CharacterActivationHandler),
+                                (r'/character/(\d+)/transactions/?$', CharacterTransactionsHandler),
                               ],
                               debug=(not is_prod_environment()),
                               config=webapp2_config

src/maeve/models.py

@@ -140,4 +140,24 @@ class ItemTypeIndex(model.Model):
   @classmethod
   def get(cls, item_id):
     idx = ItemTypeIndex.query().get()
-    return idx.items or {}
+    return (idx.items or {}).get(item_id, None)
+
+  def find(cls, name_part):
+    import re
+    name_re = re.compile('^.*?{0}.*$'.format(name_part.lower()), flags=re.IGNORECASE)
+    idx = ItemTypeIndex.query().get()
+    matches = ([(i, n) for i, n in idx.items.iteritems() if name_re.match(n)])
+    return matches
+
+
+class ItemStats(model.Model):
+  user = model.UserProperty('u', required=True)
+  character_key = model.KeyProperty('ck', kind='Character', required=True)
+  char_id = model.StringProperty('cid', required=True)
+  type_id = model.StringProperty('iid', required=True)
+  accumulated_cost = model.FloatProperty('ac', default=0, indexed=False)
+  accumulated_earnings = model.FloatProperty('ae', default=0, indexed=False)
+  items_sold = model.IntegerProperty('is', default=0)
+  items_bought = model.IntegerProperty('ib', default=0)
+  roi_yield = model.FloatProperty('ry', default=0)
+  avg_roi_yield = model.FloatProperty('ary', default=0)

src/maeve/statistics.py

@@ -1,16 +1,24 @@
 # -*- coding: UTF-8 -*-
 
-from maeve.web import BaseHandler, profile_required
-from maeve.settings import webapp2_config
-from maeve.utils import is_prod_environment
 from maeve.models import Character, WalletTransaction
-from google.appengine.ext.ndb import toplevel
-from google.appengine.api import users
+from maeve.utils import price_fmt
 from datetime import datetime
+import logging
+
+
+def extend_transactions_query_result(result):
+  results_as_dct = [r.to_dict() for r in result]
+  for dct in results_as_dct:
+    dct['balance_change'] = dct['unit_price'] * dct['quantity']
+    if dct['transaction_type'] == WalletTransaction.BUY:
+      dct['balance_change'] *= -1
+
+    dct['balance_change_str'] = price_fmt(dct['balance_change'])
+    dct['unit_price_str'] = price_fmt(dct['unit_price'])
 
 
 def get_filtered_transactions(character, filters):
-  transaction_type = filters.get('type', None)
+  transaction_type = filters.get('transaction_type', None)
   if 'limit' in filters:
     limit = filters['limit']
   else:
@@ -22,6 +30,12 @@ def get_filtered_transactions(character, filters):
   query = WalletTransaction.query(WalletTransaction.character_key == character.key)
 
   if transaction_type:
+    if type(transaction_type) in (str, unicode):
+      if transaction_type.lower() == 'sell':
+        transaction_type = WalletTransaction.SELL
+      elif transaction_type.lower() == 'buy':
+        transaction_type = WalletTransaction.BUY
+
     query = query.filter(WalletTransaction.transaction_type == int(transaction_type))
 
   if 'type_id' in filters:

src/maeve/statistics_view.py

@@ -2,13 +2,14 @@
 
 from maeve.web import BaseHandler, profile_required
 from maeve.settings import webapp2_config
-from maeve.utils import is_prod_environment, GenericModelEncoder, price_fmt
-from maeve.models import Character, WalletTransaction
-from maeve.statistics import get_filtered_transactions
+from maeve.utils import is_prod_environment, GenericModelEncoder, price_fmt, to_jstime
+from maeve.models import Character, WalletTransaction, ItemTypeIndex
+from maeve.statistics import get_filtered_transactions, extend_transactions_query_result
 from google.appengine.ext.ndb import toplevel
 from google.appengine.api import users
 import webapp2
 import json
+import numpy
 
 
 class TransactionsHandler(BaseHandler):
@@ -30,9 +31,56 @@ def get(self):
     self.render_json(results_as_dct, cls=GenericModelEncoder)
 
 
+class TransactionsAverageHandler(BaseHandler):
+
+  def get(self):
+    try:
+      char_id = self.request.get('char', None)
+      type_id = self.request.get('type_id', None)
+      quantity = int(self.request.get('quantity', 10))
+      transaction_type = int(self.request.get('transaction_type', WalletTransaction.BUY))
+
+      result = get_filtered_transactions(Character.by_char_id(char_id),
+                                         filters=dict(transaction_type=transaction_type,
+                                                      type_id=type_id,
+                                                      limit=quantity))
+
+      oldest_date, prices, i = None, [], 0
+      for t in result:
+        if i > quantity:
+          break
+
+        oldest_date = t.transaction_date
+        for j in range(t.quantity):
+          prices.append(t.unit_price)
+          i += 1
+          if i > quantity:
+            break
+
+      self.render_json(dict(prices=prices,
+                            oldest_date=to_jstime(oldest_date),
+                            median=numpy.median(prices),
+                            mean=numpy.mean(prices)))
+
+    except:
+      import traceback
+      import logging
+      logging.error(traceback.format_exc())
+      self.render_json(dict(error='Bad values'))
+
+
+class SeachCommodityHandler(BaseHandler):
+
+  def get(self):
+    query = self.request.get('query', '')
+    index = ItemTypeIndex.query().get()
+
+    self.render_json(dict(matches=index.find(query)))
 
 app = webapp2.WSGIApplication([
-                                (r'/stat/transactions?$', TransactionsHandler),
+                                (r'/stat/commodity/search/?$', SeachCommodityHandler),
+                                (r'/stat/transactions/?$', TransactionsHandler),
+                                (r'/stat/transactions/average/?$', TransactionsAverageHandler),
                               ],
                               debug=(not is_prod_environment()),
                               config=webapp2_config

src/maeve/task_view.py

@@ -31,9 +31,10 @@ def post(self):
     if character and character.active:
       account = character.account_key.get()
       items = index_character(character, account)
-      taskqueue.add(url='/_task/index',
-                    params={'values': json.dumps(items)},
-                    queue_name='index-update')
+      if items:
+        taskqueue.add(url='/_task/index',
+                      params={'values': json.dumps(items)},
+                      queue_name='index-update')
 
 
 class IndexTaskHandler(BaseHandler):

src/maeve/tasks.py

@@ -1,7 +1,7 @@
 # -*- coding: UTF-8 -*-
 
 from maeve.api import Api
-from maeve.models import Character, Account, WalletTransaction, MarketOrder, ItemTypeIndex
+from maeve.models import Character, Account, WalletTransaction, MarketOrder, ItemTypeIndex, ItemStats
 from google.appengine.ext import ndb
 from datetime import datetime
 from google.appengine.api import taskqueue
@@ -49,58 +49,75 @@ def index_all_characters():
     task_count += 1
     taskqueue.add(url='/_task/sync',
                   params={'char': character.key.urlsafe()},
-                  queue_name='transaction-sync')
+                  queue_name='transaction-sync',
+                  )
 
   logging.info('{0} sync tasks enqueued'.format(task_count))
 
 
 def index_character(character, account):
 
-  logging.info('Synching: Character {0} / {1}'.format(character.name, character.char_id))
-  orders = MarketOrder.query(MarketOrder.character_key == character.key).fetch_async()
+  try:
+    logging.info('Synching: Character {0} / {1}'.format(character.name, character.char_id))
+    item_stats = ItemStats.query(ItemStats.character_key == character.key).fetch_async()
+    orders = MarketOrder.query(MarketOrder.character_key == character.key).fetch_async()
 
-  api = Api(account.api_id, account.api_vcode)
-  api.authenticate()
-  api_char = api.get_character(character.char_id)
+    api = Api(account.api_id, account.api_vcode)
+    api.authenticate()
+    api_char = api.get_character(character.char_id)
 
-  row_count = 250
-  all_items = {}
+    row_count = 250
+    all_items = {}
 
-  character.last_update = datetime.now()
+    character.last_update = datetime.now()
 
-  last_transaction_id = character.last_transaction_id
-  last_transaction_date = character.last_transaction_date
+    last_transaction_id = character.last_transaction_id
+    last_transaction_date = character.last_transaction_date
 
-  api_wallet_transactions = api_char.WalletTransactions(rowCount=(last_transaction_id is None and 1000 or row_count))
-  newest_transaction, oldest_transaction, items = sync_transactions(character,
-                                                             api_wallet_transactions,
-                                                             last_transaction_id,
-                                                             last_transaction_date)
+    api_wallet_transactions = api_char.WalletTransactions(rowCount=(last_transaction_id is None and 1000 or row_count))
+    item_stats = dict([(i.type_id, i) for i in item_stats.get_result()])
 
-  all_items.update(items or {})
-
-  while last_transaction_id and last_transaction_date and oldest_transaction and \
-  (datetime.fromtimestamp(oldest_transaction.transactionDateTime) > last_transaction_date or oldest_transaction.transactionID > last_transaction_id):
-    logging.info('Fetching another batch from id {0}'.format(oldest_transaction.transactionID))
-
-    api_wallet_transactions = api_char.WalletTransactions(rowCount=row_count, fromID=oldest_transaction.transactionID)
     newest_transaction, oldest_transaction, items = sync_transactions(character,
                                                                api_wallet_transactions,
                                                                last_transaction_id,
-                                                               last_transaction_date)
+                                                               last_transaction_date,
+                                                               item_stats)
 
     all_items.update(items or {})
 
-  sync_orders(character,
-              api_char.MarketOrders(),
-              orders.get_result())
+    while last_transaction_id and last_transaction_date and oldest_transaction and \
+    (datetime.fromtimestamp(oldest_transaction.transactionDateTime) > last_transaction_date or oldest_transaction.transactionID > last_transaction_id):
+      logging.info('Fetching another batch from id {0}'.format(oldest_transaction.transactionID))
+
+      api_wallet_transactions = api_char.WalletTransactions(rowCount=row_count, fromID=oldest_transaction.transactionID)
+      newest_transaction, oldest_transaction, items = sync_transactions(character,
+                                                                 api_wallet_transactions,
+                                                                 last_transaction_id,
+                                                                 last_transaction_date,
+                                                                 item_stats)
+
+      all_items.update(items or {})
 
-  character.put_async()
-  logging.info('Syncing done: Character {0} / {1}'.format(character.name, character.char_id))
-  return all_items
+    sync_orders(character,
+                api_char.MarketOrders(),
+                orders.get_result())
 
+    character.put_async()
+    logging.info('Syncing done: Character {0} / {1}'.format(character.name, character.char_id))
+    return all_items
+  except:
+    import traceback
+    logging.error('Error while syncing character {0} / {1}'.format(character.name, character.char_id))
+    logging.error(traceback.format_exc())
+    return None
+
+
+def sync_transactions(character,
+                      api_wallet_transactions,
+                      last_transaction_id,
+                      last_transaction_date,
+                      item_stats):
 
-def sync_transactions(character, api_wallet_transactions, last_transaction_id, last_transaction_date):
   newest_transaction, oldest_transaction, items = None, None, {}
   to_put = []
 
@@ -123,6 +140,29 @@ def sync_transactions(character, api_wallet_transactions, last_transaction_id, l
 
       to_put.append(wt)
       items[wt.type_id] = wt.type_name
+
+      stats = item_stats.get(wt.type_id, None)
+      abs_balance_change = wt.quantity * wt.unit_price
+
+      if not stats:
+        stats = ItemStats(user=character.user,
+                          char_id=character.char_id,
+                          character_key=character.key,
+                          type_id=wt.type_id,
+                          accumulated_cost=(wt.transaction_type == WalletTransaction.BUY and abs_balance_change or 0),
+                          accumulated_earnings=(wt.transaction_type == WalletTransaction.SELL and abs_balance_change or 0),
+                          items_bought=(wt.transaction_type == WalletTransaction.BUY and wt.quantity or 0),
+                          items_sold=(wt.transaction_type == WalletTransaction.SELL and wt.quantity or 0),
+                          )
+        item_stats[wt.type_id] = stats
+      else:
+        if wt.transaction_type == WalletTransaction.BUY:
+          stats.accumulated_cost += abs_balance_change
+          stats.items_bought += wt.quantity
+        else:
+          stats.accumulated_earnings += abs_balance_change
+          stats.items_sold += wt.quantity
+
     else:
       logging.debug('Skipped transaction {0}'.format(row.transactionID))
 
@@ -138,6 +178,14 @@ def sync_transactions(character, api_wallet_transactions, last_transaction_id, l
 
   ndb.put_multi_async(to_put)
 
+  for stats in item_stats.values():
+    stats.roi_yield = stats.accumulated_cost > 0 and (stats.accumulated_earnings - stats.accumulated_cost) / stats.accumulated_cost or 0
+    avg_unit_cost = stats.items_bought > 0 and stats.accumulated_cost / stats.items_bought or 0
+    avg_unit_earnings = stats.items_sold > 0 and stats.accumulated_earnings / stats.items_sold or 0
+    stats.avg_roi_yield = avg_unit_cost > 0 and (avg_unit_earnings - avg_unit_cost) / avg_unit_cost or 0
+
+  ndb.put_multi_async(item_stats.values())
+
   return newest_transaction, oldest_transaction, items