Skip to content
This repository has been archived by the owner on Mar 15, 2018. It is now read-only.

Commit

Permalink
backend for financial sales data, currency breakdown (bug 757581)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ngokevin
ngokevin committed Jun 7, 2012
1 parent 3a2dfef commit 304a6ee
Show file tree
Hide file tree
Showing 3 changed files with 155 additions and 67 deletions.
3 changes: 3 additions & 0 deletions mkt/stats/management/commands/index_mkt_stats.py
View file
Expand Up @@ -13,6 +13,7 @@
from stats.models import Contribution
from mkt.stats.tasks import (index_finance_daily,
index_finance_total, index_finance_total_by_src,
index_finance_total_by_currency,
index_installed_daily)

log = logging.getLogger('z.stats')
Expand Down Expand Up @@ -63,6 +64,8 @@ def handle(self, *args, **kw):
{'date': 'created'}),
(Addon.objects, index_finance_total_by_src,
{'date': 'created'}),
(Addon.objects, index_finance_total_by_currency,
{'date': 'created'}),
(Installed.objects, index_installed_daily,
{'date': 'created'}),
]
Expand Down
173 changes: 108 additions & 65 deletions mkt/stats/tasks.py
View file
Expand Up @@ -15,76 +15,120 @@
@task
def index_finance_total(addons, **kw):
"""
Aggregates financial stats from all of the contributions for a given app
Aggregates financial stats from all of the contributions for a given app.
"""
es = elasticutils.get_es()
log.info('Indexing total financial stats for %s apps' %
log.info('Indexing total financial stats for %s apps.' %
len(addons))
try:
for addon in addons:
qs = Contribution.objects.filter(addon__in=addons, uuid=None)
if not qs:
continue

# Get total revenue, sales, refunds per app.
revenue = qs.values('addon').annotate(revenue=Sum('amount'))[0]
sales = qs.values('addon').annotate(sales=Count('id'))[0]
refunds = (qs.filter(refund__isnull=False).
values('addon').annotate(refunds=Count('id')))[0]
data = {
'addon': addon,
'count': sales['sales'],
'revenue': revenue['revenue'],
'refunds': refunds['refunds'],
}

for addon in addons:
qs = Contribution.objects.filter(addon__in=addons, uuid=None)
if not qs.exists():
continue

# Get total revenue, sales, refunds per app.
revenue = qs.values('addon').annotate(revenue=Sum('amount'))[0]
sales = qs.values('addon').annotate(sales=Count('id'))[0]
refunds = (qs.filter(refund__isnull=False).
values('addon').annotate(refunds=Count('id')))[0]
data = {
'addon': addon,
'count': sales['sales'],
'revenue': revenue['revenue'],
'refunds': refunds['refunds'],
}
try:
Contribution.index(data, bulk=True, id=ord_word('tot' +
str(addon)))
es.flush_bulk(forced=True)
except Exception, exc:
index_finance_total.retry(args=[addons], exc=exc)
raise
except Exception, exc:
index_finance_total.retry(args=[addons], exc=exc)
raise


@task
def index_finance_total_by_src(addons, **kw):
"""
Bug 758059
Total finance stats, source breakdown
Total finance stats, source breakdown.
"""
es = elasticutils.get_es()
log.info('Indexing total financial stats by source for %s apps' %
log.info('Indexing total financial stats by source for %s apps.' %
len(addons))
try:
for addon in addons:
qs = Contribution.objects.filter(addon__in=addons, uuid=None)
if not qs:
continue

# Get list of distinct sources.
sources = [src[0] for src in
qs.distinct('source').values_list('source')]

# Get revenue, sales, refunds by source.
for source in sources:
revenues = (qs.filter(source=source).values('addon').
annotate(revenue=Sum('amount'))[0])
sales = (qs.filter(source=source).values('addon').
annotate(sales=Count('id'))[0])
refunds = (qs.filter(source=source, refund__isnull=False).
values('addon').annotate(refunds=Count('id'))[0])
data = {
'addon': addon,
'source': source,
'count': sales['sales'],
'revenue': revenues['revenue'],
'refunds': refunds['refunds'],
}

for addon in addons:
qs = Contribution.objects.filter(addon__in=addons, uuid=None)
if not qs:
continue

# Get list of distinct sources.
sources = [src[0] for src in
qs.distinct('source').values_list('source')]
# Get revenue, sales, refunds by source.
for source in sources:
revenues = (qs.filter(source=source).values('addon').
annotate(revenue=Sum('amount'))[0])
sales = (qs.filter(source=source).values('addon').
annotate(sales=Count('id'))[0])
refunds = (qs.filter(source=source, refund__isnull=False).
values('addon').annotate(refunds=Count('id'))[0])
data = {
'addon': addon,
'source': source,
'count': sales['sales'],
'revenue': revenues['revenue'],
'refunds': refunds['refunds'],
}
try:
Contribution.index(data, bulk=True, id=ord_word('src' +
str(source) + str(addon)))
es.flush_bulk(forced=True)
except Exception, exc:
index_finance_total_by_src.retry(args=[addons], exc=exc)
raise
except Exception, exc:
index_finance_total_by_src.retry(args=[addons], exc=exc)
raise


@task
def index_finance_total_by_currency(addons, **kw):
"""
Bug 757581
Total finance stats, currency breakdown.
"""
es = elasticutils.get_es()
log.info('Indexing total financial stats by currency for %s apps.' %
len(addons))

for addon in addons:
qs = Contribution.objects.filter(addon__in=addons, uuid=None)
if not qs.exists():
continue

# Get list of distinct currencies.
currencies = [currency[0] for currency in
qs.distinct('currency').values_list('currency')]
# Get revenue, sales, refunds by currency.
for currency in currencies:
revenues = (qs.filter(currency=currency).values('addon').
annotate(revenue=Sum('amount'))[0])
sales = (qs.filter(currency=currency).values('addon').
annotate(sales=Count('id'))[0])
refunds = (qs.filter(currency=currency, refund__isnull=False).
values('addon').annotate(refunds=Count('id'))[0])
data = {
'addon': addon,
'currency': currency,
'count': sales['sales'],
'revenue': revenues['revenue'],
'refunds': refunds['refunds'],
}
try:
Contribution.index(data, bulk=True,
id=ord_word('cur' + str(currency).lower()
+ str(addon)))
es.flush_bulk(forced=True)
except Exception, exc:
index_finance_total_by_currency.retry(args=[addons], exc=exc)
raise


@task
Expand All @@ -95,27 +139,26 @@ def index_finance_daily(ids, **kw):
dictionary to not index duplicate contribution with same addon/date
pairs. For each addon-date, it stores the addon in the dict as a top
level key with a dict as its value. And it stores the date in the
addon's dict as a second level key. To check if an addon-date pair has
add-on's dict as a second level key. To check if an addon-date pair has
been already index, it looks up the dict[addon][date] to see if the
key exists.
"""
es = elasticutils.get_es()
qs = (Contribution.objects.filter(id__in=ids)
.order_by('created').values('addon', 'created'))

try:
addons_dates = defaultdict(lambda: defaultdict(dict))
for contribution in qs:
addon = contribution['addon']
date = contribution['created'].strftime('%Y%m%d')

# date for addon not processed, index it and give it key
if not date in addons_dates[addon]:
key = '%s-%s' % (addon, date)
data = search.extract_contributions_daily(contribution)
Contribution.index(data, bulk=True, id=key)
addons_dates[addon][date] = 0
addons_dates = defaultdict(lambda: defaultdict(dict))
for contribution in qs:
addon = contribution['addon']
date = contribution['created'].strftime('%Y%m%d')

try:
# Date for add-on not processed, index it and give it key.
if not date in addons_dates[addon]:
data = search.extract_contributions_daily(contribution)
Contribution.index(data, bulk=True, id=ord_word('finance' +
str(date)))
addons_dates[addon][date] = 0
if qs:
log.info('[%s] Indexing daily financial stats for %s apps' %
(qs[0]['created'], len(addons_dates)))
Expand Down
46 changes: 44 additions & 2 deletions mkt/stats/tests/test_cron.py
View file
Expand Up @@ -70,8 +70,9 @@ def test_index(self):

# Grab document for each source breakdown and compare.
for source in self.sources:
# For some reason, query fails if uppercase letter in filter.
document = (Contribution.search().filter(addon=self.app.pk,
source=source).values_dict('source', 'revenue',
source=source.lower()).values_dict('source', 'revenue',
'count', 'refunds')[0])
document = {'count': document['count'],
'revenue': int(document['revenue']),
Expand All @@ -82,6 +83,47 @@ def test_index(self):
eq_(document, self.expected[source])


class TestIndexFinanceTotalByCurrency(amo.tests.ESTestCase):

def setUp(self):
self.app = amo.tests.app_factory()

self.currencies = ['CAD', 'USD', 'EUR']
self.expected = {
'CAD': {'revenue': 0, 'count': 3, 'refunds': 1},
'USD': {'revenue': 0, 'count': 4, 'refunds': 1},
'EUR': {'revenue': 0, 'count': 2, 'refunds': 1}
}
for currency in self.currencies:
# Create sales.
for x in range(self.expected[currency]['count']):
c = Contribution.objects.create(addon_id=self.app.pk,
currency=currency, amount=str(random.randint(0, 10) + .99))
self.expected[currency]['revenue'] += c.amount
# Create refunds.
Refund.objects.create(contribution=c,
status=amo.REFUND_APPROVED)
self.refresh()

def test_index(self):
tasks.index_finance_total_by_currency([self.app.pk])
self.refresh(timesleep=1)

# Grab document for each source breakdown and compare.
for currency in self.currencies:
# For some reason, query fails if uppercase letter in filter.
document = (Contribution.search().filter(addon=self.app.pk,
currency=currency.lower()).values_dict('currency',
'revenue', 'count', 'refunds')[0])
document = {'count': document['count'],
'revenue': int(document['revenue']),
'refunds': document['refunds']}
self.expected[currency]['revenue'] = (
int(self.expected[currency]['revenue'])
)
eq_(document, self.expected[currency])


class TestIndexFinanceDaily(amo.tests.ESTestCase):

def setUp(self):
Expand Down Expand Up @@ -109,7 +151,7 @@ def test_index(self):
self.refresh(timesleep=1)

document = Contribution.search().filter(addon=self.app.pk
).values_dict('date', 'revenue', 'count', 'refunds')
).values_dict('date', 'revenue', 'count', 'refunds')[0]

date = document['date']
ex_date = self.expected['date']
Expand Down

0 comments on commit 304a6ee

Please sign in to comment.