Skip to content

Usage report updated for analyses as collections #930

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Oct 11, 2017
Merged

Usage report updated for analyses as collections #930

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
455c808
usage report updated for anlaysis containers
hkethi002 Sep 18, 2017
43ce64b
analysis input file size not counted
hkethi002 Sep 20, 2017
1e63c37
removed analysis aggregation on parent containers
hkethi002 Sep 20, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 9 additions & 27 deletions api/handlers/reporthandler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -872,25 +872,25 @@ def _build_project_report(self, base_query):
report_obj = self._create_default(project=p)

# Grab sessions and their ids
sessions = config.db.sessions.find({'project': p['_id']}, {'_id': 1, 'analyses':1})
sessions = config.db.sessions.find({'project': p['_id']}, {'_id': 1})
session_ids = [s['_id'] for s in sessions]

# Grab acquisitions and their ids
acquisitions = config.db.acquisitions.find({'session': {'$in': session_ids}}, {'_id': 1, 'analyses':1})
acquisitions = config.db.acquisitions.find({'session': {'$in': session_ids}}, {'_id': 1})
acquisition_ids = [a['_id'] for a in acquisitions]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can remove the projection for analyses above.


# For the project and each session and acquisition, create a list of analysis ids
analysis_ids = [an['_id'] for an in p.get('analyses', [])]
analysis_ids.extend([an['_id'] for an in s.get('analyses', []) for s in sessions])
analysis_ids.extend([an['_id'] for an in a.get('analyses', []) for a in acquisitions])

parent_ids = session_ids + acquisition_ids + [p['_id']]
analysis_ids = [an['_id'] for an in config.db.analyses.find({'parent.id': {'$in': parent_ids}})]

report_obj['session_count'] = len(session_ids)

# for each type of container below it will have a slightly modified match query
cont_query = {
'projects': {'_id': {'project': p['_id']}},
'sessions': {'project': p['_id']},
'acquisitions': {'session': {'$in': session_ids}}
'acquisitions': {'session': {'$in': session_ids}},
'analyses': {'parent.id' : {'$in':parent_ids}}
}

# Create queries for files and analyses based on created date if a range was provided
Expand All @@ -901,32 +901,14 @@ def _build_project_report(self, base_query):
file_q['files.created'] = base_query['created']
analysis_q['analyses.created'] = base_query['created']

for cont_name in ['projects', 'sessions', 'acquisitions']:
for cont_name in ['projects', 'sessions', 'acquisitions', 'analyses']:

# Aggregate file size in megabytes
pipeline = [
{'$match': cont_query[cont_name]},
{'$unwind': '$files'},
{'$match': file_q},
{'$project': {'mbs': {'$divide': ['$files.size', BYTES_IN_MEGABYTE]}}},
{'$group': {'_id': 1, 'mb_total': {'$sum':'$mbs'}}}
]

try:
result = self._get_result(config.db.command('aggregate', cont_name, pipeline=pipeline))
except APIReportException:
result = None

if result:
report_obj['file_mbs'] += result['mb_total']

# Aggregate analysis file size in megabytes
pipeline = [
{'$match': cont_query[cont_name]},
{'$unwind': '$analyses'},
{'$unwind': '$analyses.files'},
{'$match': analysis_q},
{'$project': {'mbs': {'$divide': ['$analyses.files.size', BYTES_IN_MEGABYTE]}}},
{'$project': {'mbs': {'$divide': [{'$cond': ['$files.input', 0, '$files.size']}, BYTES_IN_MEGABYTE]}}},
{'$group': {'_id': 1, 'mb_total': {'$sum':'$mbs'}}}
]

Expand Down
14 changes: 14 additions & 0 deletions test/integration_tests/python/test_reports.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,3 +284,17 @@ def test_usage_report(data_builder, file_form, as_user, as_admin):
assert usage[0]['session_count'] == 1
assert usage[0]['file_mbs'] > 0
assert usage[0]['gear_execution_count'] == 1

# Test if empty project breaks Usage report
group = data_builder.create_group()
r = as_admin.post('/projects', params={'inherit': 'false'}, json={'label': 'project2', 'group': group})
assert r.ok
project = r.json()['_id']

# get project-aggregated usage report
r = as_admin.get('/report/usage', params={'type': 'project'})
assert r.ok

# delete project
r= as_admin.delete('/projects/' + project)
assert r.ok