From 197e7f75c0b8c5d6c2e5ad87fab0e3be3671f75f Mon Sep 17 00:00:00 2001 From: Squirrel18 Date: Tue, 26 Jan 2021 19:27:07 -0500 Subject: [PATCH 1/4] Add course_id to the report data. --- proversity_reports_script/request_report.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/proversity_reports_script/request_report.py b/proversity_reports_script/request_report.py index 9e88cf6..6483ad3 100644 --- a/proversity_reports_script/request_report.py +++ b/proversity_reports_script/request_report.py @@ -156,21 +156,26 @@ def get_report_data(self, report_generation_request_response, request_headers): ) elif self.api_version == 'v1': response_data = report_generation_request_response.get('data', {}) - report_data = [] + report_data = {} if not response_data: print('No response data.') exit() for course_id in self.courses: + course_data = [] + for page_url in response_data.get(course_id, []): - report_data.append( + course_data.append( polling_report_data( report_data_url=page_url, request_headers=request_headers, ) ) + if course_data: + report_data.update({course_id: course_data}) + return report_data def init_report_backend(self, report_data): @@ -280,7 +285,7 @@ def polling_report_data(report_data_url, request_headers): return {} sleep(sleep_for) - print('waitig for... {}'.format(sleep_for)) + print('waiting for... {}'.format(sleep_for)) report_data = request_handler( request_url=report_data_url, request_data={}, From 80062d05e02e8c2ed059032bad801a3d733d9b54 Mon Sep 17 00:00:00 2001 From: Squirrel18 Date: Tue, 26 Jan 2021 19:30:46 -0500 Subject: [PATCH 2/4] Fix the enrollment_per_site_report backend. --- .../report_backend/enrollment_per_site_report.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/proversity_reports_script/report_backend/enrollment_per_site_report.py b/proversity_reports_script/report_backend/enrollment_per_site_report.py index 5b95096..d67cd2b 100644 --- a/proversity_reports_script/report_backend/enrollment_per_site_report.py +++ b/proversity_reports_script/report_backend/enrollment_per_site_report.py @@ -2,6 +2,7 @@ Enrollment per site report backend. """ import csv +import functools import os from collections import OrderedDict from datetime import datetime, timedelta @@ -28,7 +29,10 @@ def generate_report(self, json_report_data): """ Main logic to generate the report. """ - self.json_report_to_csv([data.get('result', {}) for data in json_report_data]) + # Get all the values of each course. + all_courses_data = functools.reduce(lambda x, y: x + y, json_report_data.values()) + + self.json_report_to_csv((data.get('result', {}) for data in all_courses_data)) def json_report_to_csv(self, json_report_data): """ From 91b05ce74a3c739753d1941ed840984b16f5b73b Mon Sep 17 00:00:00 2001 From: Squirrel18 Date: Tue, 26 Jan 2021 19:33:32 -0500 Subject: [PATCH 3/4] Update completion report to use reporting API V1. --- .../report_backend/completion_report.py | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/proversity_reports_script/report_backend/completion_report.py b/proversity_reports_script/report_backend/completion_report.py index 96715bc..fa22583 100644 --- a/proversity_reports_script/report_backend/completion_report.py +++ b/proversity_reports_script/report_backend/completion_report.py @@ -2,6 +2,7 @@ Report backend for completion report. """ import csv +import functools import os from collections import OrderedDict from datetime import datetime @@ -19,6 +20,8 @@ class CompletionReportBackend(AbstractBaseReportBackend): def __init__(self, *args, **kwargs): extra_data = kwargs.get('extra_data', {}) + self.bucket_name = extra_data.get('BUCKET_NAME', '') + self.bucket_path = extra_data.get('BUCKET_PATH', '') super(CompletionReportBackend, self).__init__(extra_data.get('SPREADSHEET_DATA', {})) def generate_report(self, json_report_data): @@ -31,21 +34,21 @@ def json_report_to_csv(self, json_report_data): """ Process json data to convert into csv format. """ - report_data = json_report_data.get('result', {}) - - if not report_data: + if not json_report_data: print('No report data...') exit() - course_list = report_data.keys() - - - for course in course_list: - course_data = report_data.get(course, []) + for course in json_report_data.keys(): + course_data = json_report_data.get(course, {}) + # Reduce all page results into a single list. + user_data = functools.reduce( + lambda accumulated_user_data, current_user_data: accumulated_user_data + current_user_data, + map(lambda course_page_data: course_page_data.get('result', []), course_data), + ) csv_data = [] general_course_data = {} - for user in course_data: + for user in user_data: username = user.get('username', '') user_id = user.get('user_id', '') cohort = user.get('cohort', '') @@ -130,15 +133,15 @@ def upload_file_to_storage(self, course, path_file): Uploads the csv report, to S3 storage. """ amazon_storage = boto3.resource('s3') - reports_bucket = amazon_storage.Bucket('proversity-custom-reports') - now = datetime.now() + reports_bucket = amazon_storage.Bucket(self.bucket_name) reports_bucket.upload_file( path_file, - 'cabinet/{course}/completion_report/{date}.csv'.format( + '{bucket_path}/{course}/{date}.csv'.format( + bucket_path=self.bucket_path, course=course, - date=now - ) + date=datetime.now(), + ), ) def _verify_name(self, name, data): From 7b0c425de2ade088b1a45a281659d95fab250953 Mon Sep 17 00:00:00 2001 From: Squirrel18 Date: Tue, 26 Jan 2021 20:49:36 -0500 Subject: [PATCH 4/4] Updating README. --- README.md | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 45f8ef5..9cb03ec 100644 --- a/README.md +++ b/README.md @@ -25,13 +25,23 @@ The configuration file must be in json format. e.g. "Add a new one, if you add a new report backend" ], "COMPLETION_REPORT": { - "REPORT_URL": "example:/proversity-reports/api/v0/generate-completion-report", + "REPORT_URL": "example:/pearson-reports/api/v1/generate-completion-report", "BACKEND_REPORT": "example:proversity_reports_script.report_backend.completion_report:CompletionReportBackend", - "SPREADSHEET_DATA": { - "completion_sheet_id_": "Spreadsheet id for completion report.", - "general_course_sheet_id_": "Spreadsheet id for course structure report." + "EXTRA_DATA": { + "SPREADSHEET_DATA": { + "completion_sheet_id_": "Spreadsheet id for completion report.", + "general_course_sheet_id_": "Spreadsheet id for course structure report." + }, + "BUCKET_NAME": "Name of the bucket to store the reports", + "SPREADSHEET_RANGE_NAME": "Name of the spreadsheet range to update.", + "BUCKET_PATH": "Path inside the bucket to store the CSV files." + }, + "EXTRA_REQUEST_DATA": { + "query_params": { + "limit": Number of results per page. + } } - }, + } "LAST_PAGE_ACCESSED": { "REPORT_URL": "path-to-report-generation-url", "BACKEND_REPORT": "absolute-path-to-report-backend-module:report-backend-class-name"