PAE-527 - Update the completion report to use API V1.

README.md

@@ -25,13 +25,23 @@ The configuration file must be in json format. e.g.
             "Add a new one, if you add a new report backend"
         ],
         "COMPLETION_REPORT": {
-            "REPORT_URL": "example:/proversity-reports/api/v0/generate-completion-report",
+            "REPORT_URL": "example:/pearson-reports/api/v1/generate-completion-report",
             "BACKEND_REPORT": "example:proversity_reports_script.report_backend.completion_report:CompletionReportBackend",
-            "SPREADSHEET_DATA": {
-                "completion_sheet_id_<course_id>": "Spreadsheet id for completion report.",
-                "general_course_sheet_id_<course_id>": "Spreadsheet id for course structure report."
+            "EXTRA_DATA": {
+                "SPREADSHEET_DATA": {
+                    "completion_sheet_id_<course_id>": "Spreadsheet id for completion report.",
+                    "general_course_sheet_id_<course_id>": "Spreadsheet id for course structure report."
+                },
+                "BUCKET_NAME": "Name of the bucket to store the reports",
+                "SPREADSHEET_RANGE_NAME": "Name of the spreadsheet range to update.",
+                "BUCKET_PATH": "Path inside the bucket to store the CSV files."
+            },
+            "EXTRA_REQUEST_DATA": {
+                "query_params": {
+                    "limit": Number of results per page.
+                }
             }
-        },
+        }
         "LAST_PAGE_ACCESSED": {
             "REPORT_URL": "path-to-report-generation-url",
             "BACKEND_REPORT": "absolute-path-to-report-backend-module:report-backend-class-name"

proversity_reports_script/report_backend/completion_report.py

@@ -2,6 +2,7 @@
 Report backend for completion report.
 """
 import csv
+import functools
 import os
 from collections import OrderedDict
 from datetime import datetime
@@ -19,6 +20,8 @@ class CompletionReportBackend(AbstractBaseReportBackend):
 
     def __init__(self, *args, **kwargs):
         extra_data = kwargs.get('extra_data', {})
+        self.bucket_name = extra_data.get('BUCKET_NAME', '')
+        self.bucket_path = extra_data.get('BUCKET_PATH', '')
         super(CompletionReportBackend, self).__init__(extra_data.get('SPREADSHEET_DATA', {}))
 
     def generate_report(self, json_report_data):
@@ -31,21 +34,21 @@ def json_report_to_csv(self, json_report_data):
         """
         Process json data to convert into csv format.
         """
-        report_data = json_report_data.get('result', {})
-
-        if not report_data:
+        if not json_report_data:
             print('No report data...')
             exit()
 
-        course_list = report_data.keys()
-
-
-        for course in course_list:
-            course_data = report_data.get(course, [])
+        for course in json_report_data.keys():
+            course_data = json_report_data.get(course, {})
+            # Reduce all page results into a single list.
+            user_data = functools.reduce(
+                lambda accumulated_user_data, current_user_data: accumulated_user_data + current_user_data,
+                map(lambda course_page_data: course_page_data.get('result', []), course_data),
+            )
             csv_data = []
             general_course_data = {}
 
-            for user in course_data:
+            for user in user_data:
                 username = user.get('username', '')
                 user_id = user.get('user_id', '')
                 cohort = user.get('cohort', '')
@@ -130,15 +133,15 @@ def upload_file_to_storage(self, course, path_file):
         Uploads the csv report, to S3 storage.
         """
         amazon_storage = boto3.resource('s3')
-        reports_bucket = amazon_storage.Bucket('proversity-custom-reports')
-        now = datetime.now()
+        reports_bucket = amazon_storage.Bucket(self.bucket_name)
 
         reports_bucket.upload_file(
             path_file,
-            'cabinet/{course}/completion_report/{date}.csv'.format(
+            '{bucket_path}/{course}/{date}.csv'.format(
+                bucket_path=self.bucket_path,
                 course=course,
-                date=now
-            )
+                date=datetime.now(),
+            ),
         )
 
     def _verify_name(self, name, data):

proversity_reports_script/report_backend/enrollment_per_site_report.py

@@ -2,6 +2,7 @@
 Enrollment per site report backend.
 """
 import csv
+import functools
 import os
 from collections import OrderedDict
 from datetime import datetime, timedelta
@@ -28,7 +29,10 @@ def generate_report(self, json_report_data):
         """
         Main logic to generate the report.
         """
-        self.json_report_to_csv([data.get('result', {}) for data in json_report_data])
+        # Get all the values of each course.
+        all_courses_data = functools.reduce(lambda x, y: x + y, json_report_data.values())
+
+        self.json_report_to_csv((data.get('result', {}) for data in all_courses_data))
 
     def json_report_to_csv(self, json_report_data):
         """

proversity_reports_script/request_report.py

@@ -156,21 +156,26 @@ def get_report_data(self, report_generation_request_response, request_headers):
             )
         elif self.api_version == 'v1':
             response_data = report_generation_request_response.get('data', {})
-            report_data = []
+            report_data = {}
 
             if not response_data:
                 print('No response data.')
                 exit()
 
             for course_id in self.courses:
+                course_data = []
+
                 for page_url in response_data.get(course_id, []):
-                    report_data.append(
+                    course_data.append(
                         polling_report_data(
                             report_data_url=page_url,
                             request_headers=request_headers,
                         )
                     )
 
+                if course_data:
+                    report_data.update({course_id: course_data})
+
         return report_data
 
     def init_report_backend(self, report_data):
@@ -280,7 +285,7 @@ def polling_report_data(report_data_url, request_headers):
             return {}
 
         sleep(sleep_for)
-        print('waitig for... {}'.format(sleep_for))
+        print('waiting for... {}'.format(sleep_for))
         report_data = request_handler(
             request_url=report_data_url,
             request_data={},