Initial implementation of #50

.vscode/launch.json

@@ -11,7 +11,7 @@
             "args": [
                 "-v"
             ],
-            "justMyCode": false
+            "justMyCode": true
         }
     ]
 }

moodle_dl/config_service/config_helper.py

@@ -81,6 +81,13 @@ def get_download_databases(self) -> bool:
         except ValueError:
             return False
 
+    def get_download_forums(self) -> bool:
+        # returns a stored boolean if forums should be downloaded
+        try:
+            return self.get_property('download_forums')
+        except ValueError:
+            return False
+
     def get_download_course_ids(self) -> str:
         # returns a stored list of course ids hat should be downloaded
         try:

moodle_dl/config_service/config_service.py

@@ -44,6 +44,7 @@ def interactively_acquire_config(self):
         self._select_should_download_descriptions()
         self._select_should_download_links_in_descriptions()
         self._select_should_download_databases()
+        self._select_should_download_forums()
         self._select_should_download_linked_files()
         self._select_should_download_also_with_cookie()
 
@@ -246,6 +247,22 @@ def _select_should_download_databases(self):
 
         self.config_helper.set_property('download_databases', download_databases)
 
+    def _select_should_download_forums(self):
+        """
+        Asks the user if forums should be downloaded
+        """
+        download_forums = self.config_helper.get_download_forums()
+
+        self.section_seperator()
+        Log.info('In forums, students and teachers can discuss and exchange information together.')
+        print('')
+
+        download_forums = cutie.prompt_yes_or_no(
+            Log.special_str('Do you want to download forums of your courses?'), default_is_yes=download_forums
+        )
+
+        self.config_helper.set_property('download_forums', download_forums)
+
     def _select_should_download_descriptions(self):
         """
         Asks the user if descriptions should be downloaded

moodle_dl/moodle_connector/assignments_handler.py

@@ -145,7 +145,11 @@ def _get_files_of_plugins(obj: {}) -> []:
                 filename = editorfield.get('description', '')
                 description = editorfield.get('text', '')
                 if filename != '' and description != '':
-                    description_file = {'filename': filename, 'description': description, 'type': 'description'}
+                    description_file = {
+                        'filename': filename,
+                        'description': description,
+                        'type': 'description',
+                    }
                     result.append(description_file)
 
         return result

moodle_dl/moodle_connector/databases_handler.py

@@ -76,8 +76,6 @@ def fetch_database_files(self, databases: {int: {int: {}}}) -> {int: {int: {}}}:
         entries. This is kind of waste of resources, because there
         is no API to get all entries at once
         @param databases: the dictionary of databases of all courses.
-        @param download_course_ids: ids of courses for that should
-                                    be downloaded
         @return: A Dictionary of all databases,
                  indexed by courses, then databases
         """
@@ -88,8 +86,9 @@ def fetch_database_files(self, databases: {int: {int: {}}}) -> {int: {int: {}}}:
 
         counter = 0
         total = 0
+        intro = '\rDownloading database information'
 
-        # count total assignments for nice console output
+        # count total databases for nice console output
         for course_id in databases:
             for database_id in databases[course_id]:
                 total += 1
@@ -105,6 +104,8 @@ def fetch_database_files(self, databases: {int: {int: {}}}) -> {int: {int: {}}}:
                 if not access.get('timeavailable', False):
                     continue
 
+                print(intro + ' %3d/%3d [%6s|%6s]\033[K' % (counter, total, course_id, real_id), end='')
+
                 data.update({'returncontents': 1})
 
                 entries = self.request_helper.post_REST('mod_data_get_entries', data)

moodle_dl/moodle_connector/forums_handler.py

@@ -0,0 +1,212 @@
+from datetime import datetime
+
+from moodle_dl.moodle_connector.request_helper import RequestHelper
+from moodle_dl.state_recorder.course import Course
+from moodle_dl.download_service.path_tools import PathTools
+
+
+class ForumsHandler:
+    """
+    Fetches and parses the various endpoints in Moodle for Forum Entries.
+    """
+
+    def __init__(self, request_helper: RequestHelper, version: int):
+        self.request_helper = request_helper
+        self.version = version
+
+    def fetch_forums(self, courses: [Course]) -> {int: {int: {}}}:
+        """
+        Fetches the Databases List for all courses from the
+        Moodle system
+        @return: A Dictionary of all databases,
+                 indexed by courses, then databases
+        """
+        # do this only if version is greater then 2.5
+        # because mod_forum_get_forums_by_courses will fail
+        if self.version < 2013051400:
+            return {}
+
+        print('\rDownloading databases information\033[K', end='')
+
+        # We create a dictionary with all the courses we want to request.
+        extra_data = {}
+        courseids = {}
+        for index, course in enumerate(courses):
+            courseids.update({str(index): course.id})
+
+        extra_data.update({'courseids': courseids})
+
+        forums = self.request_helper.post_REST('mod_forum_get_forums_by_courses', extra_data)
+
+        result = {}
+        for forum in forums:
+            # This is the instance id with which we can make the API queries.
+            forum_id = forum.get('id', 0)
+            forum_name = forum.get('name', 'forum')
+            forum_intro = forum.get('intro', '')
+            forum_course_module_id = forum.get('cmid', 0)
+            forum_introfiles = forum.get('introfiles', [])
+            course_id = forum.get('course', 0)
+
+            # normalize
+            for forum_file in forum_introfiles:
+                file_type = forum_file.get('type', '')
+                if file_type is None or file_type == '':
+                    forum_file.update({'type': 'forum_introfile'})
+
+            forum_entry = {
+                forum_course_module_id: {
+                    'id': forum_id,
+                    'name': forum_name,
+                    'intro': forum_intro,
+                    'files': forum_introfiles,
+                }
+            }
+
+            course_dic = result.get(course_id, {})
+
+            course_dic.update(forum_entry)
+
+            result.update({course_id: course_dic})
+
+        return result
+
+    def fetch_forums_posts(self, forums: {}, last_timestamps_per_forum: {}) -> {}:
+        """
+        Fetches for the forums list of all courses the additionally
+        entries. This is kind of waste of resources, because there
+        is no API to get all entries at once.
+        @param forums: the dictionary of forums of all courses.
+        @return: A Dictionary of all forums,
+                 indexed by courses, then forums
+        """
+        # do this only if version is greater then 2.8
+        # because mod_forum_get_forum_discussions_paginated will fail
+        if self.version < 2014111000:
+            return forums
+
+        counter = 0
+        total = 0
+        intro = '\rDownloading forum discussions'
+
+        # count total forums for nice console output
+        for course_id in forums:
+            for forum_id in forums[course_id]:
+                total += 1
+
+        for course_id in forums:
+            for forum_id in forums[course_id]:
+                counter += 1
+                real_id = forums[course_id][forum_id].get('id', 0)
+                page_num = 0
+                last_timestamp = last_timestamps_per_forum.get(forum_id, 0)
+                latest_discussions = []
+                done = False
+                while not done:
+                    data = {
+                        'forumid': real_id,
+                        'perpage': 10,
+                        'page': page_num,
+                    }
+
+                    print(
+                        intro + ' %3d/%3d [%6s|%6s|p%s]\033[K' % (counter, total, course_id, real_id, page_num), end=''
+                    )
+
+                    if self.version >= 2019052000:
+                        discussions_result = self.request_helper.post_REST('mod_forum_get_forum_discussions', data)
+                    else:
+                        discussions_result = self.request_helper.post_REST(
+                            'mod_forum_get_forum_discussions_paginated', data
+                        )
+
+                    discussions = discussions_result.get('discussions', [])
+
+                    if len(discussions) == 0:
+                        done = True
+                        break
+
+                    for discussion in discussions:
+
+                        timemodified = discussion.get('timemodified', 0)
+                        if discussion.get('modified', 0) > timemodified:
+                            timemodified = discussion.get('modified', 0)
+
+                        if last_timestamp < timemodified:
+                            latest_discussions.append(
+                                {
+                                    'subject': discussion.get('subject', ''),
+                                    'timemodified': timemodified,
+                                    'discussion_id': discussion.get('discussion', 0),
+                                }
+                            )
+                        else:
+                            done = True
+                            break
+                    page_num += 1
+
+                forums_files = self._get_files_of_discussions(latest_discussions)
+                forums[course_id][forum_id]['files'] += forums_files
+
+        return forums
+
+    def _get_files_of_discussions(self, latest_discussions: []) -> []:
+        result = []
+
+        for i, discussion in enumerate(latest_discussions):
+
+            print(
+                '\rDownloading posts of discussion %3d/%3d\033[K' % (i, len(latest_discussions) - 1),
+                end='',
+            )
+
+            data = {
+                'discussionid': discussion.get('discussion_id', 0),
+                'sortby': 'modified',
+                'sortdirection': 'ASC',
+            }
+
+            posts_result = self.request_helper.post_REST('mod_forum_get_forum_discussion_posts', data)
+
+            posts = posts_result.get('posts', [])
+
+            for post in posts:
+                post_message = post.get('message', '')
+                post_created = post.get('created', 0)
+                post_modified = post.get('modified', 0)
+
+                post_id = post.get('id', 0)
+                post_parent = post.get('parent', 0)
+                post_userfullname = post.get('userfullname', '')
+                post_filename = PathTools.to_valid_name(
+                    datetime.utcfromtimestamp(post_created).strftime('%Y-%m-%d %H:%M:%S')
+                    + ' ['
+                    + str(post_id)
+                    + '] '
+                    + post_userfullname
+                )
+                if post_parent != 0:
+                    post_filename = PathTools.to_valid_name(post_filename + ' response to [' + str(post_parent) + ']')
+
+                post_path = PathTools.to_valid_name(discussion.get('subject', ''))
+
+                post_files = post.get('messageinlinefiles', [])
+                post_files += post.get('attachments', [])
+
+                post_file = {
+                    'filename': post_filename,
+                    'filepath': post_path,
+                    'modified': post_modified,
+                    'message': post_message,
+                    'type': 'post',
+                }
+                result.append(post_file)
+
+                for post_file in post_files:
+                    file_type = post_file.get('type', '')
+                    if file_type is None or file_type == '':
+                        post_file.update({'type': 'forum_file'})
+                    post_file.update({'filepath': post_path})
+                    result.append(post_file)
+
+        return result

moodle_dl/moodle_connector/moodle_service.py

@@ -14,6 +14,7 @@
 from moodle_dl.moodle_connector import sso_token_receiver
 from moodle_dl.moodle_connector.cookie_handler import CookieHandler
 from moodle_dl.moodle_connector.results_handler import ResultsHandler
+from moodle_dl.moodle_connector.forums_handler import ForumsHandler
 from moodle_dl.moodle_connector.databases_handler import DatabasesHandler
 from moodle_dl.moodle_connector.assignments_handler import AssignmentsHandler
 from moodle_dl.moodle_connector.first_contact_handler import FirstContactHandler
@@ -217,6 +218,7 @@ def fetch_state(self) -> [Course]:
         dont_download_course_ids = self.config_helper.get_dont_download_course_ids()
         download_submissions = self.config_helper.get_download_submissions()
         download_databases = self.config_helper.get_download_databases()
+        download_forums = self.config_helper.get_download_forums()
         download_also_with_cookie = self.config_helper.get_download_also_with_cookie()
 
         courses = []
@@ -229,6 +231,7 @@ def fetch_state(self) -> [Course]:
             userid, version = first_contact_handler.fetch_userid_and_version()
             assignments_handler = AssignmentsHandler(request_helper, version)
             databases_handler = DatabasesHandler(request_helper, version)
+            forums_handler = ForumsHandler(request_helper, version)
             results_handler.setVersion(version)
 
             if download_also_with_cookie:
@@ -251,6 +254,11 @@ def fetch_state(self) -> [Course]:
             if download_databases:
                 databases = databases_handler.fetch_database_files(databases)
 
+            forums = forums_handler.fetch_forums(courses)
+            if download_forums:
+                last_timestamps_per_forum = {942115: 1595849253}
+                forums = forums_handler.fetch_forums_posts(forums, last_timestamps_per_forum)
+
             index = 0
             for course in courses:
                 index += 1
@@ -273,7 +281,8 @@ def fetch_state(self) -> [Course]:
 
                 course_assignments = assignments.get(course.id, {})
                 course_databases = databases.get(course.id, {})
-                results_handler.set_fetch_addons(course_assignments, course_databases)
+                course_forums = forums.get(course.id, {})
+                results_handler.set_fetch_addons(course_assignments, course_databases, course_forums)
                 course.files = results_handler.fetch_files(course.id)
 
                 filtered_courses.append(course)

moodle_dl/moodle_connector/results_handler.py

@@ -20,6 +20,7 @@ def __init__(self, request_helper: RequestHelper, moodle_domain: str, moodle_pat
         self.moodle_path = moodle_path
         self.course_assignments = {}
         self.course_databases = {}
+        self.course_forums = {}
 
     def setVersion(self, version: int):
         self.version = version
@@ -106,6 +107,13 @@ def _get_files_in_modules(self, section_name: str, section_modules: []) -> [File
 
                 files += self._handle_files(section_name, module_name, module_modname, module_id, database_files)
 
+            elif module_modname == 'forum':
+                # find forums with same module_id
+                forums = self.course_forums.get(module_id, {})
+                forums_files = forums.get('files', [])
+
+                files += self._handle_files(section_name, module_name, module_modname, module_id, forums_files)
+
         return files
 
     @staticmethod
@@ -309,15 +317,17 @@ def _handle_description(
 
         return files
 
-    def set_fetch_addons(self, course_assignments: {int: {int: {}}}, course_databases: {int: {int: {}}}):
+    def set_fetch_addons(self, course_assignments: {}, course_databases: {}, course_forums: {}):
         """
         Sets the optional data that will be added to the result list
          during the process.
         @params course_assignments: The dictionary of assignments per course
         @params course_databases: The dictionary of databases per course
+        @params course_forums: The dictionary of forums per course
         """
         self.course_assignments = course_assignments
         self.course_databases = course_databases
+        self.course_forums = course_forums
 
     def fetch_files(self, course_id: str) -> [File]:
         """