feat: Add event listener for course publish

Creates the edx-platform plugin plumbing, adds some new requirements, maps the appropriate Django Signal to push course structure to ClickHouse.

event_sink_clickhouse/sinks/course_published.py

@@ -25,19 +25,30 @@ class CoursePublishedSink(BaseSink):
     """
     @staticmethod
     def _get_detached_xblock_types():
+        """
+        Import and return DETACHED_XBLOCK_TYPES.
+        Placed here to avoid model import at startup and to facilitate mocking them in testing.
+        """
         # pylint: disable=import-outside-toplevel,import-error
         from xmodule.modulestore.store_utilities import DETACHED_XBLOCK_TYPES
         return DETACHED_XBLOCK_TYPES
 
     @staticmethod
     def _get_modulestore():
-        # Import is placed here to avoid model import at project startup.
-        from xmodule.modulestore.django import modulestore  # pylint: disable=import-outside-toplevel,import-error
+        """
+        Import and return modulestore.
+        Placed here to avoid model import at startup and to facilitate mocking them in testing.
+        """
+        # pylint: disable=import-outside-toplevel,import-error
+        from xmodule.modulestore.django import modulestore
         return modulestore()
 
     @staticmethod
     def _get_course_overview_model():
-        # Import is placed here to avoid model import at project startup.
+        """
+        Import and return CourseOverview.
+        Placed here to avoid model import at startup and to facilitate mocking them in testing.
+        """
         # pylint: disable=import-outside-toplevel,import-error
         from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
         return CourseOverview
@@ -55,7 +66,7 @@ def strip_branch_and_version(location):
     @staticmethod
     def get_course_last_published(course_key):
         """
-        Approximately when was a course last published?
+        Get approximate last publish date for the given course.
 
         We use the 'modified' column in the CourseOverview table as a quick and easy
         (although perhaps inexact) way of determining when a course was last
@@ -72,8 +83,6 @@ def get_course_last_published(course_key):
         """
         CourseOverview = CoursePublishedSink._get_course_overview_model()
         approx_last_published = CourseOverview.get_from_id(course_key).modified
-        print(approx_last_published)
-        raise Exception("foo")
         return str(approx_last_published)
 
     @staticmethod
@@ -209,7 +218,7 @@ def dump(self, course_key):
 
             self.log.info("Completed dumping %s to ClickHouse", course_key)
 
-        except Exception:  # pylint: disable=broad-except
+        except Exception:
             self.log.exception(
                 "Error trying to dump course %s to ClickHouse!",
                 course_string

requirements/base.txt

@@ -67,7 +67,7 @@ pytz==2023.3
     # via
     #   celery
     #   django
-requests==2.28.2
+requests==2.29.0
     # via -r requirements/base.in
 six==1.16.0
     # via click-repl

requirements/ci.txt

@@ -12,7 +12,7 @@ filelock==3.12.0
     #   virtualenv
 packaging==23.1
     # via tox
-platformdirs==3.2.0
+platformdirs==3.5.0
     # via virtualenv
 pluggy==1.0.0
     # via tox
@@ -29,5 +29,5 @@ tox==3.28.0
     #   tox-battery
 tox-battery==0.6.1
     # via -r requirements/ci.in
-virtualenv==20.22.0
+virtualenv==20.23.0
     # via tox

requirements/dev.txt

@@ -74,7 +74,7 @@ code-annotations==1.3.0
     # via
     #   -r requirements/quality.txt
     #   edx-lint
-coverage[toml]==7.2.3
+coverage[toml]==7.2.5
     # via
     #   -r requirements/quality.txt
     #   pytest-cov
@@ -173,7 +173,7 @@ pbr==5.11.1
     #   stevedore
 pip-tools==6.13.0
     # via -r requirements/pip-tools.txt
-platformdirs==3.2.0
+platformdirs==3.5.0
     # via
     #   -r requirements/ci.txt
     #   -r requirements/quality.txt
@@ -210,7 +210,7 @@ pydocstyle==6.3.0
     # via -r requirements/quality.txt
 pygments==2.15.1
     # via diff-cover
-pylint==2.17.2
+pylint==2.17.3
     # via
     #   -r requirements/quality.txt
     #   edx-lint
@@ -265,7 +265,12 @@ pyyaml==6.0
     #   -r requirements/quality.txt
     #   code-annotations
     #   edx-i18n-tools
-requests==2.28.2
+    #   responses
+requests==2.29.0
+    # via
+    #   -r requirements/quality.txt
+    #   responses
+responses==0.23.1
     # via -r requirements/quality.txt
 six==1.16.0
     # via
@@ -303,7 +308,7 @@ tomli==2.0.1
     #   pyproject-hooks
     #   pytest
     #   tox
-tomlkit==0.11.7
+tomlkit==0.11.8
     # via
     #   -r requirements/quality.txt
     #   pylint
@@ -314,6 +319,10 @@ tox==3.28.0
     #   tox-battery
 tox-battery==0.6.1
     # via -r requirements/ci.txt
+types-pyyaml==6.0.12.9
+    # via
+    #   -r requirements/quality.txt
+    #   responses
 typing-extensions==4.5.0
     # via
     #   -r requirements/quality.txt
@@ -323,13 +332,14 @@ urllib3==1.26.15
     # via
     #   -r requirements/quality.txt
     #   requests
+    #   responses
 vine==5.0.0
     # via
     #   -r requirements/quality.txt
     #   amqp
     #   celery
     #   kombu
-virtualenv==20.22.0
+virtualenv==20.23.0
     # via
     #   -r requirements/ci.txt
     #   tox

requirements/doc.txt

@@ -67,7 +67,7 @@ click-repl==0.2.0
     #   celery
 code-annotations==1.3.0
     # via -r requirements/test.txt
-coverage[toml]==7.2.3
+coverage[toml]==7.2.5
     # via
     #   -r requirements/test.txt
     #   pytest-cov
@@ -218,21 +218,25 @@ pyyaml==6.0
     # via
     #   -r requirements/test.txt
     #   code-annotations
+    #   responses
 readme-renderer==37.3
     # via twine
-requests==2.28.2
+requests==2.29.0
     # via
     #   -r requirements/test.txt
     #   requests-toolbelt
+    #   responses
     #   sphinx
     #   twine
-requests-toolbelt==0.10.1
+requests-toolbelt==1.0.0
     # via twine
+responses==0.23.1
+    # via -r requirements/test.txt
 restructuredtext-lint==1.4.0
     # via doc8
 rfc3986==2.0.0
     # via twine
-rich==13.3.4
+rich==13.3.5
     # via twine
 six==1.16.0
     # via
@@ -288,6 +292,10 @@ tomli==2.0.1
     #   pytest
 twine==4.0.2
     # via -r requirements/doc.in
+types-pyyaml==6.0.12.9
+    # via
+    #   -r requirements/test.txt
+    #   responses
 typing-extensions==4.5.0
     # via
     #   pydata-sphinx-theme
@@ -296,6 +304,7 @@ urllib3==1.26.15
     # via
     #   -r requirements/test.txt
     #   requests
+    #   responses
     #   twine
 vine==5.0.0
     # via

requirements/pip.txt

@@ -8,7 +8,7 @@ wheel==0.40.0
     # via -r requirements/pip.in
 
 # The following packages are considered to be unsafe in a requirements file:
-pip==23.1.1
+pip==23.1.2
     # via -r requirements/pip.in
 setuptools==67.7.2
     # via -r requirements/pip.in

requirements/quality.txt

@@ -63,7 +63,7 @@ code-annotations==1.3.0
     # via
     #   -r requirements/test.txt
     #   edx-lint
-coverage[toml]==7.2.3
+coverage[toml]==7.2.5
     # via
     #   -r requirements/test.txt
     #   pytest-cov
@@ -133,7 +133,7 @@ pbr==5.11.1
     # via
     #   -r requirements/test.txt
     #   stevedore
-platformdirs==3.2.0
+platformdirs==3.5.0
     # via pylint
 pluggy==1.0.0
     # via
@@ -155,7 +155,7 @@ pycparser==2.21
     #   cffi
 pydocstyle==6.3.0
     # via -r requirements/quality.in
-pylint==2.17.2
+pylint==2.17.3
     # via
     #   edx-lint
     #   pylint-celery
@@ -199,7 +199,12 @@ pyyaml==6.0
     # via
     #   -r requirements/test.txt
     #   code-annotations
-requests==2.28.2
+    #   responses
+requests==2.29.0
+    # via
+    #   -r requirements/test.txt
+    #   responses
+responses==0.23.1
     # via -r requirements/test.txt
 six==1.16.0
     # via
@@ -228,8 +233,12 @@ tomli==2.0.1
     #   coverage
     #   pylint
     #   pytest
-tomlkit==0.11.7
+tomlkit==0.11.8
     # via pylint
+types-pyyaml==6.0.12.9
+    # via
+    #   -r requirements/test.txt
+    #   responses
 typing-extensions==4.5.0
     # via
     #   astroid
@@ -238,6 +247,7 @@ urllib3==1.26.15
     # via
     #   -r requirements/test.txt
     #   requests
+    #   responses
 vine==5.0.0
     # via
     #   -r requirements/test.txt

requirements/test.in

@@ -6,3 +6,4 @@
 pytest-cov                # pytest extension for code coverage statistics
 pytest-django             # pytest extension for better Django support
 code-annotations          # provides commands used by the pii_check make target.
+responses                 # mocks for the requests library

requirements/test.txt

@@ -53,7 +53,7 @@ click-repl==0.2.0
     #   celery
 code-annotations==1.3.0
     # via -r requirements/test.in
-coverage[toml]==7.2.3
+coverage[toml]==7.2.5
     # via pytest-cov
     # via
     #   -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt
@@ -136,9 +136,15 @@ pytz==2023.3
     #   celery
     #   django
 pyyaml==6.0
-    # via code-annotations
-requests==2.28.2
-    # via -r requirements/base.txt
+    # via
+    #   code-annotations
+    #   responses
+requests==2.29.0
+    # via
+    #   -r requirements/base.txt
+    #   responses
+responses==0.23.1
+    # via -r requirements/test.in
 six==1.16.0
     # via
     #   -r requirements/base.txt
@@ -159,10 +165,13 @@ tomli==2.0.1
     # via
     #   coverage
     #   pytest
+types-pyyaml==6.0.12.9
+    # via responses
 urllib3==1.26.15
     # via
     #   -r requirements/base.txt
     #   requests
+    #   responses
 vine==5.0.0
     # via
     #   -r requirements/base.txt