feat: Add completion_mode to xblock dump

event_sink_clickhouse/serializers.py

@@ -127,7 +127,6 @@ def get_course_data_json(self, overview):
             "entrance_exam_enabled": getattr(overview, "entrance_exam_enabled", ""),
             "external_id": getattr(overview, "external_id", ""),
             "language": getattr(overview, "language", ""),
-            "course_tree_location": getattr(overview, "language", "{}"),
         }
         return json.dumps(json_fields)
 

event_sink_clickhouse/sinks/course_published.py

@@ -69,7 +69,6 @@ def serialize_item(self, item, many=False, initial=None):
         """
         Serialize an XBlock into a dict
         """
-        print(item)
         course_key = CourseKey.from_string(item["course_key"])
         modulestore = get_modulestore()
         detached_xblock_types = get_detached_xblock_types()
@@ -94,8 +93,6 @@ def serialize_item(self, item, many=False, initial=None):
                 initial["time_last_dumped"],
             )
 
-            print(fields["xblock_data_json"]["block_type"])
-
             if fields["xblock_data_json"]["block_type"] == "chapter":
                 section_idx += 1
                 subsection_idx = 0
@@ -106,11 +103,9 @@ def serialize_item(self, item, many=False, initial=None):
             elif fields["xblock_data_json"]["block_type"] == "vertical":
                 unit_idx += 1
 
-            fields["xblock_data_json"]["course_tree_location"] = {
-                "section": section_idx,
-                "subsection": subsection_idx,
-                "unit": unit_idx
-            }
+            fields["xblock_data_json"]["section"] = section_idx
+            fields["xblock_data_json"]["subsection"] = subsection_idx
+            fields["xblock_data_json"]["unit"] = unit_idx
 
             fields["xblock_data_json"] = json.dumps(fields["xblock_data_json"])
             location_to_node[
@@ -172,6 +167,7 @@ def serialize_xblock(
             "block_type": block_type,
             "detached": 1 if block_type in detached_xblock_types else 0,
             "graded": 1 if getattr(item, "graded", False) else 0,
+            "completion_mode": getattr(item, "completion_mode", ""),
         }
 
         # Core table data, if things change here it's a big deal.

test_utils/helpers.py

@@ -52,7 +52,7 @@ class FakeXBlock:
     """
     Fakes the parameters of an XBlock that we care about.
     """
-    def __init__(self, identifier, block_type="vertical", graded=False):
+    def __init__(self, identifier, block_type="vertical", graded=False, completion_mode="unknown"):
         self.block_type = block_type
         self.scope_ids = Mock()
         self.scope_ids.usage_id.course_key = course_key_factory()
@@ -62,6 +62,7 @@ def __init__(self, identifier, block_type="vertical", graded=False):
         self.edited_on = datetime.now()
         self.children = []
         self.graded = graded
+        self.completion_mode = completion_mode
 
     def get_children(self):
         """
@@ -234,6 +235,11 @@ def course_factory():
     for i in range(3):
         course.append(FakeXBlock(f"Graded {i}", graded=True))
 
+    # Create some completable blocks at the top level
+    course.append(FakeXBlock("Completable", completion_mode="completable"))
+    course.append(FakeXBlock("Aggregator", completion_mode="aggregator"))
+    course.append(FakeXBlock("Excluded", completion_mode="excluded"))
+
     return course
 
 

tests/test_course_published.py

@@ -4,7 +4,7 @@
 import json
 import logging
 from datetime import datetime
-from unittest.mock import patch, MagicMock
+from unittest.mock import MagicMock, patch
 
 import pytest
 import requests
@@ -238,11 +238,14 @@ def test_xblock_tree_structure(mock_modulestore, mock_detached):
     results = sink.serialize_item(fake_serialized_course_overview, initial=initial_data)
 
     def _check_tree_location(block, expected_section=0, expected_subsection=0, expected_unit=0):
+        """
+        Assert the expected values in certain returned blocks or print useful debug information.
+        """
         try:
             j = json.loads(block["xblock_data_json"])
-            assert j["course_tree_location"]["section"] == expected_section
-            assert j["course_tree_location"]["subsection"] == expected_subsection
-            assert j["course_tree_location"]["unit"] == expected_unit
+            assert j["section"] == expected_section
+            assert j["subsection"] == expected_subsection
+            assert j["unit"] == expected_unit
         except AssertionError as e:
             print(e)
             print(block)
@@ -267,3 +270,49 @@ def _check_tree_location(block, expected_section=0, expected_subsection=0, expec
     _check_tree_location(results[25], 3, 3, 1)
     _check_tree_location(results[26], 3, 3, 2)
     _check_tree_location(results[27], 3, 3, 3)
+
+
+@patch("event_sink_clickhouse.sinks.course_published.get_detached_xblock_types")
+@patch("event_sink_clickhouse.sinks.course_published.get_modulestore")
+def test_xblock_graded_completable_mode(mock_modulestore, mock_detached):
+    """
+    Test that our grading and completion fields serialize.
+    """
+    # Create a fake course structure with a few fake XBlocks
+    course = course_factory()
+    course_overview = fake_course_overview_factory(modified=datetime.now())
+    mock_modulestore.return_value.get_items.return_value = course
+
+    # Fake the "detached types" list since we can't import it here
+    mock_detached.return_value = mock_detached_xblock_types()
+
+    fake_serialized_course_overview = fake_serialize_fake_course_overview(course_overview)
+    sink = XBlockSink(connection_overrides={}, log=MagicMock())
+
+    # Remove the relationships sink, we're just checking the structure here.
+    sink.serialize_relationships = MagicMock()
+    initial_data = {"dump_id": "xyz", "time_last_dumped": "2023-09-05"}
+    results = sink.serialize_item(fake_serialized_course_overview, initial=initial_data)
+
+    def _check_item_serialized_location(block, expected_graded=0, expected_completion_mode="unknown"):
+        """
+        Assert the expected values in certain returned blocks or print useful debug information.
+        """
+        try:
+            j = json.loads(block["xblock_data_json"])
+            assert j["graded"] == expected_graded
+            assert j["completion_mode"] == expected_completion_mode
+        except AssertionError as e:
+            print(e)
+            print(block)
+            raise
+
+    # These tree indexes are the only ones which should have gradable set
+    _check_item_serialized_location(results[31], 1)
+    _check_item_serialized_location(results[32], 1)
+    _check_item_serialized_location(results[33], 1)
+
+    # These tree indexes are the only ones which should have non-"unknown" completion_modes.
+    _check_item_serialized_location(results[34], 0, "completable")
+    _check_item_serialized_location(results[35], 0, "aggregator")
+    _check_item_serialized_location(results[36], 0, "excluded")