Fixes tl-its-umich-edu#1327 - Improve verify_course_ids to only consi…

…der courses with

local data

dashboard/cron.py

@@ -143,7 +143,7 @@ def verify_course_ids(self):
         logger.debug("in checking course")
 
         # loop through multiple course ids
-        for course_id in Course.objects.get_supported_courses():
+        for course_id, data_last_updated in Course.objects.get_supported_courses(include_last_updated=True):
             logger.debug(course_id)
             # select course based on course id
             course_sql = queries['course'].format(course_id=course_id)
@@ -154,8 +154,12 @@ def verify_course_ids(self):
 
             # error out when course id is invalid, otherwise add DataFrame to list
             if course_df.empty:
-                logger.error(f"""Course {course_id} don't have the entry in data warehouse yet. """)
-                invalid_course_id_list.append(course_id)
+                # Check if the course was ever locally updated. If it was updated it is invalid, otherwise don't consider this an error and skip it.
+                if data_last_updated:
+                    logger.error(f"""Course {course_id} doesn't have an entry in data warehouse yet. It has local data, so marking invalid.""")
+                    invalid_course_id_list.append(course_id)
+                else:
+                    logger.info(f"""Course {course_id} doesn't have an entry in data warehouse yet. It hasn't been updated locally, so skipping.""")
             else:
                 course_dfs.append(course_df)
 

dashboard/models.py

@@ -139,14 +139,19 @@ class Meta:
 
 
 class CourseQuerySet(models.QuerySet):
-    def get_supported_courses(self) -> QuerySet:
+    def get_supported_courses(self, include_last_updated=False) -> QuerySet:
         """Returns the list of supported courses from the database
 
+        :param include_last_updated: Whether or not to include the last updated value in the return, otherwise just returns the id's
         :return: [List of supported course ids]
         :rtype: [list of str (possibly incremented depending on parameter)]
         """
         try:
-            return self.values_list('id', flat=True)
+            # If we want to include the last updated date, otherwise just return a flat set
+            if include_last_updated:
+                return self.values_list('id', 'data_last_updated')
+            else:
+                return self.values_list('id', flat=True)
         except self.model.DoesNotExist:
             logger.info("Courses did not exist", exc_info = True)
         return Course.objects.none()