Merge pull request #948 from uw-it-aca/develop

Develop

docker/prod-values.yml

@@ -248,17 +248,6 @@ cronjob:
         requests:
           cpu: 25m
           memory: 128Mi
-    - name: delete-unused-courses
-      schedule: "30 11 * * 1-5"
-      command: ["/scripts/management_command.sh"]
-      args: ["delete_unused_courses"]
-      resources:
-        limits:
-          cpu: 500m
-          memory: 256Mi
-        requests:
-          cpu: 25m
-          memory: 128Mi
     - name: sync-external-tools
       schedule: "15 14 * * 1-6"
       command: ["/scripts/management_command.sh"]

sis_provisioner/builders/courses.py

@@ -2,7 +2,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
 
-from django.utils.timezone import utc
 from sis_provisioner.builders import Builder
 from sis_provisioner.csv.format import CourseCSV, SectionCSV, TermCSV, XlistCSV
 from sis_provisioner.dao.course import (
@@ -16,7 +15,6 @@
 from restclients_core.exceptions import DataFailureException
 from uw_sws.exceptions import InvalidCanvasIndependentStudyCourse
 from logging import getLogger
-from datetime import datetime
 import csv
 import re
 

sis_provisioner/builders/enrollments.py

@@ -12,9 +12,8 @@
 from uw_sws.models import Registration
 from uw_sws.exceptions import InvalidCanvasIndependentStudyCourse
 from restclients_core.exceptions import DataFailureException
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from django.conf import settings
-from django.utils.timezone import utc
 
 
 class EnrollmentBuilder(Builder):
@@ -123,7 +122,7 @@ def _skip_enrollment_event(self, enrollment, err):
             enrollment.reg_id, enrollment.course_id, err))
 
     def _init_build(self, **kwargs):
-        now = datetime.utcnow().replace(tzinfo=utc)
+        now = datetime.now(timezone.utc)
         timeout = getattr(settings, 'MISSING_LOGIN_ID_RETRY_TIMEOUT', 48)
         self.retry_missing_id = now - timedelta(hours=timeout)
 
@@ -133,7 +132,7 @@ class InvalidEnrollmentBuilder(Builder):
     Generates import data for each of the passed InvalidEnrollment models.
     """
     def _process(self, inv_enrollment):
-        now = datetime.utcnow().replace(tzinfo=utc)
+        now = datetime.now(timezone.utc)
         grace_dt = now - timedelta(days=getattr(
             settings, 'INVALID_ENROLLMENT_GRACE_DAYS', 90))
         status = None

sis_provisioner/dao/group.py

@@ -3,14 +3,14 @@
 
 
 from django.conf import settings
-from django.utils.timezone import utc
 from uw_gws import GWS
 from uw_gws.exceptions import InvalidGroupID
 from restclients_core.exceptions import DataFailureException
 from sis_provisioner.dao.user import valid_net_id
 from sis_provisioner.exceptions import (
     UserPolicyException, GroupPolicyException, GroupNotFoundException,
     GroupUnauthorizedException)
+from datetime import timezone
 from logging import getLogger
 import re
 
@@ -33,7 +33,7 @@ def valid_group_id(group_id):
 def is_modified_group(group_id, changed_since_dt):
     try:
         group = GWS().get_group_by_id(group_id)
-        member_mtime = group.membership_modified.replace(tzinfo=utc)
+        member_mtime = group.membership_modified.replace(tzinfo=timezone.utc)
         return (member_mtime > changed_since_dt)
     except DataFailureException as err:
         if err.status == 404:

sis_provisioner/management/commands/__init__.py

@@ -4,10 +4,9 @@
 
 from django.core.management.base import BaseCommand
 from django.core.mail import mail_admins
-from django.utils.timezone import utc
 from sis_provisioner.models import Job
 from logging import getLogger
-import datetime
+from datetime import datetime, timedelta, timezone
 import sys
 
 
@@ -29,26 +28,26 @@ def is_active_job(self):
             job = Job(name=name,
                       title=self.title_from_name(name),
                       is_active=False)
-            job.changed_date = datetime.datetime.utcnow().replace(tzinfo=utc)
+            job.changed_date = datetime.now(timezone.utc)
             job.save()
 
         return True if job.is_active else False
 
     def update_job(self):
         job = Job.objects.get(name=self.name_from_argv())
-        job.last_run_date = datetime.datetime.utcnow().replace(tzinfo=utc)
+        job.last_run_date = datetime.now(timezone.utc)
         job.save()
 
     def health_check(self):
         """Override to sanity check specific job environment"""
         pass
 
     def squawk(self, message="Problem with Provisioning Job"):
-        now = datetime.datetime.utcnow().replace(tzinfo=utc)
+        now = datetime.now(timezone.utc)
         job = Job.objects.get(name=self.name_from_argv())
         job.health_status = message
         if (not job.last_status_date or
-                (now - job.last_status_date) > datetime.timedelta(hours=1)):
+                (now - job.last_status_date) > timedelta(hours=1)):
             try:
                 mail_admins(
                     'Provisioning job "{}" may be having issues'.format(

sis_provisioner/management/commands/generate_user_report.py

@@ -6,7 +6,7 @@
 from django.conf import settings
 from sis_provisioner.dao.user import valid_reg_id, valid_gmail_id
 from sis_provisioner.exceptions import UserPolicyException
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 import csv
 
 
@@ -112,4 +112,5 @@ def handle(self, *args, **options):
     def logged_in_past_year(self, last_access_str):
         last_access_dt = datetime.strptime(last_access_str[:-6],
                                            '%Y-%m-%dT%H:%M:%S')
-        return last_access_dt < datetime.utcnow() - timedelta(days=365)
+        return last_access_dt < (
+            datetime.now(timezone.utc) - timedelta(days=365))

sis_provisioner/models/__init__.py

@@ -4,13 +4,13 @@
 
 from django.db import models
 from django.db.models import Q
-from django.utils.timezone import utc, localtime
+from django.utils.timezone import localtime
 from sis_provisioner.dao.canvas import (
     sis_import_by_path, get_sis_import_status, delete_sis_import)
 from sis_provisioner.exceptions import MissingImportPathException
 from restclients_core.exceptions import DataFailureException
 from importlib import import_module
-from datetime import datetime
+from datetime import datetime, timezone
 from logging import getLogger
 import json
 import re
@@ -160,7 +160,7 @@ def update_import_status(self):
         try:
             sis_import = get_sis_import_status(self.canvas_id)
             self.monitor_status = 200
-            self.monitor_date = datetime.utcnow().replace(tzinfo=utc)
+            self.monitor_date = datetime.now(timezone.utc)
             self.canvas_state = sis_import.workflow_state
             self.canvas_progress = sis_import.progress
             self.canvas_warnings = None

sis_provisioner/models/admin.py

@@ -4,15 +4,15 @@
 
 from django.db import models
 from django.conf import settings
-from django.utils.timezone import utc, localtime
+from django.utils.timezone import localtime
 from sis_provisioner.dao.astra import ASTRA
 from sis_provisioner.dao.canvas import get_account_role_data
 from sis_provisioner.models import Import, ImportResource
 from sis_provisioner.models.account import Account
 from sis_provisioner.models.user import User
 from sis_provisioner.exceptions import (
     AccountPolicyException, EmptyQueueException)
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 
 
 class AdminManager(models.Manager):
@@ -64,7 +64,7 @@ def start_reconcile(self, queue_id):
 
     def finish_reconcile(self, queue_id):
         retention = getattr(settings, 'REMOVED_ADMIN_RETENTION_DAYS', 90)
-        now_dt = datetime.utcnow().replace(tzinfo=utc)
+        now_dt = datetime.now(timezone.utc)
         retention_dt = now_dt - timedelta(days=retention)
 
         # Set deleted date for admins who were just deleted

sis_provisioner/models/course.py

@@ -5,7 +5,7 @@
 from django.db import models
 from django.db.models import Q
 from django.conf import settings
-from django.utils.timezone import utc, localtime
+from django.utils.timezone import localtime
 from sis_provisioner.models import Import, ImportResource
 from sis_provisioner.models.group import Group
 from sis_provisioner.models.user import User
@@ -19,7 +19,7 @@
 from sis_provisioner.exceptions import (
     CoursePolicyException, EmptyQueueException)
 from restclients_core.exceptions import DataFailureException
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from logging import getLogger
 
 logger = getLogger(__name__)
@@ -187,14 +187,15 @@ def add_all_courses_for_term(self, term):
                 term_id=term_id, course_type=Course.SDB_TYPE
             ).values_list('course_id', 'priority')))
 
-        last_search_date = datetime.utcnow().replace(tzinfo=utc)
+        last_search_date = datetime.now(timezone.utc)
         try:
             delta = Term.objects.get(term_id=term_id)
         except Term.DoesNotExist:
             delta = Term(term_id=term_id)
 
         if delta.last_course_search_date is None:
-            delta.courses_changed_since_date = datetime.fromtimestamp(0, utc)
+            delta.courses_changed_since_date = datetime.fromtimestamp(
+                0, timezone.utc)
         else:
             delta.courses_changed_since_date = (
                 delta.last_course_search_date - timedelta(days=1))
@@ -303,10 +304,10 @@ def update_priority(self, priority):
 
     @property
     def default_expiration_date(self):
-        now = datetime.now()
+        now = datetime.now(timezone.utc)
         expiration = datetime(
             now.year + self.RETENTION_LIFE_SPAN, self.RETENTION_EXPIRE_MONTH,
-            self.RETENTION_EXPIRE_DAY, 12).replace(tzinfo=utc)
+            self.RETENTION_EXPIRE_DAY, 12, 0, 0, tzinfo=timezone.utc)
         try:
             (year, quarter, c, n, s) = self.course_id.split('-')
             year = int(year) + self.RETENTION_LIFE_SPAN + (1 if (
@@ -324,7 +325,7 @@ def default_expiration_date(self):
 
     def is_expired(self):
         if self.expiration_date is not None:
-            return self.expiration_date < datetime.utcnow().replace(tzinfo=utc)
+            return self.expiration_date < datetime.now(timezone.utc)
         return False
 
     def json_data(self, include_sws_url=False):
@@ -425,18 +426,17 @@ def queued_adhoc_courses(self, queue_id):
             queue_id=queue_id, course_type=Course.ADHOC_TYPE)
 
     def dequeue(self, sis_import):
-        utcnow = datetime.utcnow().replace(tzinfo=utc)
         if sis_import.is_imported():
             # Dequeue the sdb courses
             self.queued_sdb_courses(sis_import.pk).update(
                 queue_id=None, priority=Course.PRIORITY_NONE,
-                deleted_date=utcnow)
+                deleted_date=datetime.now(timezone.utc))
 
         # Delete the adhoc courses via the Canvas api
         for course in self.queued_adhoc_courses(sis_import.pk):
             try:
                 delete_course(course.canvas_course_id)
-                course.deleted_date = utcnow
+                course.deleted_date = datetime.now(timezone.utc)
                 logger.info(f"DELETE adhoc course '{course.canvas_course_id}'")
             except DataFailureException as err:
                 course.provisioned_error = True

sis_provisioner/models/enrollment.py

@@ -5,7 +5,7 @@
 from django.db import models, IntegrityError
 from django.db.models import F
 from django.conf import settings
-from django.utils.timezone import utc, localtime
+from django.utils.timezone import localtime
 from sis_provisioner.models import Import, ImportResource
 from sis_provisioner.models.course import Course
 from sis_provisioner.models.user import User
@@ -14,7 +14,7 @@
     get_instructor_sis_import_role, ENROLLMENT_ACTIVE)
 from sis_provisioner.exceptions import EmptyQueueException
 from restclients_core.exceptions import DataFailureException
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from logging import getLogger
 
 logger = getLogger(__name__)
@@ -60,7 +60,7 @@ def dequeue(self, sis_import):
         self.purge_expired()
 
     def purge_expired(self):
-        retention_dt = datetime.utcnow().replace(tzinfo=utc) - timedelta(
+        retention_dt = datetime.now(timezone.utc) - timedelta(
             days=getattr(settings, 'ENROLLMENT_EVENT_RETENTION_DAYS', 180))
         return super(EnrollmentManager, self).get_queryset().filter(
             priority=Enrollment.PRIORITY_NONE,
@@ -87,7 +87,8 @@ def _log(outcome, status, full_course_id, reg_id, duplicate_code, role,
         reg_id = enrollment_data.get('UWRegID')
         role = enrollment_data.get('Role')
         status = enrollment_data.get('Status').lower()
-        last_modified = enrollment_data.get('LastModified').replace(tzinfo=utc)
+        last_modified = enrollment_data.get('LastModified').replace(
+            tzinfo=timezone.utc)
         request_date = enrollment_data.get('RequestDate')
         duplicate_code = enrollment_data.get('DuplicateCode', '')
         instructor_reg_id = enrollment_data.get('InstructorUWRegID', None)

sis_provisioner/models/external_tools.py

@@ -4,12 +4,12 @@
 
 from django.conf import settings
 from django.db import models
-from django.utils.timezone import utc, localtime
+from django.utils.timezone import localtime
 from sis_provisioner.models.account import Account
 from sis_provisioner.dao.canvas import (
     get_account_by_id, get_sub_accounts, get_external_tools,
     create_external_tool, update_external_tool, delete_external_tool)
-from datetime import datetime
+from datetime import datetime, timezone
 import string
 import random
 import json
@@ -55,7 +55,7 @@ def import_tools_in_account(self, account_id, changed_by):
 
             tool.config = json.dumps(config)
             tool.changed_by = changed_by
-            tool.changed_date = datetime.utcnow().replace(tzinfo=utc)
+            tool.changed_date = datetime.now(timezone.utc)
             tool.queue_id = None
             tool.save()
 
@@ -67,7 +67,7 @@ def create_tool(self, account_id, config, created_by):
         tool = ExternalTool(
             account=account,
             changed_by=created_by,
-            changed_date=datetime.utcnow().replace(tzinfo=utc))
+            changed_date=datetime.now(timezone.utc))
 
         try:
             keystore = BLTIKeyStore.objects.get(
@@ -87,7 +87,7 @@ def create_tool(self, account_id, config, created_by):
 
         tool.canvas_id = new_config.get('id')
         tool.config = json.dumps(new_config)
-        tool.provisioned_date = datetime.utcnow().replace(tzinfo=utc)
+        tool.provisioned_date = datetime.now(timezone.utc)
         tool.save()
         return tool
 
@@ -113,8 +113,8 @@ def update_tool(self, canvas_id, config, updated_by):
 
         tool.config = json.dumps(new_config)
         tool.changed_by = updated_by
-        tool.changed_date = datetime.utcnow().replace(tzinfo=utc)
-        tool.provisioned_date = datetime.utcnow().replace(tzinfo=utc)
+        tool.changed_date = datetime.now(timezone.utc)
+        tool.provisioned_date = datetime.now(timezone.utc)
         tool.save()
         return tool
 

sis_provisioner/models/group.py

@@ -5,14 +5,14 @@
 from django.db import models
 from django.db.models import F
 from django.conf import settings
-from django.utils.timezone import utc, localtime
+from django.utils.timezone import localtime
 from restclients_core.exceptions import DataFailureException
 from sis_provisioner.dao.group import is_modified_group
 from sis_provisioner.models import Import, ImportResource
 from sis_provisioner.models.user import User
 from sis_provisioner.exceptions import (
     EmptyQueueException, GroupNotFoundException)
-from datetime import datetime
+from datetime import datetime, timezone
 from logging import getLogger
 
 logger = getLogger(__name__)
@@ -145,7 +145,7 @@ def delete_group_not_found(self, group_id):
             group_id=group_id, is_deleted__isnull=True
         ).update(
             is_deleted=True, deleted_by='gws',
-            deleted_date=datetime.utcnow().replace(tzinfo=utc)
+            deleted_date=datetime.now(timezone.utc)
         )