Skip to content

Commit

Permalink
issue tl-its-umich-edu#827 update the cron query to execute in batch …
Browse files Browse the repository at this point in the history 
…mode, instead of retrieve info course by course
  • Loading branch information
@zqian
zqian committed Sep 6, 2022
1 parent 2435e43 commit f365937
Show file tree
Hide file tree
Showing 3 changed files with 108 additions and 118 deletions.
37 changes: 23 additions & 14 deletions config/cron.hjson
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,26 +7,26 @@
"user" :
'''
with
enroll_data as (select id as enroll_id, user_id, type from enrollment_dim where course_id='{course_id}'
enroll_data as (select id as enroll_id, user_id, type from enrollment_dim where course_id in %(course_ids)s
and type in ('StudentEnrollment', 'TaEnrollment', 'TeacherEnrollment') and workflow_state= 'active'),
user_info as (select p.unique_name,p.sis_user_id, u.name, u.id as user_id, u.global_canvas_id
from (SELECT ROW_NUMBER() OVER (PARTITION BY user_id order by sis_user_id asc) AS row_number, * FROM pseudonym_dim) as p
join user_dim u on u.id = p.user_id WHERE row_number = 1),
user_enroll as (select u.unique_name, u.sis_user_id, u.name, u.user_id, e.enroll_id,
u.global_canvas_id, e.type from enroll_data e join user_info u on e.user_id= u.user_id),
course_fact as (select enrollment_id, current_score, final_score from course_score_fact
where course_id='{course_id}'),
course_fact as (select enrollment_id, course_id, current_score, final_score from course_score_fact
where course_id in %(course_ids)s),
final as (select cast(u.global_canvas_id as BIGINT) as user_id,u.name, u.unique_name as sis_name,
'{course_id}' as course_id, c.current_score as current_grade, c.final_score as final_grade,
c.course_id as course_id, c.current_score as current_grade, c.final_score as final_grade,
u.type as enrollment_type
from user_enroll u left join course_fact c on u.enroll_id= c.enrollment_id)
select * from final
order by user_id
''',
"assignment_groups":
'''
with assignment_details as (select ad.due_at,ad.title,af.course_id ,af.assignment_id,af.points_possible,af.assignment_group_id from assignment_fact af inner join assignment_dim ad on af.assignment_id = ad.id where af.course_id='{course_id}' and ad.visibility = 'everyone' and ad.workflow_state='published'),
assignment_grp as (select agf.*, agd.name from assignment_group_dim agd join assignment_group_fact agf on agd.id = agf.assignment_group_id where agd.course_id='{course_id}' and workflow_state='available'),
with assignment_details as (select ad.due_at,ad.title,af.course_id ,af.assignment_id,af.points_possible,af.assignment_group_id from assignment_fact af inner join assignment_dim ad on af.assignment_id = ad.id where af.course_id in %(course_ids)s and ad.visibility = 'everyone' and ad.workflow_state='published'),
assignment_grp as (select agf.*, agd.name from assignment_group_dim agd join assignment_group_fact agf on agd.id = agf.assignment_group_id where agd.course_id in %(course_ids)s and workflow_state='available'),
assign_more as (select distinct(a.assignment_group_id) ,da.group_points from assignment_details a join (select assignment_group_id, sum(points_possible) as group_points from assignment_details group by assignment_group_id) as da on a.assignment_group_id = da.assignment_group_id ),
grp_full as (select a.group_points, b.assignment_group_id from assign_more a right join assignment_grp b on a.assignment_group_id = b.assignment_group_id),
assign_rules as (select DISTINCT ad.assignment_group_id,agr.drop_lowest,agr.drop_highest from grp_full ad join assignment_group_rule_dim agr on ad.assignment_group_id=agr.assignment_group_id),
Expand All @@ -41,16 +41,25 @@
(select ad.due_at AS due_date,ad.due_at at time zone 'America/Detroit' as local_date,
ad.title AS name,af.course_id AS course_id,af.assignment_id AS id,
af.points_possible AS points_possible,af.assignment_group_id AS assignment_group_id
from assignment_fact af inner join assignment_dim ad on af.assignment_id = ad.id where af.course_id='{course_id}'
from assignment_fact af inner join assignment_dim ad on af.assignment_id = ad.id where af.course_id in %(course_ids)s
and ad.visibility = 'everyone' and ad.workflow_state='published')
select * from assignment_info
order by id
''',
"assignment_weight":
'''
with course as (select course_id, sum(group_weight) as group_weight from assignment_group_fact
where course_id = '{course_id}' group by course_id having sum(group_weight)>1)
(select CASE WHEN EXISTS (SELECT * FROM course WHERE group_weight > 1) THEN CAST(1 AS BOOLEAN) ELSE CAST(0 AS BOOLEAN) END)
select
course_id,
case
when sum(group_weight) >1
then
CAST(1 AS BOOLEAN)
else
CAST(0 AS BOOLEAN)
end as consider_weight
from assignment_group_fact
where course_id in %(course_ids)s
group by course_id
''',
"term":
'''
Expand All @@ -60,7 +69,7 @@
'''
select id, canvas_id, enrollment_term_id, name, start_at, conclude_at
from course_dim c
where c.id = '{course_id}'
where c.id in %(course_ids)s
''',
"resource":
'''
Expand All @@ -80,15 +89,15 @@
global_canvas_id,
published_score
from submission_fact sf join user_dim u on sf.user_id = u.id
where course_id = '{course_id}'
where course_id in %(course_ids)s
),
enrollment as
(
select
distinct(user_id)
from enrollment_dim
where
course_id = '{course_id}'
course_id in %(course_ids)s
and workflow_state='active'
and type = 'StudentEnrollment'
),
Expand Down Expand Up @@ -116,7 +125,7 @@
from
assignment_dim a join sub_with_enroll s on s.assignment_id=a.id
where
a.course_id='{course_id}'
a.course_id in %(course_ids)s
and a.workflow_state='published'
),
assign_sub_time as
Expand Down
75 changes: 40 additions & 35 deletions config/cron_udp.hjson
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,16 @@
"metadata":
## placeholder query before the unizin_metadata table is implemented in UDP
'''
select "learner_activity" as pkey, max(updated_date) as pvalue from entity.learner_activity
select 'canvasdatadate' as pkey, min(dag_run) as pvalue from report.publish_info pi2
''',
"user" :
'''
select
({canvas_data_id_increment} + cast(p2.lms_ext_id as bigint)) as user_id,
(
cast(%(canvas_data_id_increment)s as bigint)
+
cast(p2.lms_ext_id as bigint)
) as user_id,
case
when p.first_name is not null then p.first_name || ' ' || p.last_name
else p2.sis_ext_id end as name,
Expand Down Expand Up @@ -38,7 +42,7 @@
left join entity.course_grade cg
on cse.course_section_id = cg.course_section_id and cse.person_id = cg.person_id
where
co.lms_int_id ='{course_id}'
co.lms_int_id in %(course_ids)s
and cse.role in ('Student', 'TeachingAssistant', 'Teacher')
and cse.role_status = 'Enrolled'
order by user_id
Expand All @@ -52,14 +56,14 @@
la.visibility = 'everyone'
and la.status = 'published'
and la.course_offering_id = co.id
and co.lms_int_id = '{course_id}'
and co.lms_int_id in %(course_ids)s
), assignment_grp as (
select lg.*
from entity.learner_activity_group lg, keymap.course_offering co
where
lg.status = 'available'
and lg.course_offering_id = co.id
and co.lms_int_id = '{course_id}'
and co.lms_int_id in %(course_ids)s
), assign_more as (
select distinct(a.learner_activity_group_id), da.group_points
from assignment_details a
Expand Down Expand Up @@ -121,7 +125,7 @@
la.visibility = 'everyone'
and la.status = 'published'
and la.course_offering_id = co.id
and co.lms_int_id = '{course_id}'
and co.lms_int_id in %(course_ids)s
and la.learner_activity_id = la_km.id
and la.learner_activity_group_id = lag_km.id
)
Expand All @@ -130,21 +134,21 @@
''',
"assignment_weight":
'''
with course as (
select course_offering_id as course_id,
sum(group_weight) as group_weight
from entity.learner_activity_group lag, keymap.course_offering co_km
where
select
co_km.lms_int_id as course_id,
case when sum(group_weight) > 1
then
cast(1 as boolean)
else
cast(0 as boolean)
end as consider_weight
from
entity.learner_activity_group lag,
keymap.course_offering co_km
where
lag.course_offering_id = co_km.id
and co_km.lms_int_id = '{course_id}'
group by course_offering_id
having sum(group_weight) > 1
)
(select case when exists (
select *
from course
where group_weight > 1
) then cast(1 as boolean) else cast(0 as boolean) end)
and co_km.lms_int_id in %(course_ids)s
group by co_km.lms_int_id
''',
"term":
'''
Expand Down Expand Up @@ -180,7 +184,7 @@
keymap.course_offering co2,
keymap.academic_term at2,
no_term_temp
WHERE co2.lms_int_id = '{course_id}'
WHERE co2.lms_int_id in %(course_ids)s
and co.course_offering_id = co2.id
and (
(co.academic_session_id is null and at2.id = no_term_temp.academic_term_id)
Expand Down Expand Up @@ -215,35 +219,35 @@
left join keymap.learner_activity la2 on la.learner_activity_id = la2.id
left join keymap.course_offering co on co.id = la.course_offering_id
where
co.lms_int_id = '{course_id}'
co.lms_int_id in %(course_ids)s
and la.status = 'published'
),
enrollment as
),
enrollment as
(
select
distinct cse.person_id as short_user_id
from entity.course_section_enrollment cse
left join entity.course_section cs on cse.course_section_id = cs.course_section_id
left join keymap.course_offering co on cs.course_offering_id = co.id
where
co.lms_int_id = '{course_id}'
co.lms_int_id in %(course_ids)s
and cse.role_status ='Enrolled'
and cse."role" = 'Student'
),
course_assignment_enrollment as
(
select *
from
course_assignment cross join enrollment
),
course_assignment_submission as
),
course_assignment_enrollment as
(
select
select *
from
course_assignment cross join enrollment
),
course_assignment_submission as
(
select
cast(lar2.lms_int_id as BIGINT) as submission_id,
cae.assignment_id as assignment_id,
cae.course_id as course_id,
lar.published_score as published_score,
({canvas_data_id_increment} + cast(p.lms_ext_id as bigint)) as user_id,
(cast(%(canvas_data_id_increment)s as bigint) + cast(p.lms_ext_id as bigint)) as user_id,
lar.response_date as submitted_at,
lar.graded_date as graded_at,
lar.posted_at at time zone 'America/Detroit' as grade_posted_local_date,
Expand Down Expand Up @@ -310,6 +314,7 @@
group by assignment_id
) as f1
on f.assignment_id = f1.assignment_id
where f.id is not null
order by assignment_id, user_id
'''
}

0 comments on commit f365937

Please sign in to comment.