Skip to content

Commit

Permalink
Reverse counting logic in stream filters
Browse files Browse the repository at this point in the history 
We count all the exercises that are marked as 'read' so that we can compute the
'unread' count for the sidebar navigation in each category.

Since some things will be marked as read both based on a watermark and
based on direct views, we would double count them. To avoid that we need
to either exclude the relevant views when counting the watermarks, or
exclude the relevant watermarks when counting the relevant views.

The performance of using NOT IN against the views table (7MM rows) was
dismal to the point of taking down production when certain users
accessed the activity stream.

This reverses the logic so we're excluding watermarks when counting
direct views.
  • Loading branch information
Katrina Owen committed Oct 13, 2016
1 parent 603c43d commit 96744de
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 48 deletions.
60 changes: 30 additions & 30 deletions lib/exercism/team_stream_filters.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,12 @@ def views_sql
AND ex.user_id IN (#{user_ids_param})
AND views.user_id=#{viewer_id}
AND views.last_viewed_at > ex.last_activity_at
AND ex.language || ex.slug NOT IN (
SELECT track_id || slug
FROM watermarks
WHERE user_id=#{viewer_id}
AND at > ex.last_activity_at
)
SQL
end

Expand All @@ -109,12 +115,6 @@ def watermarks_sql
AND ex.iteration_count > 0
AND ex.user_id IN (#{user_ids_param})
AND mark.user_id=#{viewer_id}
AND ex.id NOT IN (
SELECT exercise_id
FROM views
WHERE user_id=#{viewer_id}
AND last_viewed_at > ex.last_activity_at
)
AND mark.at > ex.last_activity_at
SQL
end
Expand Down Expand Up @@ -159,6 +159,12 @@ def views_sql
AND ex.user_id IN (#{user_ids_param})
AND views.user_id=#{viewer_id}
AND views.last_viewed_at > ex.last_activity_at
AND ex.language || ex.slug NOT IN (
SELECT track_id || slug
FROM watermarks
WHERE user_id=#{viewer_id}
AND at > ex.last_activity_at
)
GROUP BY ex.language
SQL
end
Expand All @@ -174,12 +180,6 @@ def watermarks_sql
AND ex.iteration_count > 0
AND ex.user_id IN (#{user_ids_param})
AND mark.user_id=#{viewer_id}
AND ex.id NOT IN (
SELECT exercise_id
FROM views
WHERE user_id=#{viewer_id}
AND last_viewed_at > ex.last_activity_at
)
AND mark.at > ex.last_activity_at
GROUP BY ex.language
SQL
Expand Down Expand Up @@ -231,6 +231,12 @@ def views_sql
AND ex.language='#{current_id}'
AND views.user_id=#{viewer_id}
AND views.last_viewed_at > ex.last_activity_at
AND ex.language || ex.slug NOT IN (
SELECT track_id || slug
FROM watermarks
WHERE user_id=#{viewer_id}
AND at > ex.last_activity_at
)
GROUP BY ex.slug
SQL
end
Expand All @@ -247,12 +253,6 @@ def watermarks_sql
AND ex.user_id IN (#{user_ids_param})
AND ex.language='#{current_id}'
AND mark.user_id=#{viewer_id}
AND ex.id NOT IN (
SELECT exercise_id
FROM views
WHERE user_id=#{viewer_id}
AND last_viewed_at > ex.last_activity_at
)
AND mark.at > ex.last_activity_at
GROUP BY ex.slug
SQL
Expand Down Expand Up @@ -306,6 +306,12 @@ def views_sql
AND ex.user_id IN (#{user_ids_param})
AND views.user_id=#{viewer_id}
AND views.last_viewed_at > ex.last_activity_at
AND ex.language || ex.slug NOT IN (
SELECT track_id || slug
FROM watermarks
WHERE user_id=#{viewer_id}
AND at > ex.last_activity_at
)
GROUP BY u.username
SQL
end
Expand All @@ -323,12 +329,6 @@ def watermarks_sql
AND ex.iteration_count > 0
AND ex.user_id IN (#{user_ids_param})
AND mark.user_id=#{viewer_id}
AND ex.id NOT IN (
SELECT exercise_id
FROM views
WHERE user_id=#{viewer_id}
AND last_viewed_at > ex.last_activity_at
)
AND mark.at > ex.last_activity_at
GROUP BY u.username
SQL
Expand Down Expand Up @@ -374,6 +374,12 @@ def views_sql
AND ex.user_id IN (#{user_ids_param})
AND views.user_id=#{viewer_id}
AND views.last_viewed_at > ex.last_activity_at
AND ex.language || ex.slug NOT IN (
SELECT track_id || slug
FROM watermarks
WHERE user_id=#{viewer_id}
AND at > ex.last_activity_at
)
GROUP BY ex.language
SQL
end
Expand All @@ -389,12 +395,6 @@ def watermarks_sql
AND ex.iteration_count > 0
AND ex.user_id IN (#{user_ids_param})
AND mark.user_id=#{viewer_id}
AND ex.id NOT IN (
SELECT exercise_id
FROM views
WHERE user_id=#{viewer_id}
AND last_viewed_at > ex.last_activity_at
)
AND mark.at > ex.last_activity_at
GROUP BY ex.language
SQL
Expand Down
36 changes: 18 additions & 18 deletions lib/exercism/track_stream_filters.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,12 @@ def views_sql
AND ex.archived='f'
AND ex.iteration_count > 0
AND views.last_viewed_at > ex.last_activity_at
AND ex.language || ex.slug NOT IN (
SELECT track_id || slug
FROM watermarks
WHERE user_id=#{viewer_id}
AND at > ex.last_activity_at
)
GROUP BY ex.language
SQL
end
Expand All @@ -99,12 +105,6 @@ def watermarks_sql
AND ex.slug=mark.slug
AND acls.user_id=mark.user_id
WHERE acls.user_id=#{viewer_id}
AND ex.id NOT IN (
SELECT exercise_id
FROM views
WHERE user_id=#{viewer_id}
AND last_viewed_at > ex.last_activity_at
)
AND mark.at > ex.last_activity_at
GROUP BY ex.language
SQL
Expand Down Expand Up @@ -170,6 +170,12 @@ def views_sql
AND ex.archived='f'
AND ex.iteration_count > 0
AND views.last_viewed_at > ex.last_activity_at
AND ex.language || ex.slug NOT IN (
SELECT track_id || slug
FROM watermarks
WHERE user_id=#{viewer_id}
AND at > ex.last_activity_at
)
GROUP BY ex.slug
SQL
end
Expand All @@ -189,12 +195,6 @@ def watermarks_sql
AND ex.language='#{track_id}'
AND ex.archived='f'
AND ex.iteration_count > 0
AND ex.id NOT IN (
SELECT exercise_id
FROM views
WHERE user_id=#{viewer_id}
AND last_viewed_at > ex.last_activity_at
)
AND mark.at > ex.last_activity_at
GROUP BY ex.slug
SQL
Expand Down Expand Up @@ -253,6 +253,12 @@ def views_sql
AND ex.user_id=#{viewer_id}
AND ex.language='#{track_id}'
AND views.last_viewed_at > ex.last_activity_at
AND ex.language || ex.slug NOT IN (
SELECT track_id || slug
FROM watermarks
WHERE user_id=#{viewer_id}
AND at > ex.last_activity_at
)
GROUP BY u.username
SQL
end
Expand All @@ -271,12 +277,6 @@ def watermarks_sql
AND ex.iteration_count > 0
AND ex.user_id=#{viewer_id}
AND ex.language='#{track_id}'
AND ex.id NOT IN (
SELECT exercise_id
FROM views
WHERE user_id=#{viewer_id}
AND last_viewed_at > ex.last_activity_at
)
AND mark.at > ex.last_activity_at
GROUP BY u.username
SQL
Expand Down

0 comments on commit 96744de

Please sign in to comment.