Skip to content
Permalink
Browse files
Add trending links (#16917)
* Add trending links

* Add overriding specific links trendability

* Add link type to preview cards and only trend articles

Change trends review notifications from being sent every 5 minutes to being sent every 2 hours

Change threshold from 5 unique accounts to 15 unique accounts

* Fix tests
  • Loading branch information
Gargron committed Nov 25, 2021
1 parent 46e62fc commit 6e50134a42cb303e6e42f89f9ddb5aacf83e7a6d
Showing with 2,064 additions and 715 deletions.
  1. +1 −1 app/chewy/tags_index.rb
  2. +1 −1 app/controllers/admin/dashboard_controller.rb
  3. +3 −73 app/controllers/admin/tags_controller.rb
  4. +41 −0 app/controllers/admin/trends/links/preview_card_providers_controller.rb
  5. +45 −0 app/controllers/admin/trends/links_controller.rb
  6. +41 −0 app/controllers/admin/trends/tags_controller.rb
  7. +2 −1 app/controllers/api/v1/admin/dimensions_controller.rb
  8. +2 −1 app/controllers/api/v1/admin/measures_controller.rb
  9. +16 −0 app/controllers/api/v1/admin/trends/tags_controller.rb
  10. +0 −16 app/controllers/api/v1/admin/trends_controller.rb
  11. +21 −0 app/controllers/api/v1/trends/links_controller.rb
  12. +21 −0 app/controllers/api/v1/trends/tags_controller.rb
  13. +0 −15 app/controllers/api/v1/trends_controller.rb
  14. +2 −0 app/helpers/admin/filter_helper.rb
  15. +94 −0 app/helpers/languages_helper.rb
  16. +1 −88 app/helpers/settings_helper.rb
  17. +3 −2 app/javascript/mastodon/components/admin/Counter.js
  18. +3 −2 app/javascript/mastodon/components/admin/Dimension.js
  19. +1 −1 app/javascript/mastodon/components/admin/Trends.js
  20. +16 −0 app/javascript/styles/mastodon/accounts.scss
  21. +10 −0 app/javascript/styles/mastodon/dashboard.scss
  22. +0 −2 app/lib/activitypub/activity.rb
  23. +2 −3 app/lib/activitypub/activity/announce.rb
  24. +6 −1 app/lib/activitypub/activity/create.rb
  25. +7 −2 app/lib/admin/metrics/dimension.rb
  26. +11 −2 app/lib/admin/metrics/dimension/base_dimension.rb
  27. +3 −1 app/lib/admin/metrics/dimension/languages_dimension.rb
  28. +36 −0 app/lib/admin/metrics/dimension/tag_languages_dimension.rb
  29. +35 −0 app/lib/admin/metrics/dimension/tag_servers_dimension.rb
  30. +8 −2 app/lib/admin/metrics/measure.rb
  31. +2 −2 app/lib/admin/metrics/measure/active_users_measure.rb
  32. +12 −3 app/lib/admin/metrics/measure/base_measure.rb
  33. +2 −2 app/lib/admin/metrics/measure/interactions_measure.rb
  34. +41 −0 app/lib/admin/metrics/measure/tag_accounts_measure.rb
  35. +47 −0 app/lib/admin/metrics/measure/tag_servers_measure.rb
  36. +41 −0 app/lib/admin/metrics/measure/tag_uses_measure.rb
  37. +48 −1 app/lib/link_details_extractor.rb
  38. +17 −5 app/mailers/admin_mailer.rb
  39. +2 −2 app/models/account_statuses_cleanup_policy.rb
  40. +65 −0 app/models/form/preview_card_batch.rb
  41. +33 −0 app/models/form/preview_card_provider_batch.rb
  42. +6 −2 app/models/form/tag_batch.rb
  43. +38 −4 app/models/preview_card.rb
  44. +53 −0 app/models/preview_card_filter.rb
  45. +57 −0 app/models/preview_card_provider.rb
  46. +49 −0 app/models/preview_card_provider_filter.rb
  47. +4 −19 app/models/tag.rb
  48. +35 −21 app/models/tag_filter.rb
  49. +0 −128 app/models/trending_tags.rb
  50. +27 −0 app/models/trends.rb
  51. +80 −0 app/models/trends/base.rb
  52. +98 −0 app/models/trends/history.rb
  53. +117 −0 app/models/trends/links.rb
  54. +111 −0 app/models/trends/tags.rb
  55. +11 −0 app/policies/preview_card_policy.rb
  56. +11 −0 app/policies/preview_card_provider_policy.rb
  57. +5 −0 app/serializers/rest/trends/link_serializer.rb
  58. +2 −1 app/services/fetch_link_card_service.rb
  59. +2 −1 app/services/post_status_service.rb
  60. +1 −1 app/services/process_hashtags_service.rb
  61. +2 −11 app/services/reblog_service.rb
  62. +1 −1 app/views/admin/dashboard/index.html.haml
  63. +0 −19 app/views/admin/tags/_tag.html.haml
  64. +0 −74 app/views/admin/tags/index.html.haml
  65. +44 −24 app/views/admin/tags/show.html.haml
  66. +30 −0 app/views/admin/trends/links/_preview_card.html.haml
  67. +41 −0 app/views/admin/trends/links/index.html.haml
  68. +16 −0 app/views/admin/trends/links/preview_card_providers/_preview_card_provider.html.haml
  69. +43 −0 app/views/admin/trends/links/preview_card_providers/index.html.haml
  70. +24 −0 app/views/admin/trends/tags/_tag.html.haml
  71. +38 −0 app/views/admin/trends/tags/index.html.haml
  72. +16 −0 app/views/admin_mailer/new_trending_links.text.erb
  73. +0 −5 app/views/admin_mailer/new_trending_tag.text.erb
  74. +16 −0 app/views/admin_mailer/new_trending_tags.text.erb
  75. +1 −1 app/views/application/_sidebar.html.haml
  76. +2 −2 app/workers/scheduler/{trending_tags_scheduler.rb → trends/refresh_scheduler.rb}
  77. +11 −0 app/workers/scheduler/trends/review_notifications_scheduler.rb
  78. +81 −31 config/brakeman.ignore
  79. +55 −18 config/locales/en.yml
  80. +2 −2 config/locales/simple_form.en.yml
  81. +5 −1 config/navigation.rb
  82. +29 −7 config/routes.rb
  83. +6 −2 config/sidekiq.yml
  84. +12 −0 db/migrate/20211031031021_create_preview_card_providers.rb
  85. +7 −0 db/migrate/20211112011713_add_language_to_preview_cards.rb
  86. +5 −0 db/migrate/20211115032527_add_trendable_to_preview_cards.rb
  87. +5 −0 db/migrate/20211123212714_add_link_type_to_preview_cards.rb
  88. +20 −1 db/schema.rb
  89. +1 −4 lib/mastodon/snowflake.rb
  90. +1 −1 lib/tasks/repo.rake
  91. +0 −12 spec/controllers/admin/tags_controller_spec.rb
  92. +22 −0 spec/controllers/api/v1/trends/tags_controller_spec.rb
  93. +0 −18 spec/controllers/api/v1/trends_controller_spec.rb
  94. +2 −7 spec/helpers/{settings_helper_spec.rb → languages_helper_spec.rb}
  95. +10 −0 spec/mailers/previews/admin_mailer_preview.rb
  96. +0 −68 spec/models/trending_tags_spec.rb
  97. +67 −0 spec/models/trends/tags_spec.rb
@@ -31,7 +31,7 @@ class TagsIndex < Chewy::Index
end

field :reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? }
field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day[:accounts].to_i } }
field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day.accounts } }
field :last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at }
end
end
@@ -4,7 +4,7 @@ module Admin
class DashboardController < BaseController
def index
@system_checks = Admin::SystemCheck.perform
@time_period = (1.month.ago.to_date...Time.now.utc.to_date)
@time_period = (29.days.ago.to_date...Time.now.utc.to_date)
@pending_users_count = User.pending.count
@pending_reports_count = Report.unresolved.count
@pending_tags_count = Tag.pending_review.count
@@ -2,38 +2,12 @@

module Admin
class TagsController < BaseController
before_action :set_tag, except: [:index, :batch, :approve_all, :reject_all]
before_action :set_usage_by_domain, except: [:index, :batch, :approve_all, :reject_all]
before_action :set_counters, except: [:index, :batch, :approve_all, :reject_all]

def index
authorize :tag, :index?

@tags = filtered_tags.page(params[:page])
@form = Form::TagBatch.new
end

def batch
@form = Form::TagBatch.new(form_tag_batch_params.merge(current_account: current_account, action: action_from_button))
@form.save
rescue ActionController::ParameterMissing
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
ensure
redirect_to admin_tags_path(filter_params)
end

def approve_all
Form::TagBatch.new(current_account: current_account, tag_ids: Tag.pending_review.pluck(:id), action: 'approve').save
redirect_to admin_tags_path(filter_params)
end

def reject_all
Form::TagBatch.new(current_account: current_account, tag_ids: Tag.pending_review.pluck(:id), action: 'reject').save
redirect_to admin_tags_path(filter_params)
end
before_action :set_tag

def show
authorize @tag, :show?

@time_period = (6.days.ago.to_date...Time.now.utc.to_date)
end

def update
@@ -52,52 +26,8 @@ def set_tag
@tag = Tag.find(params[:id])
end

def set_usage_by_domain
@usage_by_domain = @tag.statuses
.with_public_visibility
.excluding_silenced_accounts
.where(Status.arel_table[:id].gteq(Mastodon::Snowflake.id_at(Time.now.utc.beginning_of_day)))
.joins(:account)
.group('accounts.domain')
.reorder(statuses_count: :desc)
.pluck(Arel.sql('accounts.domain, count(*) AS statuses_count'))
end

def set_counters
@accounts_today = @tag.history.first[:accounts]
@accounts_week = Redis.current.pfcount(*current_week_days.map { |day| "activity:tags:#{@tag.id}:#{day}:accounts" })
end

def filtered_tags
TagFilter.new(filter_params).results
end

def filter_params
params.slice(:page, *TagFilter::KEYS).permit(:page, *TagFilter::KEYS)
end

def tag_params
params.require(:tag).permit(:name, :trendable, :usable, :listable)
end

def current_week_days
now = Time.now.utc.beginning_of_day.to_date

(Date.commercial(now.cwyear, now.cweek)..now).map do |date|
date.to_time(:utc).beginning_of_day.to_i
end
end

def form_tag_batch_params
params.require(:form_tag_batch).permit(:action, tag_ids: [])
end

def action_from_button
if params[:approve]
'approve'
elsif params[:reject]
'reject'
end
end
end
end
@@ -0,0 +1,41 @@
# frozen_string_literal: true

class Admin::Trends::Links::PreviewCardProvidersController < Admin::BaseController
def index
authorize :preview_card_provider, :index?

@preview_card_providers = filtered_preview_card_providers.page(params[:page])
@form = Form::PreviewCardProviderBatch.new
end

def batch
@form = Form::PreviewCardProviderBatch.new(form_preview_card_provider_batch_params.merge(current_account: current_account, action: action_from_button))
@form.save
rescue ActionController::ParameterMissing
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
ensure
redirect_to admin_trends_links_preview_card_providers_path(filter_params)
end

private

def filtered_preview_card_providers
PreviewCardProviderFilter.new(filter_params).results
end

def filter_params
params.slice(:page, *PreviewCardProviderFilter::KEYS).permit(:page, *PreviewCardProviderFilter::KEYS)
end

def form_preview_card_provider_batch_params
params.require(:form_preview_card_provider_batch).permit(:action, preview_card_provider_ids: [])
end

def action_from_button
if params[:approve]
'approve'
elsif params[:reject]
'reject'
end
end
end
@@ -0,0 +1,45 @@
# frozen_string_literal: true

class Admin::Trends::LinksController < Admin::BaseController
def index
authorize :preview_card, :index?

@preview_cards = filtered_preview_cards.page(params[:page])
@form = Form::PreviewCardBatch.new
end

def batch
@form = Form::PreviewCardBatch.new(form_preview_card_batch_params.merge(current_account: current_account, action: action_from_button))
@form.save
rescue ActionController::ParameterMissing
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
ensure
redirect_to admin_trends_links_path(filter_params)
end

private

def filtered_preview_cards
PreviewCardFilter.new(filter_params.with_defaults(trending: 'all')).results
end

def filter_params
params.slice(:page, *PreviewCardFilter::KEYS).permit(:page, *PreviewCardFilter::KEYS)
end

def form_preview_card_batch_params
params.require(:form_preview_card_batch).permit(:action, preview_card_ids: [])
end

def action_from_button
if params[:approve]
'approve'
elsif params[:approve_all]
'approve_all'
elsif params[:reject]
'reject'
elsif params[:reject_all]
'reject_all'
end
end
end
@@ -0,0 +1,41 @@
# frozen_string_literal: true

class Admin::Trends::TagsController < Admin::BaseController
def index
authorize :tag, :index?

@tags = filtered_tags.page(params[:page])
@form = Form::TagBatch.new
end

def batch
@form = Form::TagBatch.new(form_tag_batch_params.merge(current_account: current_account, action: action_from_button))
@form.save
rescue ActionController::ParameterMissing
flash[:alert] = I18n.t('admin.accounts.no_account_selected')
ensure
redirect_to admin_trends_tags_path(filter_params)
end

private

def filtered_tags
TagFilter.new(filter_params).results
end

def filter_params
params.slice(:page, *TagFilter::KEYS).permit(:page, *TagFilter::KEYS)
end

def form_tag_batch_params
params.require(:form_tag_batch).permit(:action, tag_ids: [])
end

def action_from_button
if params[:approve]
'approve'
elsif params[:reject]
'reject'
end
end
end
@@ -17,7 +17,8 @@ def set_dimensions
params[:keys],
params[:start_at],
params[:end_at],
params[:limit]
params[:limit],
params
)
end
end
@@ -16,7 +16,8 @@ def set_measures
@measures = Admin::Metrics::Measure.retrieve(
params[:keys],
params[:start_at],
params[:end_at]
params[:end_at],
params
)
end
end
@@ -0,0 +1,16 @@
# frozen_string_literal: true

class Api::V1::Admin::Trends::TagsController < Api::BaseController
before_action :require_staff!
before_action :set_tags

def index
render json: @tags, each_serializer: REST::Admin::TagSerializer
end

private

def set_tags
@tags = Trends.tags.get(false, limit_param(10))
end
end

This file was deleted.

@@ -0,0 +1,21 @@
# frozen_string_literal: true

class Api::V1::Trends::LinksController < Api::BaseController
before_action :set_links

def index
render json: @links, each_serializer: REST::Trends::LinkSerializer
end

private

def set_links
@links = begin
if Setting.trends
Trends.links.get(true, limit_param(10))
else
[]
end
end
end
end
@@ -0,0 +1,21 @@
# frozen_string_literal: true

class Api::V1::Trends::TagsController < Api::BaseController
before_action :set_tags

def index
render json: @tags, each_serializer: REST::TagSerializer
end

private

def set_tags
@tags = begin
if Setting.trends
Trends.tags.get(true, limit_param(10))
else
[]
end
end
end
end

This file was deleted.

@@ -6,6 +6,8 @@ module Admin::FilterHelper
CustomEmojiFilter::KEYS,
ReportFilter::KEYS,
TagFilter::KEYS,
PreviewCardProviderFilter::KEYS,
PreviewCardFilter::KEYS,
InstanceFilter::KEYS,
InviteFilter::KEYS,
RelationshipFilter::KEYS,
Loading

0 comments on commit 6e50134

Please sign in to comment.