From ed0765dc781f6665c705c8b4470c21e6cac70ddb Mon Sep 17 00:00:00 2001 From: cyrillefr Date: Thu, 24 Feb 2022 10:54:15 +0100 Subject: [PATCH 1/3] Download Wikidata stats for a campaign - Add link to wikidata stats download to UI - Back end code (controller + lib) - Refactoring CourseWikidataCsvBuilder to use with multiple courses in order to also use it with a Campaign. - Small refactoring in worker(rubocop) - Add locales in en and fr - Add route - Add tests: units, controllers - Add factory for CourseStat --- .../campaign_stats_download_modal.jsx | 15 ++++++ app/controllers/campaigns_controller.rb | 7 ++- app/workers/campaign_csv_worker.rb | 30 ++++++----- config/locales/en.yml | 2 + config/locales/fr.yml | 2 + config/routes.rb | 1 + lib/analytics/campaign_csv_builder.rb | 6 +++ lib/analytics/course_wikidata_csv_builder.rb | 20 +++++--- spec/controllers/campaigns_controller_spec.rb | 9 ++++ spec/factories/course_stats.rb | 10 ++++ .../course_wikidata_csv_builder_spec.rb | 51 +++++++++++++++++++ 11 files changed, 133 insertions(+), 20 deletions(-) create mode 100644 spec/factories/course_stats.rb create mode 100644 spec/lib/analytics/course_wikidata_csv_builder_spec.rb diff --git a/app/assets/javascripts/components/campaign/campaign_stats_download_modal.jsx b/app/assets/javascripts/components/campaign/campaign_stats_download_modal.jsx index 9006169d89..c3af7c5a62 100644 --- a/app/assets/javascripts/components/campaign/campaign_stats_download_modal.jsx +++ b/app/assets/javascripts/components/campaign/campaign_stats_download_modal.jsx @@ -11,6 +11,20 @@ const CampaignStatsDownloadModal = ({ match }) => { const editorsLink = `/campaigns/${campaignSlug}/students.csv`; const editorsByCourseLink = `/campaigns/${campaignSlug}/students.csv?course=true`; const instructorsLink = `/campaigns/${campaignSlug}/instructors.csv?course=true`; + const wikidataLink = `/campaigns/${campaignSlug}/wikidata.csv`; + + let wikidataCsvLink; + if (Features.wikiEd) { + wikidataCsvLink = ( + <> +
+

+ {I18n.t('campaign.data_wikidata')} + {I18n.t('campaign.data_wikidata_info')} +

+ + ); + } if (!show) { return ( @@ -52,6 +66,7 @@ const CampaignStatsDownloadModal = ({ match }) => { {I18n.t('campaign.data_instructors')} {I18n.t('campaign.data_instructors_info')}

+ {wikidataCsvLink} ); }; diff --git a/app/controllers/campaigns_controller.rb b/app/controllers/campaigns_controller.rb index 0b73a6abf2..2881d523f2 100644 --- a/app/controllers/campaigns_controller.rb +++ b/app/controllers/campaigns_controller.rb @@ -11,7 +11,8 @@ class CampaignsController < ApplicationController before_action :set_campaign, only: %i[overview programs articles users edit update destroy add_organizer remove_organizer remove_course courses ores_plot articles_csv - revisions_csv alerts students instructors] + revisions_csv alerts students instructors + wikidata] before_action :require_create_permissions, only: [:create] before_action :require_write_permissions, only: %i[update destroy add_organizer remove_organizer remove_course edit] @@ -196,6 +197,10 @@ def revisions_csv csv_of('revisions') end + def wikidata + csv_of('wikidata') + end + private def csv_of(type) diff --git a/app/workers/campaign_csv_worker.rb b/app/workers/campaign_csv_worker.rb index fab4b5c212..0634045dda 100644 --- a/app/workers/campaign_csv_worker.rb +++ b/app/workers/campaign_csv_worker.rb @@ -13,23 +13,29 @@ def self.generate_csv(campaign:, filename:, type:, include_course:) def perform(campaign_id, filename, type, include_course) campaign = Campaign.find(campaign_id) builder = CampaignCsvBuilder.new(campaign) - data = case type - when 'instructors' - campaign.users_to_csv(:instructors, course: include_course) - when 'students' - campaign.users_to_csv(:students, course: include_course) - when 'courses' - builder.courses_to_csv - when 'articles' - builder.articles_to_csv - when 'revisions' - builder.revisions_to_csv - end + data = to_csv(type, campaign, builder, include_course) write_csv(filename, data) CsvCleanupWorker.perform_at(1.week.from_now, filename) end + def to_csv(type, campaign, builder, include_course) + case type + when 'instructors' + campaign.users_to_csv(:instructors, course: include_course) + when 'students' + campaign.users_to_csv(:students, course: include_course) + when 'courses' + builder.courses_to_csv + when 'articles' + builder.articles_to_csv + when 'revisions' + builder.revisions_to_csv + when 'wikidata' + builder.wikidata_to_csv + end + end + private def write_csv(filename, data) diff --git a/config/locales/en.yml b/config/locales/en.yml index 95551b3f90..55aa1aec06 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -361,6 +361,8 @@ en: data_editors_by_course_info: List of enrolled editors and their courses/programs data_editor_usernames: Editor usernames data_editor_usernames_info: List of editor usernames, one per line + data_wikidata: Wikidata stats + data_wikidata_info: Detailed breakdown of Wikidata activity, based on edit summaries default_course_type: Default Course Type default_passcode: Default Passcode default_passcode_explanation: "By default, new programs in this campaign should have:" diff --git a/config/locales/fr.yml b/config/locales/fr.yml index 73731ad2aa..6a68b2c230 100644 --- a/config/locales/fr.yml +++ b/config/locales/fr.yml @@ -394,6 +394,8 @@ fr: data_editors_by_course_info: Liste des rédacteurs inscrits avec leurs cours/programmes data_editor_usernames: Noms d’utilisateur des rédacteurs data_editor_usernames_info: Liste des noms d'utilisateur des rédacteurs, un par + data_wikidata: Statistiques de Wikidata + data_wikidata_info: Répartition détaillée de l’activité de Wikidata, d’après les résumés de modification ligne default_course_type: Type de cours par défaut default_passcode: Mot de passe par défaut diff --git a/config/routes.rb b/config/routes.rb index 0acd185219..1384f1f816 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -244,6 +244,7 @@ get 'articles_csv' get 'revisions_csv' get 'alerts' + get 'wikidata' put 'add_organizer' put 'remove_organizer' put 'remove_course' diff --git a/lib/analytics/campaign_csv_builder.rb b/lib/analytics/campaign_csv_builder.rb index 92090c2004..f8022de104 100644 --- a/lib/analytics/campaign_csv_builder.rb +++ b/lib/analytics/campaign_csv_builder.rb @@ -5,6 +5,7 @@ require_dependency "#{Rails.root}/lib/analytics/course_articles_csv_builder" require_dependency "#{Rails.root}/lib/analytics/course_revisions_csv_builder" require_dependency "#{Rails.root}/app/workers/campaign_csv_worker" +require "#{Rails.root}/lib/analytics/course_wikidata_csv_builder" class CampaignCsvBuilder def initialize(campaign) @@ -44,6 +45,11 @@ def revisions_to_csv CSV.generate { |csv| csv_data.each { |line| csv << line } } end + def wikidata_to_csv + courses = @campaign.courses.joins(:course_stat) + CourseWikidataCsvBuilder.new(courses).generate_csv + end + class AllCourses def self.courses Course.all diff --git a/lib/analytics/course_wikidata_csv_builder.rb b/lib/analytics/course_wikidata_csv_builder.rb index 115458088a..7329241941 100644 --- a/lib/analytics/course_wikidata_csv_builder.rb +++ b/lib/analytics/course_wikidata_csv_builder.rb @@ -4,22 +4,28 @@ class CourseWikidataCsvBuilder def initialize(course) - @course = course + @courses = course.is_a?(ActiveRecord::Relation) ? course : [course] end def generate_csv csv_data = [CSV_HEADERS] - stats = if @course.course_stat - @course.course_stat.stats_hash['www.wikidata.org'] - else - { 'total revisions' => 0 } - end - stats.each do |revision_type, count| + aggregate_stats.each do |revision_type, count| csv_data << [revision_type, count] end CSV.generate { |csv| csv_data.each { |line| csv << line } } end + def aggregate_stats + @courses.each.with_object({}) do |course, stats_aggrted| + stats = if course.course_stat + course.course_stat.stats_hash['www.wikidata.org'] + else + { 'total revisions' => 0 } + end + stats_aggrted.merge!(stats) { |_, old, new| old + new } + end + end + CSV_HEADERS = %w[ revision_type count diff --git a/spec/controllers/campaigns_controller_spec.rb b/spec/controllers/campaigns_controller_spec.rb index c1f18e8a19..fa181765e1 100644 --- a/spec/controllers/campaigns_controller_spec.rb +++ b/spec/controllers/campaigns_controller_spec.rb @@ -437,6 +437,15 @@ expect(csv).to include(article.title) expect(csv).to include('references_added') end + + it 'returns a csv of wikidata' do + expect(CsvCleanupWorker).to receive(:perform_at) + get "/campaigns/#{campaign.slug}/wikidata.csv" + get "/campaigns/#{campaign.slug}/wikidata.csv" + follow_redirect! + csv = response.body.force_encoding('utf-8') + expect(csv).to include('revision_type,count') + end end describe '#overview' do diff --git a/spec/factories/course_stats.rb b/spec/factories/course_stats.rb new file mode 100644 index 0000000000..ce0e72e4bd --- /dev/null +++ b/spec/factories/course_stats.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: course_stats + +FactoryBot.define do + factory :course_stats, class: 'CourseStat' do + nil + end +end diff --git a/spec/lib/analytics/course_wikidata_csv_builder_spec.rb b/spec/lib/analytics/course_wikidata_csv_builder_spec.rb new file mode 100644 index 0000000000..47b63ca8d9 --- /dev/null +++ b/spec/lib/analytics/course_wikidata_csv_builder_spec.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +require 'rails_helper' +require "#{Rails.root}/lib/analytics/course_wikidata_csv_builder.rb" + +describe CourseWikidataCsvBuilder do + let(:course) { create(:course) } + let(:another_course) { create(:course, slug: 'myschool/mycourse') } + let(:builder) { described_class.new(course) } + let(:campaign_builder) { described_class.new(ActiveRecord::Relation.new(Course)) } + let(:create_course_stat) do + create(:course_stats, + stats_hash: { 'www.wikidata.org' => { 'claims created' => 2 } }, course_id: course.id) + end + let(:create_another_course_stat) do + create(:course_stats, + stats_hash: { 'www.wikidata.org' => { 'claims created' => 9 } }, + course_id: another_course.id) + end + + context 'when no course_stat' do + it 'generates only headers' do + expect(builder.generate_csv).to eq "revision_type,count\ntotal revisions,0\n" + end + end + + context 'when course_stat exists' do + before do + create_course_stat + end + + it 'generates csv data' do + expect(builder.generate_csv).to eq "revision_type,count\nclaims created,2\n" + end + end + + # generate_csv also used in campaign context + # see controller campaign and CSV actions + # see also campaigns_controller_spec + context 'when multiple courses in a campaign' do + before do + create_course_stat + create_another_course_stat + end + + # Since 2 + 9 = 11 + it 'generates csv of aggregated data' do + expect(campaign_builder.generate_csv).to eq "revision_type,count\nclaims created,11\n" + end + end +end From 6ea9af719450efdb937b2eedddc51ead288313e3 Mon Sep 17 00:00:00 2001 From: cyrillefr Date: Fri, 4 Mar 2022 00:45:53 +0100 Subject: [PATCH 2/3] Changes after code review --- .../campaign_stats_download_modal.jsx | 18 ++----- .../overview/course_stats_download_modal.jsx | 2 +- lib/analytics/campaign_csv_builder.rb | 18 ++++++- lib/analytics/course_wikidata_csv_builder.rb | 52 +++++++++++++------ spec/controllers/campaigns_controller_spec.rb | 2 +- .../course_wikidata_csv_builder_spec.rb | 34 ++++-------- 6 files changed, 68 insertions(+), 58 deletions(-) diff --git a/app/assets/javascripts/components/campaign/campaign_stats_download_modal.jsx b/app/assets/javascripts/components/campaign/campaign_stats_download_modal.jsx index c3af7c5a62..a66049b79f 100644 --- a/app/assets/javascripts/components/campaign/campaign_stats_download_modal.jsx +++ b/app/assets/javascripts/components/campaign/campaign_stats_download_modal.jsx @@ -13,18 +13,6 @@ const CampaignStatsDownloadModal = ({ match }) => { const instructorsLink = `/campaigns/${campaignSlug}/instructors.csv?course=true`; const wikidataLink = `/campaigns/${campaignSlug}/wikidata.csv`; - let wikidataCsvLink; - if (Features.wikiEd) { - wikidataCsvLink = ( - <> -
-

- {I18n.t('campaign.data_wikidata')} - {I18n.t('campaign.data_wikidata_info')} -

- - ); - } if (!show) { return ( @@ -66,7 +54,11 @@ const CampaignStatsDownloadModal = ({ match }) => { {I18n.t('campaign.data_instructors')} {I18n.t('campaign.data_instructors_info')}

- {wikidataCsvLink} +
+

+ {I18n.t('campaign.data_wikidata')} + {I18n.t('campaign.data_wikidata_info')} +

); }; diff --git a/app/assets/javascripts/components/overview/course_stats_download_modal.jsx b/app/assets/javascripts/components/overview/course_stats_download_modal.jsx index 74792a6b5f..adce6661de 100644 --- a/app/assets/javascripts/components/overview/course_stats_download_modal.jsx +++ b/app/assets/javascripts/components/overview/course_stats_download_modal.jsx @@ -35,7 +35,7 @@ const CourseStatsDownloadModal = createReactClass({ const wikidataCsvLink = `/course_wikidata_csv?course=${this.props.course.slug}`; let wikidataLink; - if (Features.wikiEd && this.props.course.home_wiki.project === 'wikidata') { + if (this.props.course.course_stats && this.props.course.home_wiki.project === 'wikidata') { wikidataLink = ( <>
diff --git a/lib/analytics/campaign_csv_builder.rb b/lib/analytics/campaign_csv_builder.rb index f8022de104..b2f5f5d8ec 100644 --- a/lib/analytics/campaign_csv_builder.rb +++ b/lib/analytics/campaign_csv_builder.rb @@ -46,8 +46,22 @@ def revisions_to_csv end def wikidata_to_csv - courses = @campaign.courses.joins(:course_stat) - CourseWikidataCsvBuilder.new(courses).generate_csv + csv_data = [CourseWikidataCsvBuilder::CSV_HEADERS] + courses = @campaign.courses + .joins(:course_stat) + .where(home_wiki_id: Wiki.find_by(project: 'wikidata').id) + courses.find_each do |course| + csv_data << CourseWikidataCsvBuilder.new(course).stat_row + end + + csv_data << sum_wiki_columns(csv_data) if courses.any? + + CSV.generate { |csv| csv_data.each { |line| csv << line } } + end + + def sum_wiki_columns(csv_data) + # Skip 1st header row + 1st column course name + csv_data[1..].transpose[1..].map(&:sum).unshift('Total') end class AllCourses diff --git a/lib/analytics/course_wikidata_csv_builder.rb b/lib/analytics/course_wikidata_csv_builder.rb index 7329241941..c4547539a5 100644 --- a/lib/analytics/course_wikidata_csv_builder.rb +++ b/lib/analytics/course_wikidata_csv_builder.rb @@ -4,30 +4,50 @@ class CourseWikidataCsvBuilder def initialize(course) - @courses = course.is_a?(ActiveRecord::Relation) ? course : [course] + @course = course end def generate_csv csv_data = [CSV_HEADERS] - aggregate_stats.each do |revision_type, count| - csv_data << [revision_type, count] - end + csv_data << stat_row if @course.course_stat && @course.home_wiki.project == 'wikidata' + CSV.generate { |csv| csv_data.each { |line| csv << line } } end - def aggregate_stats - @courses.each.with_object({}) do |course, stats_aggrted| - stats = if course.course_stat - course.course_stat.stats_hash['www.wikidata.org'] - else - { 'total revisions' => 0 } - end - stats_aggrted.merge!(stats) { |_, old, new| old + new } - end + def stat_row + hash_stats = @course + .course_stat.stats_hash['www.wikidata.org'] + .merge({ 'course name' => @course.title }) + CSV_HEADERS.map { |elmnt| hash_stats.fetch elmnt, '' } end - CSV_HEADERS = %w[ - revision_type - count + CSV_HEADERS = [ + 'course name', + 'claims created', + 'claims changed', + 'claims removed', + 'items created', + 'labels added', + 'labels changed', + 'labels removed', + 'descriptions added', + 'descriptions changed', + 'descriptions removed', + 'aliases added', + 'aliases changed', + 'aliases removed', + 'merged from', + 'merged to', + 'interwiki links added', + 'interwiki links removed', + 'redirects created', + 'reverts performed', + 'restorations performed', + 'items cleared', + 'qualifiers added', + 'other updates', + 'unknown', + 'no data', + 'total revisions' ].freeze end diff --git a/spec/controllers/campaigns_controller_spec.rb b/spec/controllers/campaigns_controller_spec.rb index fa181765e1..2c05fb6c72 100644 --- a/spec/controllers/campaigns_controller_spec.rb +++ b/spec/controllers/campaigns_controller_spec.rb @@ -444,7 +444,7 @@ get "/campaigns/#{campaign.slug}/wikidata.csv" follow_redirect! csv = response.body.force_encoding('utf-8') - expect(csv).to include('revision_type,count') + expect(csv).to include('course name,claims created') end end diff --git a/spec/lib/analytics/course_wikidata_csv_builder_spec.rb b/spec/lib/analytics/course_wikidata_csv_builder_spec.rb index 47b63ca8d9..9ffb8e6135 100644 --- a/spec/lib/analytics/course_wikidata_csv_builder_spec.rb +++ b/spec/lib/analytics/course_wikidata_csv_builder_spec.rb @@ -4,23 +4,20 @@ require "#{Rails.root}/lib/analytics/course_wikidata_csv_builder.rb" describe CourseWikidataCsvBuilder do - let(:course) { create(:course) } - let(:another_course) { create(:course, slug: 'myschool/mycourse') } + let(:wikidata) { Wiki.get_or_create(language: nil, project: 'wikidata') } + let(:course) { create(:course, home_wiki: wikidata) } let(:builder) { described_class.new(course) } - let(:campaign_builder) { described_class.new(ActiveRecord::Relation.new(Course)) } let(:create_course_stat) do create(:course_stats, stats_hash: { 'www.wikidata.org' => { 'claims created' => 2 } }, course_id: course.id) end - let(:create_another_course_stat) do - create(:course_stats, - stats_hash: { 'www.wikidata.org' => { 'claims created' => 9 } }, - course_id: another_course.id) - end + + before { stub_wiki_validation } context 'when no course_stat' do it 'generates only headers' do - expect(builder.generate_csv).to eq "revision_type,count\ntotal revisions,0\n" + expect(builder.generate_csv).to start_with('course name,') + expect(builder.generate_csv.lines.count).to eq(1) end end @@ -30,22 +27,9 @@ end it 'generates csv data' do - expect(builder.generate_csv).to eq "revision_type,count\nclaims created,2\n" - end - end - - # generate_csv also used in campaign context - # see controller campaign and CSV actions - # see also campaigns_controller_spec - context 'when multiple courses in a campaign' do - before do - create_course_stat - create_another_course_stat - end - - # Since 2 + 9 = 11 - it 'generates csv of aggregated data' do - expect(campaign_builder.generate_csv).to eq "revision_type,count\nclaims created,11\n" + expect(builder.generate_csv.lines.first).to start_with('course name,claims created') + expect(builder.generate_csv.lines.last).to start_with(course.title + ',2') + expect(builder.generate_csv.lines.count).to eq 2 end end end From 8a7bc2d38ecbbcfb5b5e97ee44b99066f85adcd3 Mon Sep 17 00:00:00 2001 From: cyrillefr Date: Fri, 4 Mar 2022 08:30:27 +0100 Subject: [PATCH 3/3] Fix spec error --- spec/controllers/campaigns_controller_spec.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/spec/controllers/campaigns_controller_spec.rb b/spec/controllers/campaigns_controller_spec.rb index 2c05fb6c72..557fb7889a 100644 --- a/spec/controllers/campaigns_controller_spec.rb +++ b/spec/controllers/campaigns_controller_spec.rb @@ -400,7 +400,9 @@ end describe 'CSV actions' do + let(:wikidata) { Wiki.get_or_create(language: nil, project: 'wikidata') } let(:course) { create(:course) } + let(:another_course) { create(:course, home_wiki: wikidata, slug: 'campaign/acourse') } let(:campaign) { create(:campaign) } let(:article) { create(:article) } let(:user) { create(:user) } @@ -408,8 +410,9 @@ let(:request_params) { { slug: campaign.slug, format: :csv } } before do + stub_wiki_validation login_as(user) - campaign.courses << course + campaign.courses.push course, another_course create(:courses_user, course: course, user: user) end