From ed0765dc781f6665c705c8b4470c21e6cac70ddb Mon Sep 17 00:00:00 2001
From: cyrillefr
Date: Thu, 24 Feb 2022 10:54:15 +0100
Subject: [PATCH 1/3] Download Wikidata stats for a campaign
- Add link to wikidata stats download to UI
- Back end code (controller + lib)
- Refactoring CourseWikidataCsvBuilder to use with multiple courses
in order to also use it with a Campaign.
- Small refactoring in worker(rubocop)
- Add locales in en and fr
- Add route
- Add tests: units, controllers
- Add factory for CourseStat
---
.../campaign_stats_download_modal.jsx | 15 ++++++
app/controllers/campaigns_controller.rb | 7 ++-
app/workers/campaign_csv_worker.rb | 30 ++++++-----
config/locales/en.yml | 2 +
config/locales/fr.yml | 2 +
config/routes.rb | 1 +
lib/analytics/campaign_csv_builder.rb | 6 +++
lib/analytics/course_wikidata_csv_builder.rb | 20 +++++---
spec/controllers/campaigns_controller_spec.rb | 9 ++++
spec/factories/course_stats.rb | 10 ++++
.../course_wikidata_csv_builder_spec.rb | 51 +++++++++++++++++++
11 files changed, 133 insertions(+), 20 deletions(-)
create mode 100644 spec/factories/course_stats.rb
create mode 100644 spec/lib/analytics/course_wikidata_csv_builder_spec.rb
diff --git a/app/assets/javascripts/components/campaign/campaign_stats_download_modal.jsx b/app/assets/javascripts/components/campaign/campaign_stats_download_modal.jsx
index 9006169d89..c3af7c5a62 100644
--- a/app/assets/javascripts/components/campaign/campaign_stats_download_modal.jsx
+++ b/app/assets/javascripts/components/campaign/campaign_stats_download_modal.jsx
@@ -11,6 +11,20 @@ const CampaignStatsDownloadModal = ({ match }) => {
const editorsLink = `/campaigns/${campaignSlug}/students.csv`;
const editorsByCourseLink = `/campaigns/${campaignSlug}/students.csv?course=true`;
const instructorsLink = `/campaigns/${campaignSlug}/instructors.csv?course=true`;
+ const wikidataLink = `/campaigns/${campaignSlug}/wikidata.csv`;
+
+ let wikidataCsvLink;
+ if (Features.wikiEd) {
+ wikidataCsvLink = (
+ <>
+
+
+ {I18n.t('campaign.data_wikidata')}
+ {I18n.t('campaign.data_wikidata_info')}
+
+ >
+ );
+ }
if (!show) {
return (
@@ -52,6 +66,7 @@ const CampaignStatsDownloadModal = ({ match }) => {
{I18n.t('campaign.data_instructors')}
{I18n.t('campaign.data_instructors_info')}
+ {wikidataCsvLink}
);
};
diff --git a/app/controllers/campaigns_controller.rb b/app/controllers/campaigns_controller.rb
index 0b73a6abf2..2881d523f2 100644
--- a/app/controllers/campaigns_controller.rb
+++ b/app/controllers/campaigns_controller.rb
@@ -11,7 +11,8 @@ class CampaignsController < ApplicationController
before_action :set_campaign, only: %i[overview programs articles users edit
update destroy add_organizer remove_organizer
remove_course courses ores_plot articles_csv
- revisions_csv alerts students instructors]
+ revisions_csv alerts students instructors
+ wikidata]
before_action :require_create_permissions, only: [:create]
before_action :require_write_permissions, only: %i[update destroy add_organizer
remove_organizer remove_course edit]
@@ -196,6 +197,10 @@ def revisions_csv
csv_of('revisions')
end
+ def wikidata
+ csv_of('wikidata')
+ end
+
private
def csv_of(type)
diff --git a/app/workers/campaign_csv_worker.rb b/app/workers/campaign_csv_worker.rb
index fab4b5c212..0634045dda 100644
--- a/app/workers/campaign_csv_worker.rb
+++ b/app/workers/campaign_csv_worker.rb
@@ -13,23 +13,29 @@ def self.generate_csv(campaign:, filename:, type:, include_course:)
def perform(campaign_id, filename, type, include_course)
campaign = Campaign.find(campaign_id)
builder = CampaignCsvBuilder.new(campaign)
- data = case type
- when 'instructors'
- campaign.users_to_csv(:instructors, course: include_course)
- when 'students'
- campaign.users_to_csv(:students, course: include_course)
- when 'courses'
- builder.courses_to_csv
- when 'articles'
- builder.articles_to_csv
- when 'revisions'
- builder.revisions_to_csv
- end
+ data = to_csv(type, campaign, builder, include_course)
write_csv(filename, data)
CsvCleanupWorker.perform_at(1.week.from_now, filename)
end
+ def to_csv(type, campaign, builder, include_course)
+ case type
+ when 'instructors'
+ campaign.users_to_csv(:instructors, course: include_course)
+ when 'students'
+ campaign.users_to_csv(:students, course: include_course)
+ when 'courses'
+ builder.courses_to_csv
+ when 'articles'
+ builder.articles_to_csv
+ when 'revisions'
+ builder.revisions_to_csv
+ when 'wikidata'
+ builder.wikidata_to_csv
+ end
+ end
+
private
def write_csv(filename, data)
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 95551b3f90..55aa1aec06 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -361,6 +361,8 @@ en:
data_editors_by_course_info: List of enrolled editors and their courses/programs
data_editor_usernames: Editor usernames
data_editor_usernames_info: List of editor usernames, one per line
+ data_wikidata: Wikidata stats
+ data_wikidata_info: Detailed breakdown of Wikidata activity, based on edit summaries
default_course_type: Default Course Type
default_passcode: Default Passcode
default_passcode_explanation: "By default, new programs in this campaign should have:"
diff --git a/config/locales/fr.yml b/config/locales/fr.yml
index 73731ad2aa..6a68b2c230 100644
--- a/config/locales/fr.yml
+++ b/config/locales/fr.yml
@@ -394,6 +394,8 @@ fr:
data_editors_by_course_info: Liste des rédacteurs inscrits avec leurs cours/programmes
data_editor_usernames: Noms d’utilisateur des rédacteurs
data_editor_usernames_info: Liste des noms d'utilisateur des rédacteurs, un par
+ data_wikidata: Statistiques de Wikidata
+ data_wikidata_info: Répartition détaillée de l’activité de Wikidata, d’après les résumés de modification
ligne
default_course_type: Type de cours par défaut
default_passcode: Mot de passe par défaut
diff --git a/config/routes.rb b/config/routes.rb
index 0acd185219..1384f1f816 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -244,6 +244,7 @@
get 'articles_csv'
get 'revisions_csv'
get 'alerts'
+ get 'wikidata'
put 'add_organizer'
put 'remove_organizer'
put 'remove_course'
diff --git a/lib/analytics/campaign_csv_builder.rb b/lib/analytics/campaign_csv_builder.rb
index 92090c2004..f8022de104 100644
--- a/lib/analytics/campaign_csv_builder.rb
+++ b/lib/analytics/campaign_csv_builder.rb
@@ -5,6 +5,7 @@
require_dependency "#{Rails.root}/lib/analytics/course_articles_csv_builder"
require_dependency "#{Rails.root}/lib/analytics/course_revisions_csv_builder"
require_dependency "#{Rails.root}/app/workers/campaign_csv_worker"
+require "#{Rails.root}/lib/analytics/course_wikidata_csv_builder"
class CampaignCsvBuilder
def initialize(campaign)
@@ -44,6 +45,11 @@ def revisions_to_csv
CSV.generate { |csv| csv_data.each { |line| csv << line } }
end
+ def wikidata_to_csv
+ courses = @campaign.courses.joins(:course_stat)
+ CourseWikidataCsvBuilder.new(courses).generate_csv
+ end
+
class AllCourses
def self.courses
Course.all
diff --git a/lib/analytics/course_wikidata_csv_builder.rb b/lib/analytics/course_wikidata_csv_builder.rb
index 115458088a..7329241941 100644
--- a/lib/analytics/course_wikidata_csv_builder.rb
+++ b/lib/analytics/course_wikidata_csv_builder.rb
@@ -4,22 +4,28 @@
class CourseWikidataCsvBuilder
def initialize(course)
- @course = course
+ @courses = course.is_a?(ActiveRecord::Relation) ? course : [course]
end
def generate_csv
csv_data = [CSV_HEADERS]
- stats = if @course.course_stat
- @course.course_stat.stats_hash['www.wikidata.org']
- else
- { 'total revisions' => 0 }
- end
- stats.each do |revision_type, count|
+ aggregate_stats.each do |revision_type, count|
csv_data << [revision_type, count]
end
CSV.generate { |csv| csv_data.each { |line| csv << line } }
end
+ def aggregate_stats
+ @courses.each.with_object({}) do |course, stats_aggrted|
+ stats = if course.course_stat
+ course.course_stat.stats_hash['www.wikidata.org']
+ else
+ { 'total revisions' => 0 }
+ end
+ stats_aggrted.merge!(stats) { |_, old, new| old + new }
+ end
+ end
+
CSV_HEADERS = %w[
revision_type
count
diff --git a/spec/controllers/campaigns_controller_spec.rb b/spec/controllers/campaigns_controller_spec.rb
index c1f18e8a19..fa181765e1 100644
--- a/spec/controllers/campaigns_controller_spec.rb
+++ b/spec/controllers/campaigns_controller_spec.rb
@@ -437,6 +437,15 @@
expect(csv).to include(article.title)
expect(csv).to include('references_added')
end
+
+ it 'returns a csv of wikidata' do
+ expect(CsvCleanupWorker).to receive(:perform_at)
+ get "/campaigns/#{campaign.slug}/wikidata.csv"
+ get "/campaigns/#{campaign.slug}/wikidata.csv"
+ follow_redirect!
+ csv = response.body.force_encoding('utf-8')
+ expect(csv).to include('revision_type,count')
+ end
end
describe '#overview' do
diff --git a/spec/factories/course_stats.rb b/spec/factories/course_stats.rb
new file mode 100644
index 0000000000..ce0e72e4bd
--- /dev/null
+++ b/spec/factories/course_stats.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: course_stats
+
+FactoryBot.define do
+ factory :course_stats, class: 'CourseStat' do
+ nil
+ end
+end
diff --git a/spec/lib/analytics/course_wikidata_csv_builder_spec.rb b/spec/lib/analytics/course_wikidata_csv_builder_spec.rb
new file mode 100644
index 0000000000..47b63ca8d9
--- /dev/null
+++ b/spec/lib/analytics/course_wikidata_csv_builder_spec.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+require "#{Rails.root}/lib/analytics/course_wikidata_csv_builder.rb"
+
+describe CourseWikidataCsvBuilder do
+ let(:course) { create(:course) }
+ let(:another_course) { create(:course, slug: 'myschool/mycourse') }
+ let(:builder) { described_class.new(course) }
+ let(:campaign_builder) { described_class.new(ActiveRecord::Relation.new(Course)) }
+ let(:create_course_stat) do
+ create(:course_stats,
+ stats_hash: { 'www.wikidata.org' => { 'claims created' => 2 } }, course_id: course.id)
+ end
+ let(:create_another_course_stat) do
+ create(:course_stats,
+ stats_hash: { 'www.wikidata.org' => { 'claims created' => 9 } },
+ course_id: another_course.id)
+ end
+
+ context 'when no course_stat' do
+ it 'generates only headers' do
+ expect(builder.generate_csv).to eq "revision_type,count\ntotal revisions,0\n"
+ end
+ end
+
+ context 'when course_stat exists' do
+ before do
+ create_course_stat
+ end
+
+ it 'generates csv data' do
+ expect(builder.generate_csv).to eq "revision_type,count\nclaims created,2\n"
+ end
+ end
+
+ # generate_csv also used in campaign context
+ # see controller campaign and CSV actions
+ # see also campaigns_controller_spec
+ context 'when multiple courses in a campaign' do
+ before do
+ create_course_stat
+ create_another_course_stat
+ end
+
+ # Since 2 + 9 = 11
+ it 'generates csv of aggregated data' do
+ expect(campaign_builder.generate_csv).to eq "revision_type,count\nclaims created,11\n"
+ end
+ end
+end
From 6ea9af719450efdb937b2eedddc51ead288313e3 Mon Sep 17 00:00:00 2001
From: cyrillefr
Date: Fri, 4 Mar 2022 00:45:53 +0100
Subject: [PATCH 2/3] Changes after code review
---
.../campaign_stats_download_modal.jsx | 18 ++-----
.../overview/course_stats_download_modal.jsx | 2 +-
lib/analytics/campaign_csv_builder.rb | 18 ++++++-
lib/analytics/course_wikidata_csv_builder.rb | 52 +++++++++++++------
spec/controllers/campaigns_controller_spec.rb | 2 +-
.../course_wikidata_csv_builder_spec.rb | 34 ++++--------
6 files changed, 68 insertions(+), 58 deletions(-)
diff --git a/app/assets/javascripts/components/campaign/campaign_stats_download_modal.jsx b/app/assets/javascripts/components/campaign/campaign_stats_download_modal.jsx
index c3af7c5a62..a66049b79f 100644
--- a/app/assets/javascripts/components/campaign/campaign_stats_download_modal.jsx
+++ b/app/assets/javascripts/components/campaign/campaign_stats_download_modal.jsx
@@ -13,18 +13,6 @@ const CampaignStatsDownloadModal = ({ match }) => {
const instructorsLink = `/campaigns/${campaignSlug}/instructors.csv?course=true`;
const wikidataLink = `/campaigns/${campaignSlug}/wikidata.csv`;
- let wikidataCsvLink;
- if (Features.wikiEd) {
- wikidataCsvLink = (
- <>
-
-
- {I18n.t('campaign.data_wikidata')}
- {I18n.t('campaign.data_wikidata_info')}
-
- >
- );
- }
if (!show) {
return (
@@ -66,7 +54,11 @@ const CampaignStatsDownloadModal = ({ match }) => {
{I18n.t('campaign.data_instructors')}
{I18n.t('campaign.data_instructors_info')}
- {wikidataCsvLink}
+
+
+ {I18n.t('campaign.data_wikidata')}
+ {I18n.t('campaign.data_wikidata_info')}
+
);
};
diff --git a/app/assets/javascripts/components/overview/course_stats_download_modal.jsx b/app/assets/javascripts/components/overview/course_stats_download_modal.jsx
index 74792a6b5f..adce6661de 100644
--- a/app/assets/javascripts/components/overview/course_stats_download_modal.jsx
+++ b/app/assets/javascripts/components/overview/course_stats_download_modal.jsx
@@ -35,7 +35,7 @@ const CourseStatsDownloadModal = createReactClass({
const wikidataCsvLink = `/course_wikidata_csv?course=${this.props.course.slug}`;
let wikidataLink;
- if (Features.wikiEd && this.props.course.home_wiki.project === 'wikidata') {
+ if (this.props.course.course_stats && this.props.course.home_wiki.project === 'wikidata') {
wikidataLink = (
<>
diff --git a/lib/analytics/campaign_csv_builder.rb b/lib/analytics/campaign_csv_builder.rb
index f8022de104..b2f5f5d8ec 100644
--- a/lib/analytics/campaign_csv_builder.rb
+++ b/lib/analytics/campaign_csv_builder.rb
@@ -46,8 +46,22 @@ def revisions_to_csv
end
def wikidata_to_csv
- courses = @campaign.courses.joins(:course_stat)
- CourseWikidataCsvBuilder.new(courses).generate_csv
+ csv_data = [CourseWikidataCsvBuilder::CSV_HEADERS]
+ courses = @campaign.courses
+ .joins(:course_stat)
+ .where(home_wiki_id: Wiki.find_by(project: 'wikidata').id)
+ courses.find_each do |course|
+ csv_data << CourseWikidataCsvBuilder.new(course).stat_row
+ end
+
+ csv_data << sum_wiki_columns(csv_data) if courses.any?
+
+ CSV.generate { |csv| csv_data.each { |line| csv << line } }
+ end
+
+ def sum_wiki_columns(csv_data)
+ # Skip 1st header row + 1st column course name
+ csv_data[1..].transpose[1..].map(&:sum).unshift('Total')
end
class AllCourses
diff --git a/lib/analytics/course_wikidata_csv_builder.rb b/lib/analytics/course_wikidata_csv_builder.rb
index 7329241941..c4547539a5 100644
--- a/lib/analytics/course_wikidata_csv_builder.rb
+++ b/lib/analytics/course_wikidata_csv_builder.rb
@@ -4,30 +4,50 @@
class CourseWikidataCsvBuilder
def initialize(course)
- @courses = course.is_a?(ActiveRecord::Relation) ? course : [course]
+ @course = course
end
def generate_csv
csv_data = [CSV_HEADERS]
- aggregate_stats.each do |revision_type, count|
- csv_data << [revision_type, count]
- end
+ csv_data << stat_row if @course.course_stat && @course.home_wiki.project == 'wikidata'
+
CSV.generate { |csv| csv_data.each { |line| csv << line } }
end
- def aggregate_stats
- @courses.each.with_object({}) do |course, stats_aggrted|
- stats = if course.course_stat
- course.course_stat.stats_hash['www.wikidata.org']
- else
- { 'total revisions' => 0 }
- end
- stats_aggrted.merge!(stats) { |_, old, new| old + new }
- end
+ def stat_row
+ hash_stats = @course
+ .course_stat.stats_hash['www.wikidata.org']
+ .merge({ 'course name' => @course.title })
+ CSV_HEADERS.map { |elmnt| hash_stats.fetch elmnt, '' }
end
- CSV_HEADERS = %w[
- revision_type
- count
+ CSV_HEADERS = [
+ 'course name',
+ 'claims created',
+ 'claims changed',
+ 'claims removed',
+ 'items created',
+ 'labels added',
+ 'labels changed',
+ 'labels removed',
+ 'descriptions added',
+ 'descriptions changed',
+ 'descriptions removed',
+ 'aliases added',
+ 'aliases changed',
+ 'aliases removed',
+ 'merged from',
+ 'merged to',
+ 'interwiki links added',
+ 'interwiki links removed',
+ 'redirects created',
+ 'reverts performed',
+ 'restorations performed',
+ 'items cleared',
+ 'qualifiers added',
+ 'other updates',
+ 'unknown',
+ 'no data',
+ 'total revisions'
].freeze
end
diff --git a/spec/controllers/campaigns_controller_spec.rb b/spec/controllers/campaigns_controller_spec.rb
index fa181765e1..2c05fb6c72 100644
--- a/spec/controllers/campaigns_controller_spec.rb
+++ b/spec/controllers/campaigns_controller_spec.rb
@@ -444,7 +444,7 @@
get "/campaigns/#{campaign.slug}/wikidata.csv"
follow_redirect!
csv = response.body.force_encoding('utf-8')
- expect(csv).to include('revision_type,count')
+ expect(csv).to include('course name,claims created')
end
end
diff --git a/spec/lib/analytics/course_wikidata_csv_builder_spec.rb b/spec/lib/analytics/course_wikidata_csv_builder_spec.rb
index 47b63ca8d9..9ffb8e6135 100644
--- a/spec/lib/analytics/course_wikidata_csv_builder_spec.rb
+++ b/spec/lib/analytics/course_wikidata_csv_builder_spec.rb
@@ -4,23 +4,20 @@
require "#{Rails.root}/lib/analytics/course_wikidata_csv_builder.rb"
describe CourseWikidataCsvBuilder do
- let(:course) { create(:course) }
- let(:another_course) { create(:course, slug: 'myschool/mycourse') }
+ let(:wikidata) { Wiki.get_or_create(language: nil, project: 'wikidata') }
+ let(:course) { create(:course, home_wiki: wikidata) }
let(:builder) { described_class.new(course) }
- let(:campaign_builder) { described_class.new(ActiveRecord::Relation.new(Course)) }
let(:create_course_stat) do
create(:course_stats,
stats_hash: { 'www.wikidata.org' => { 'claims created' => 2 } }, course_id: course.id)
end
- let(:create_another_course_stat) do
- create(:course_stats,
- stats_hash: { 'www.wikidata.org' => { 'claims created' => 9 } },
- course_id: another_course.id)
- end
+
+ before { stub_wiki_validation }
context 'when no course_stat' do
it 'generates only headers' do
- expect(builder.generate_csv).to eq "revision_type,count\ntotal revisions,0\n"
+ expect(builder.generate_csv).to start_with('course name,')
+ expect(builder.generate_csv.lines.count).to eq(1)
end
end
@@ -30,22 +27,9 @@
end
it 'generates csv data' do
- expect(builder.generate_csv).to eq "revision_type,count\nclaims created,2\n"
- end
- end
-
- # generate_csv also used in campaign context
- # see controller campaign and CSV actions
- # see also campaigns_controller_spec
- context 'when multiple courses in a campaign' do
- before do
- create_course_stat
- create_another_course_stat
- end
-
- # Since 2 + 9 = 11
- it 'generates csv of aggregated data' do
- expect(campaign_builder.generate_csv).to eq "revision_type,count\nclaims created,11\n"
+ expect(builder.generate_csv.lines.first).to start_with('course name,claims created')
+ expect(builder.generate_csv.lines.last).to start_with(course.title + ',2')
+ expect(builder.generate_csv.lines.count).to eq 2
end
end
end
From 8a7bc2d38ecbbcfb5b5e97ee44b99066f85adcd3 Mon Sep 17 00:00:00 2001
From: cyrillefr
Date: Fri, 4 Mar 2022 08:30:27 +0100
Subject: [PATCH 3/3] Fix spec error
---
spec/controllers/campaigns_controller_spec.rb | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/spec/controllers/campaigns_controller_spec.rb b/spec/controllers/campaigns_controller_spec.rb
index 2c05fb6c72..557fb7889a 100644
--- a/spec/controllers/campaigns_controller_spec.rb
+++ b/spec/controllers/campaigns_controller_spec.rb
@@ -400,7 +400,9 @@
end
describe 'CSV actions' do
+ let(:wikidata) { Wiki.get_or_create(language: nil, project: 'wikidata') }
let(:course) { create(:course) }
+ let(:another_course) { create(:course, home_wiki: wikidata, slug: 'campaign/acourse') }
let(:campaign) { create(:campaign) }
let(:article) { create(:article) }
let(:user) { create(:user) }
@@ -408,8 +410,9 @@
let(:request_params) { { slug: campaign.slug, format: :csv } }
before do
+ stub_wiki_validation
login_as(user)
- campaign.courses << course
+ campaign.courses.push course, another_course
create(:courses_user, course: course, user: user)
end