From e8c826df4ffc2fc5a33c4220b41a60fa526c5ade Mon Sep 17 00:00:00 2001 From: nalabjp Date: Mon, 25 Dec 2017 03:15:27 +0900 Subject: [PATCH 1/6] Support the parameter named `ymd` on Connpass --- lib/statistics/client.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/statistics/client.rb b/lib/statistics/client.rb index 987c2ff60..50785abf9 100644 --- a/lib/statistics/client.rb +++ b/lib/statistics/client.rb @@ -36,13 +36,14 @@ def search(keyword:) @client.get('event/', { keyword: keyword, count: 100 }) end - def fetch_events(series_id:, yyyymm: nil) + def fetch_events(series_id:, yyyymm: nil, yyyymmdd: nil) params = { series_id: series_id, start: 1, count: 100 } params[:ym] = yyyymm if yyyymm + params[:ymd] = yyyymmdd if yyyymmdd events = [] loop do From 5338d086363c8fe7d2b3bfd4a6a51fb0b0f05d15 Mon Sep 17 00:00:00 2001 From: nalabjp Date: Mon, 25 Dec 2017 03:16:33 +0900 Subject: [PATCH 2/6] Support weekly aggregation --- lib/statistics/aggregation.rb | 74 +++++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 21 deletions(-) diff --git a/lib/statistics/aggregation.rb b/lib/statistics/aggregation.rb index 40eaa5da1..a16f249be 100644 --- a/lib/statistics/aggregation.rb +++ b/lib/statistics/aggregation.rb @@ -1,24 +1,42 @@ module Statistics class Aggregation class << self - def run(date:) - cnps_dojos = Dojo.eager_load(:dojo_event_services).where(dojo_event_services: { name: :connpass }).to_a - drkp_dojos = Dojo.eager_load(:dojo_event_services).where(dojo_event_services: { name: :doorkeeper }).to_a - fsbk_dojos = Dojo.eager_load(:dojo_event_services).where(dojo_event_services: { name: :facebook }).to_a + def run(date:, weekly:) + cnps_dojos, drkp_dojos, fsbk_dojos = fetch_dojos - Connpass.run(cnps_dojos, date) - Doorkeeper.run(drkp_dojos, date) - Facebook.run(fsbk_dojos, date) + Connpass.run(cnps_dojos, date, weekly) + Doorkeeper.run(drkp_dojos, date, weekly) + Facebook.run(fsbk_dojos, date, weekly) + end + + def fetch_dojos + [ + Dojo.eager_load(:dojo_event_services).where(dojo_event_services: { name: :connpass }).to_a, + Dojo.eager_load(:dojo_event_services).where(dojo_event_services: { name: :doorkeeper }).to_a, + Dojo.eager_load(:dojo_event_services).where(dojo_event_services: { name: :facebook }).to_a + ] end end class Connpass class << self - def run(dojos, date) + def run(dojos, date, weekly) cnps = Client::Connpass.new - params = { - yyyymm: "#{date.year}#{date.month}" - } + params = if weekly + week_days = loop.with_object([date]) { |_, list| + nd = list.last.next_day + raise StopIteration if nd > date.end_of_week + list << nd + }.map { |date| date.strftime('%Y%m%d') } + + { + yyyymmdd: week_days.join(',') + } + else + { + yyyymm: "#{date.year}#{date.month}" + } + end dojos.each do |dojo| dojo.dojo_event_services.each do |dojo_event_service| @@ -42,12 +60,19 @@ def run(dojos, date) class Doorkeeper class << self - def run(dojos, date) + def run(dojos, date, weekly) drkp = Client::Doorkeeper.new - params = { - since_at: date.beginning_of_month, - until_at: date.end_of_month - } + params = if weekly + { + since_at: date.beginning_of_week, + until_at: date.end_of_week + } + else + { + since_at: date.beginning_of_month, + until_at: date.end_of_month + } + end dojos.each do |dojo| dojo.dojo_event_services.each do |dojo_event_service| @@ -71,12 +96,19 @@ def run(dojos, date) class Facebook class << self - def run(dojos, date) + def run(dojos, date, weekly) fsbk = Client::Facebook.new - params = { - since_at: date.beginning_of_month, - until_at: date.end_of_month - } + params = if weekly + { + since_at: date.beginning_of_week, + until_at: date.end_of_week + } + else + { + since_at: date.beginning_of_month, + until_at: date.end_of_month + } + end dojos.each do |dojo| dojo.dojo_event_services.each do |dojo_event_service| From d7be1d96e5bffc0b06245a264e8d127d3e00e14a Mon Sep 17 00:00:00 2001 From: nalabjp Date: Mon, 25 Dec 2017 03:17:07 +0900 Subject: [PATCH 3/6] Support weekly aggregation on Rake task --- lib/tasks/statistics.rake | 63 +++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 15 deletions(-) diff --git a/lib/tasks/statistics.rake b/lib/tasks/statistics.rake index 91920218a..34d43130c 100644 --- a/lib/tasks/statistics.rake +++ b/lib/tasks/statistics.rake @@ -1,7 +1,7 @@ require_relative '../statistics.rb' namespace :statistics do - desc '月次のイベント履歴を集計します' + desc '年次/月次/週次のイベント履歴を集計します' task :aggregation, [:from, :to] => :environment do |tasks, args| date_from_str = -> (str) { formats = %w(%Y%m %Y/%m %Y-%m) @@ -20,42 +20,75 @@ namespace :statistics do puts `curl --data-urlencode "source=#{msg}" -s #{ENV['IDOBATA_HOOK_URL']} -o /dev/null -w "idobata: %{http_code}"` if ENV.key?('IDOBATA_HOOK_URL') } + ym_format = -> (date) { date.strftime('%Y/%m') } + ymd_format = -> (date) { date.strftime('%Y/%m/%d') } + + montly_aggregation = -> (from, to) { + loop.with_object([from]) { |_, list| + nm = list.last.next_month + raise StopIteration if nm > to + list << nm + }.each { |date| + puts "Aggregate for #{ym_format.call(date)}" + Statistics::Aggregation.run(date: date, weekly: false) + } + } + + weekly_aggregation = -> (from, to) { + loop.with_object([from]) { |_, list| + nw = list.last.next_week + raise StopIteration if nw > to + list << nw + }.each { |date| + puts "Aggregate for #{ymd_format.call(date)}~#{ymd_format.call(date.end_of_week)}" + Statistics::Aggregation.run(date: date, weekly: true) + } + } + + weekly = true + from = if args[:from] if args[:from].length == 4 + weekly = false date_from_str.call(args[:from]).beginning_of_year - else + elsif args[:from].length == 6 + weekly = false date_from_str.call(args[:from]).beginning_of_month + else + date_from_str.call(args[:from]).beginning_of_week end else - Time.current.prev_month.beginning_of_month + Time.current.prev_week.beginning_of_week end to = if args[:to] if args[:to].length == 4 date_from_str.call(args[:to]).end_of_year - else + elsif args[:to].length == 6 date_from_str.call(args[:to]).end_of_month + else + date_from_str.call(args[:to]).end_of_week end else - Time.current.prev_month.end_of_month + Time.current.prev_week.end_of_week end Statistics::Client::Facebook.access_token = Koala::Facebook::OAuth.new(ENV['FACEBOOK_APP_ID'], ENV['FACEBOOK_APP_SECRET']).get_app_access_token EventHistory.where(evented_at: from..to).delete_all + from_str = weekly ? ym_format.call(from) : ymd_format.call(from) + to_str = weekly ? ym_format.call(to) : ymd_format.call(to) + begin - loop.with_object([from]) { |_, list| - nm = list.last.next_month - raise StopIteration if nm > to - list << nm - }.each { |date| - puts "Aggregate for #{date.strftime('%Y/%m')}" - Statistics::Aggregation.run(date: date) - } + if weekly + weekly_aggregation.call(from, to) + else + montly_aggregation.call(from, to) + end - notify_idobata.call("#{from.strftime('%Y/%m')}~#{to.strftime('%Y/%m')}のイベント履歴の集計を行いました") + notify_idobata.call("#{from_str}~#{to_str}のイベント履歴の集計を行いました") rescue => e - notify_idobata.call("#{from.strftime('%Y/%m')}~#{to.strftime('%Y/%m')}のイベント履歴の集計でエラーが発生しました\n#{e.message}") + notify_idobata.call("#{from_str}~#{to_str}のイベント履歴の集計でエラーが発生しました\n#{e.message}") raise e end end From b5f8ca69de8a9bf7d4fcbb114f504315d94f111c Mon Sep 17 00:00:00 2001 From: nalabjp Date: Mon, 25 Dec 2017 03:17:23 +0900 Subject: [PATCH 4/6] Fix spec for montly aggregation --- spec/lib/statistics/aggregation_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/lib/statistics/aggregation_spec.rb b/spec/lib/statistics/aggregation_spec.rb index 92150c43b..eb55477a4 100644 --- a/spec/lib/statistics/aggregation_spec.rb +++ b/spec/lib/statistics/aggregation_spec.rb @@ -24,7 +24,7 @@ DojoEventService.create(dojo_id: d3.id, name: :facebook, group_id: 123451234512345) end - subject { Statistics::Aggregation.run(date: Time.current) } + subject { Statistics::Aggregation.run(date: Time.current, weekly: false) } it do expect{ subject }.to change{EventHistory.count}.from(0).to(3) From 6a8cecba94913fff7473f92e08b5591c1342d706 Mon Sep 17 00:00:00 2001 From: nalabjp Date: Wed, 3 Jan 2018 11:47:14 +0900 Subject: [PATCH 5/6] Add date format --- lib/tasks/statistics.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/statistics.rake b/lib/tasks/statistics.rake index 34d43130c..76fa4ad9d 100644 --- a/lib/tasks/statistics.rake +++ b/lib/tasks/statistics.rake @@ -4,7 +4,7 @@ namespace :statistics do desc '年次/月次/週次のイベント履歴を集計します' task :aggregation, [:from, :to] => :environment do |tasks, args| date_from_str = -> (str) { - formats = %w(%Y%m %Y/%m %Y-%m) + formats = %w(%Y%m%d %Y/%m/%d %Y-%m-%d %Y%m %Y/%m %Y-%m) d = formats.map { |fmt| begin Time.zone.strptime(str, fmt) From d4351c1ce572915466ed922d2b3c6d7ea15bdac5 Mon Sep 17 00:00:00 2001 From: nalabjp Date: Wed, 3 Jan 2018 11:53:04 +0900 Subject: [PATCH 6/6] Tweak --- lib/tasks/statistics.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/statistics.rake b/lib/tasks/statistics.rake index 76fa4ad9d..3b022cf74 100644 --- a/lib/tasks/statistics.rake +++ b/lib/tasks/statistics.rake @@ -1,7 +1,7 @@ require_relative '../statistics.rb' namespace :statistics do - desc '年次/月次/週次のイベント履歴を集計します' + desc '月次/週次のイベント履歴を集計します' task :aggregation, [:from, :to] => :environment do |tasks, args| date_from_str = -> (str) { formats = %w(%Y%m%d %Y/%m/%d %Y-%m-%d %Y%m %Y/%m %Y-%m)