Merge pull request #4197 from Ana06/project_log_rotate

 [api] Move ProjectLogRotate to 1 event per job

.rubocop.yml

@@ -107,11 +107,3 @@ Rails/Output:
     - 'src/api/config/*'
     - 'src/api/db/**/*'
     - 'src/api/lib/**/*'
-
-# Checks for the use of methods which skip validations.
-Rails/SkipsModelValidations:
-  Exclude:
-    - 'src/api/db/migrate/*'
-    - 'src/api/app/jobs/*'
-    - 'src/api/spec/models/bs_request_action_spec.rb'
-    - 'src/api/spec/models/review_spec.rb'

.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2017-12-04 09:56:31 +0000 using RuboCop version 0.51.0.
+# on 2017-12-08 08:39:43 +0000 using RuboCop version 0.51.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -185,7 +185,7 @@ Layout/MultilineHashBraceLayout:
 Layout/MultilineMethodCallBraceLayout:
   Enabled: false
 
-# Offense count: 29
+# Offense count: 28
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle, SupportedStyles, IndentationWidth.
 # SupportedStyles: aligned, indented, indented_relative_to_receiver
@@ -198,7 +198,6 @@ Layout/MultilineMethodCallIndentation:
     - 'src/api/app/jobs/project_create_auto_cleanup_requests.rb'
     - 'src/api/app/mixins/has_relationships.rb'
     - 'src/api/app/models/owner.rb'
-    - 'src/api/spec/controllers/webui/request_controller_spec.rb'
     - 'src/api/spec/features/webui/projects_spec.rb'
     - 'src/api/spec/models/repository_spec.rb'
     - 'src/api/test/unit/issue_test.rb'
@@ -211,7 +210,7 @@ Layout/MultilineMethodCallIndentation:
 Layout/MultilineOperationIndentation:
   Enabled: false
 
-# Offense count: 75
+# Offense count: 76
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle, SupportedStyles, SupportedStylesForEmptyBraces.
 # SupportedStyles: space, no_space
@@ -228,7 +227,7 @@ Layout/SpaceInLambdaLiteral:
     - 'src/api/app/models/package_issue.rb'
     - 'src/api/config/routes.rb'
 
-# Offense count: 125
+# Offense count: 140
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle, SupportedStyles, EnforcedStyleForEmptyBraces, SupportedStylesForEmptyBraces, SpaceBeforeBlockParameters.
 # SupportedStyles: space, no_space
@@ -385,7 +384,7 @@ Lint/UriEscapeUnescape:
 Metrics/AbcSize:
   Max: 248
 
-# Offense count: 411
+# Offense count: 413
 # Configuration parameters: CountComments, ExcludedMethods.
 Metrics/BlockLength:
   Max: 988
@@ -400,11 +399,11 @@ Metrics/BlockNesting:
 Metrics/ClassLength:
   Max: 1157
 
-# Offense count: 244
+# Offense count: 243
 Metrics/CyclomaticComplexity:
   Max: 55
 
-# Offense count: 958
+# Offense count: 957
 # Configuration parameters: CountComments.
 Metrics/MethodLength:
   Max: 264
@@ -660,6 +659,19 @@ Rails/ReversibleMigration:
     - 'src/api/db/migrate/20160824132643_fix_bs_request_counter.rb'
     - 'src/api/db/migrate/20170103132257_change_project_package_name_to_string.rb'
 
+# Offense count: 7
+# Configuration parameters: Blacklist.
+# Blacklist: decrement!, decrement_counter, increment!, increment_counter, toggle!, touch, update_all, update_attribute, update_column, update_columns, update_counters
+Rails/SkipsModelValidations:
+  Exclude:
+    - 'src/api/db/migrate/20151030130011_mark_events.rb'
+    - 'src/api/db/migrate/20161128115942_add_when_attribute_to_bs_request.rb'
+    - 'src/api/db/migrate/20170621100321_add_channel_to_event_subscriptions.rb'
+    - 'src/api/db/migrate/20170630144825_convert_tokens_to_service_tokens.rb'
+    - 'src/api/lib/tasks/project_log_rotate_manually.rake'
+    - 'src/api/spec/models/bs_request_action_spec.rb'
+    - 'src/api/spec/models/review_spec.rb'
+
 # Offense count: 98
 # Configuration parameters: EnforcedStyle, SupportedStyles.
 # SupportedStyles: strict, flexible
@@ -691,7 +703,7 @@ Style/BlockDelimiters:
 Style/BracesAroundHashParameters:
   Enabled: false
 
-# Offense count: 83
+# Offense count: 67
 # Configuration parameters: EnforcedStyle, SupportedStyles.
 # SupportedStyles: nested, compact
 Style/ClassAndModuleChildren:
@@ -767,7 +779,7 @@ Style/Dir:
   Exclude:
     - 'src/api/script/start_test_backend'
 
-# Offense count: 634
+# Offense count: 631
 Style/Documentation:
   Enabled: false
 
@@ -1054,7 +1066,7 @@ Style/StderrPuts:
   Exclude:
     - 'src/api/test/functional/webui/spider_test.rb'
 
-# Offense count: 9303
+# Offense count: 9358
 # Cop supports --auto-correct.
 # Configuration parameters: EnforcedStyle, SupportedStyles, ConsistentQuotesInMultiline.
 # SupportedStyles: single_quotes, double_quotes

src/api/app/jobs/create_project_log_entry_job.rb

@@ -0,0 +1,9 @@
+class CreateProjectLogEntryJob < ApplicationJob
+  queue_as :project_log_rotate
+
+  def perform(event_id)
+    event = Event::Base.find(event_id)
+    entry = ProjectLogEntry.create_from(event)
+    event.update_attributes(project_logged: true) if entry.persisted?
+  end
+end

src/api/app/models/event.rb

@@ -1,2 +1,20 @@
 module Event
+  PROJECT_CLASSES = ["Event::CommentForProject",
+                     "Event::CreateProject",
+                     "Event::DeleteProject",
+                     "Event::UndeleteProject",
+                     "Event::UpdateProjectConfig",
+                     "Event::UpdateProject"].freeze
+  PACKAGE_CLASSES = ["Event::BranchCommand",
+                     "Event::Build",
+                     "Event::CommentForPackage",
+                     "Event::Commit",
+                     "Event::CreatePackage",
+                     "Event::DeletePackage",
+                     "Event::ServiceFail",
+                     "Event::ServiceSuccess",
+                     "Event::UndeletePackage",
+                     "Event::UpdatePackage",
+                     "Event::Upload",
+                     "Event::VersionChange"].freeze
 end

src/api/app/models/event/base.rb

@@ -8,6 +8,7 @@ class Base < ApplicationRecord
     self.table_name = 'events'
 
     before_save :shorten_payload_if_necessary
+    after_create :create_project_log_entry_job, if: -> { (PROJECT_CLASSES | PACKAGE_CLASSES).include?(self.class.name) }
 
     EXPLANATION_FOR_NOTIFICATIONS =  {
       'Event::BuildFail'          => 'Receive notifications of build failures for packages for which you are...',
@@ -158,6 +159,10 @@ def payload
       @payload ||= Yajl::Parser.parse(read_attribute(:payload))
     end
 
+    def create_project_log_entry_job
+      CreateProjectLogEntryJob.perform_later(id)
+    end
+
     after_create :perform_create_jobs
 
     def perform_create_jobs

src/api/config/clock.rb

@@ -20,10 +20,6 @@ module Clockwork
     SendEventEmailsJob.perform_later
   end
 
-  every(10.minutes, 'project log rotates') do
-    ProjectLogRotateJob.perform_later
-  end
-
   every(49.minutes, 'rescale history') do
     StatusHistoryRescalerJob.perform_later
   end

src/api/lib/tasks/project_log_rotate_manually.rake

@@ -0,0 +1,20 @@
+desc 'Run project log rotate job manually'
+task :project_log_rotate_manually do
+  event_types = Event::PROJECT_CLASSES | Event::PACKAGE_CLASSES
+  oldest_date = 10.days.ago
+
+  # First, skip old events and mark them all as "logged" (even those that
+  # don't belong to the event_classes)
+  Event::Base.where(project_logged: false).where(["created_at < ?", oldest_date]).update_all(project_logged: true)
+
+  # Create log entries based on the events in event_classes
+  Event::Base.where(project_logged: false, eventtype: event_types).find_in_batches(batch_size: 10000) do |events_batch|
+    events_batch.each do |event|
+      entry = ProjectLogEntry.create_from(event)
+      event.update_attributes(project_logged: true) if entry.persisted?
+    end
+  end
+
+  # Clean up old entries
+  ProjectLogEntry.clean_older_than oldest_date
+end

src/api/script/delayed_job_stats.rb

@@ -1,23 +1,5 @@
 puts "-----------EVENT-STATS------------"
-# Package and Project events
-event_types = ["Event::BranchCommand",
-               "Event::Build",
-               "Event::CommentForPackage",
-               "Event::Commit",
-               "Event::CreatePackage",
-               "Event::DeletePackage",
-               "Event::ServiceFail",
-               "Event::ServiceSuccess",
-               "Event::UndeletePackage",
-               "Event::UpdatePackage",
-               "Event::Upload",
-               "Event::VersionChange",
-               "Event::CommentForProject",
-               "Event::CreateProject",
-               "Event::DeleteProject",
-               "Event::UndeleteProject",
-               "Event::UpdateProjectConfig",
-               "Event::UpdateProject"]
+Event::PROJECT_CLASSES | Event::PACKAGE_CLASSES
 oldest_date = 10.days.ago
 project_logged = Event::Base.where(project_logged: false, eventtype: event_types).where(["created_at > ?", oldest_date]).count
 mails_sent = Event::Base.where(mails_sent: false).count

src/api/spec/jobs/project_log_rotate_job_spec.rb

@@ -1,8 +1,6 @@
 require 'rails_helper'
 
-RSpec.describe ProjectLogRotateJob, type: :job do
-  include ActiveJob::TestHelper
-
+RSpec.describe CreateProjectLogEntryJob, type: :job do
   describe '#perform' do
     let!(:user1) { create(:confirmed_user) }
     let!(:user2) { create(:confirmed_user) }
@@ -19,8 +17,6 @@
       end
     end
 
-    subject! { ProjectLogRotateJob.new.perform }
-
     it 'creates a ProjectLogEntry' do
       expect(ProjectLogEntry.count).to eq(10)
     end