Merge branch 'master' into serializers

* master: (33 commits) Link identities to repository users Store repository_user_id when downloading repo owners Add repository_user_id field and index to Repositories Add uid index on identities table Update dependencies, fixes #1346 Ben-Laurie (#1349) Split status check queries into groups of 500 to avoid timeouts Add repository_organisation_id index on repositories Add pghero dashboard for visualizing db stats Updated json gem enable pg_stat_statements extension Add user_id index on identities table Update readme indexes Remove readmes from admin counts Show readme counts on admin overview Add host_type scopes to org uniqueness validations Handle Bitbucket Unauthorized exceptions properly Update gitlab Update dependencies, fixes #1343 user routes should specify host_type from record ...
librariesio · Apr 13, 2017 · 31e4090 · 31e4090
2 parents 2cd2f45 + 1cbd5aa
commit 31e4090
Show file tree

Hide file tree

Showing 63 changed files with 322 additions and 178 deletions.
diff --git a/Gemfile b/Gemfile
@@ -13,8 +13,8 @@ gem 'octokit'
 gem 'bootstrap-sass'
 gem 'will_paginate-bootstrap'
 gem 'elasticsearch', '~> 2'
-gem 'elasticsearch-model', git: 'https://github.com/elasticsearch/elasticsearch-rails'
-gem 'elasticsearch-rails', git: 'https://github.com/elasticsearch/elasticsearch-rails'
+gem 'elasticsearch-model'
+gem 'elasticsearch-rails'
 gem 'nokogiri'
 gem 'typhoeus'
 gem 'redis'
@@ -32,7 +32,7 @@ gem 'dalli'
 gem 'counter_culture'
 gem 'simple_form'
 gem 'sidekiq'
-gem 'sidekiq-unique-jobs', git: 'https://github.com/mhenrixon/sidekiq-unique-jobs'
+gem 'sidekiq-unique-jobs'
 gem 'font-awesome-rails'
 gem 'mime-types', require: 'mime/types/columnar'
 gem 'fast_blank'
@@ -82,6 +82,8 @@ gem 'github-linguist'
 gem 'appsignal'
 gem 'rack-cors', :require => 'rack/cors'
 gem 'active_model_serializers'
+gem 'pghero'
+gem 'pg_query'
 
 group :development do
   gem 'spring'

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,13 +1,3 @@
-GIT
-  remote: https://github.com/elasticsearch/elasticsearch-rails
-  revision: 109c155df6c8b8989aa935bc1fe163a1452a2b4e
-  specs:
-    elasticsearch-model (0.1.9)
-      activesupport (> 3)
-      elasticsearch (> 1)
-      hashie
-    elasticsearch-rails (0.1.9)
-
 GIT
   remote: https://github.com/gocardless/postgres_ext
   revision: 17e4358fe013c05fed530af58cb4bd96225d79fb
@@ -32,11 +22,10 @@ GIT
 
 GIT
   remote: https://github.com/librariesio/gitlab
-  revision: a060f1939bb776d96e9e9c76a2cf5543f22755f0
+  revision: 69b59cd43b29c316b4f380610bb961ec398b22ee
   specs:
-    gitlab (3.7.0)
+    gitlab (4.0.0)
       httparty
-      terminal-table
 
 GIT
   remote: https://github.com/librariesio/payola
@@ -57,14 +46,6 @@ GIT
       actionpack (>= 3.1)
       activesupport (>= 3.1)
 
-GIT
-  remote: https://github.com/mhenrixon/sidekiq-unique-jobs
-  revision: abb71d5808592376f60654409dde240febecb73a
-  specs:
-    sidekiq-unique-jobs (5.0.0.beta)
-      sidekiq (>= 4.0)
-      thor
-
 GEM
   remote: https://rubygems.org/
   specs:
@@ -128,7 +109,7 @@ GEM
     asciidoctor (1.5.5)
     autoprefixer-rails (6.7.7.1)
       execjs
-    bibliothecary (5.3.1)
+    bibliothecary (5.3.2)
       deb_control
       librariesio-gem-parser
       ox
@@ -141,7 +122,7 @@ GEM
       autoprefixer-rails (>= 5.2.1)
       sass (>= 3.3.4)
     brakeman (3.6.1)
-    bugsnag (5.2.0)
+    bugsnag (5.3.0)
     builder (3.2.3)
     capistrano (3.8.0)
       airbrussh (>= 1.0.0)
@@ -199,6 +180,11 @@ GEM
       elasticsearch-transport (= 2.0.2)
     elasticsearch-api (2.0.2)
       multi_json
+    elasticsearch-model (5.0.0)
+      activesupport (> 3)
+      elasticsearch (> 1)
+      hashie
+    elasticsearch-rails (5.0.0)
     elasticsearch-transport (2.0.2)
       faraday
       multi_json
@@ -277,7 +263,7 @@ GEM
       thor (>= 0.14, < 2.0)
     js_cookie_rails (2.1.4)
       railties (>= 3.1)
-    json (2.0.3)
+    json (2.0.4)
     json_spec (1.1.4)
       multi_json (~> 1.0)
       rspec (>= 2.0, < 4.0)
@@ -343,10 +329,14 @@ GEM
       omniauth (~> 1.2)
     org-ruby (0.9.12)
       rubypants (~> 0.2)
-    ox (2.4.11)
+    ox (2.4.12)
     parallel (1.11.1)
     pg (0.20.0)
     pg_array_parser (0.0.9)
+    pg_query (0.11.4)
+      json (>= 1.8, < 3)
+    pghero (1.6.4)
+      activerecord
     pictogram (2.0.8)
       railties
     poltergeist (1.14.0)
@@ -429,7 +419,7 @@ GEM
       rspec-expectations (~> 3.5.0)
       rspec-mocks (~> 3.5.0)
       rspec-support (~> 3.5.0)
-    rspec-sidekiq (3.0.0)
+    rspec-sidekiq (3.0.1)
       rspec-core (~> 3.0, >= 3.0.0)
       sidekiq (>= 2.4.0)
     rspec-support (3.5.0)
@@ -471,6 +461,9 @@ GEM
       connection_pool (~> 2.2, >= 2.2.0)
       rack-protection (>= 1.5.0)
       redis (~> 3.2, >= 3.2.1)
+    sidekiq-unique-jobs (5.0.0)
+      sidekiq (>= 4.0)
+      thor
     simple-rss (1.3.1)
     simple_form (3.4.0)
       actionpack (> 4, < 5.1)
@@ -506,13 +499,11 @@ GEM
       net-ssh (>= 2.8.0)
     sshkit-interactive (0.2.0)
       sshkit (~> 1.9)
-    stripe (2.2.0)
-      faraday (~> 0)
+    stripe (2.2.1)
+      faraday (~> 0.9)
     stripe_event (1.6.0)
       activesupport (>= 3.1)
       stripe (>= 1.6, < 3.0)
-    terminal-table (1.7.3)
-      unicode-display_width (~> 1.1.1)
     thor (0.19.4)
     thread_safe (0.3.6)
     tilt (2.0.7)
@@ -525,9 +516,8 @@ GEM
       ethon (>= 0.9.0)
     tzinfo (1.2.3)
       thread_safe (~> 0.1)
-    uglifier (3.1.13)
+    uglifier (3.2.0)
       execjs (>= 0.3.0, < 3)
-    unicode-display_width (1.1.3)
     websocket-driver (0.6.5)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.2)
@@ -567,8 +557,8 @@ DEPENDENCIES
   database_cleaner
   dotenv-rails
   elasticsearch (~> 2)
-  elasticsearch-model!
-  elasticsearch-rails!
+  elasticsearch-model
+  elasticsearch-rails
   escape_utils
   factory_girl_rails
   faker
@@ -607,6 +597,8 @@ DEPENDENCIES
   parallel
   payola-payments!
   pg
+  pg_query
+  pghero
   pictogram
   poltergeist
   postgres_ext!
@@ -632,7 +624,7 @@ DEPENDENCIES
   semantic_range
   shoulda
   sidekiq
-  sidekiq-unique-jobs!
+  sidekiq-unique-jobs
   simple-rss
   simple_form
   simplecov

diff --git a/app/assets/images/ben-laurie.jpg b/app/assets/images/ben-laurie.jpg
diff --git a/app/controllers/admin/stats_controller.rb b/app/controllers/admin/stats_controller.rb
@@ -15,7 +15,6 @@ def index
   def repositories
     @new_users          = stats_for(RepositoryUser)
     @new_manifests      = stats_for(Manifest)
-    @new_readmes        = stats_for(Readme)
     @new_orgs           = stats_for(RepositoryOrganisation)
   end
 

diff --git a/app/controllers/api/docs_controller.rb b/app/controllers/api/docs_controller.rb
@@ -22,6 +22,6 @@ def index
 
     @search = Project.search('grunt').records
 
-    @repository_user = RepositoryUser.find_by_login('andrew') || RepositoryUser.first
+    @repository_user = RepositoryUser.host('GitHub').find_by_login('andrew') || RepositoryUser.first
   end
 end
diff --git a/app/controllers/api/repository_users_controller.rb b/app/controllers/api/repository_users_controller.rb
@@ -19,8 +19,8 @@ def projects
   private
 
   def find_user
-    @repository_user = RepositoryUser.visible.where("lower(login) = ?", params[:login].downcase).first
-    @repository_user = RepositoryOrganisation.visible.where("lower(login) = ?", params[:login].downcase).first if @repository_user.nil?
+    @repository_user = RepositoryUser.host(current_host).visible.where("lower(login) = ?", params[:login].downcase).first
+    @repository_user = RepositoryOrganisation.host(current_host).visible.where("lower(login) = ?", params[:login].downcase).first if @repository_user.nil?
     raise ActiveRecord::RecordNotFound if @repository_user.nil?
   end
 end
diff --git a/app/controllers/api/status_controller.rb b/app/controllers/api/status_controller.rb
@@ -4,8 +4,8 @@ class Api::StatusController < Api::ApplicationController
   def check
     if params[:projects].any?
       @projects = params[:projects].group_by{|project| project[:platform] }.map do |platform, projects|
-        projects.each_slice(200).map do |slice|
-          Project.lookup_multiple(find_platform_by_name(platform), projects.map{|project| project[:name] }).paginate(page: 1, per_page: 200).records.includes(:repository, :versions)
+        projects.each_slice(500).map do |slice|
+          Project.lookup_multiple(find_platform_by_name(platform), projects.map{|project| project[:name] }).paginate(page: 1, per_page: 500).records.includes(:repository, :versions)
         end.flatten.compact
       end.flatten.compact
     else

diff --git a/app/controllers/repository_users_controller.rb b/app/controllers/repository_users_controller.rb
@@ -42,9 +42,8 @@ def contributors
   private
 
   def find_user
-    raise ActiveRecord::RecordNotFound unless current_host == 'github'
-    @user = RepositoryUser.visible.where("lower(login) = ?", params[:login].downcase).first
-    @user = RepositoryOrganisation.visible.where("lower(login) = ?", params[:login].downcase).first if @user.nil?
+    @user = RepositoryUser.host(current_host).visible.where("lower(login) = ?", params[:login].downcase).first
+    @user = RepositoryOrganisation.host(current_host).visible.where("lower(login) = ?", params[:login].downcase).first if @user.nil?
     raise ActiveRecord::RecordNotFound if @user.nil?
     redirect_to url_for(login: @user.login), :status => :moved_permanently if params[:login] != @user.login
   end

diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb
@@ -200,7 +200,7 @@ def meta_tags_for(record)
       })
     when 'RepositoryUser', 'RepositoryOrganisation'
       hash = record.meta_tags.merge({
-        url: user_url(record.login)
+        url: user_url(record.to_param)
       })
     else
       hash = {}

diff --git a/app/models/concerns/github_identity.rb b/app/models/concerns/github_identity.rb
@@ -5,7 +5,7 @@ def github_enabled?
 
   def repository_user
     return unless github_enabled?
-    RepositoryUser.find_by_uuid(github_identity.uid)
+    RepositoryUser.host('GitHub').find_by_uuid(github_identity.uid)
   end
 
   def hidden
@@ -75,7 +75,7 @@ def update_repo_permissions
 
   def download_self
     return unless github_identity
-    RepositoryUser.create_from_github(OpenStruct.new({id: github_identity.uid, login: github_identity.nickname, type: 'User'}))
+    RepositoryUser.create_from_github(OpenStruct.new({id: github_identity.uid, login: github_identity.nickname, type: 'User', host_type: 'GitHub'}))
     RepositoryUpdateUserWorker.perform_async(nickname)
   end
 

diff --git a/app/models/concerns/profile.rb b/app/models/concerns/profile.rb
diff --git a/app/models/identity.rb b/app/models/identity.rb
@@ -1,9 +1,17 @@
 class Identity < ApplicationRecord
   belongs_to :user
+  belongs_to :repository_user
   validates_presence_of :uid, :provider
 
   scope :viewable, -> { where(provider: ['github', 'gitlab', 'bitbucket']) }
 
+  def to_param
+    {
+        host_type: provider.downcase,
+        login: nickname
+      }
+  end
+
   def self.find_with_omniauth(auth)
     find_by(uid: auth['uid'], provider: auth['provider'])
   end

diff --git a/app/models/package_manager/base.rb b/app/models/package_manager/base.rb
@@ -168,7 +168,7 @@ def self.save_dependencies(mapped_project)
       name = mapped_project[:name]
       proj = Project.find_by(name: name, platform: self.name.demodulize)
       proj.versions.includes(:dependencies).each do |version|
-        deps = dependencies(name, version.number, mapped_project)
+        deps = dependencies(name, version.number, mapped_project) rescue []
         next unless deps && deps.any? && version.dependencies.empty?
         deps.each do |dep|
           unless version.dependencies.find_by_project_name dep[:project_name]

diff --git a/app/models/project.rb b/app/models/project.rb
@@ -177,7 +177,7 @@ def install_instructions(version = nil)
 
   def owner
     return nil unless repository && repository.host_type == 'GitHub'
-    RepositoryUser.visible.find_by_login repository.owner_name
+    RepositoryUser.host('GitHub').visible.find_by_login repository.owner_name
   end
 
   def platform_class

diff --git a/app/models/repository.rb b/app/models/repository.rb
@@ -30,8 +30,8 @@ class Repository < ApplicationRecord
   belongs_to :source, primary_key: :full_name, foreign_key: :source_name, anonymous_class: Repository
   has_many :forked_repositories, primary_key: :full_name, foreign_key: :source_name, anonymous_class: Repository
 
-  validates :full_name, uniqueness: true, if: lambda { self.full_name_changed? }
-  validates :uuid, uniqueness: true, if: lambda { self.uuid_changed? }
+  validates :full_name, uniqueness: {scope: :host_type}, if: lambda { self.full_name_changed? }
+  validates :uuid, uniqueness: {scope: :host_type}, if: lambda { self.uuid_changed? }
 
   before_save  :normalize_license_and_language
   after_commit :update_all_info_async, on: :create
@@ -138,7 +138,6 @@ def repository_dependencies
   end
 
   def owner
-    return nil unless host_type == 'GitHub'
     repository_organisation_id.present? ? repository_organisation : repository_user
   end
 
@@ -190,7 +189,11 @@ def load_dependencies_tree(date = nil)
   end
 
   def id_or_name
-    uuid || full_name
+    if host_type == 'GitHub'
+      uuid || full_name
+    else
+      full_name
+    end
   end
 
   def update_all_info_async(token = nil)

diff --git a/app/models/repository_host/bitbucket.rb b/app/models/repository_host/bitbucket.rb
@@ -5,7 +5,8 @@ class Bitbucket < Base
       BitBucket::Error::Forbidden,
       BitBucket::Error::ServiceError,
       BitBucket::Error::InternalServerError,
-      BitBucket::Error::ServiceUnavailable]
+      BitBucket::Error::ServiceUnavailable,
+      BitBucket::Error::Unauthorized]
 
     def self.api_missing_error_class
       BitBucket::Error::NotFound