diff --git a/Gemfile b/Gemfile index 9a1688629..5fec8246b 100644 --- a/Gemfile +++ b/Gemfile @@ -66,7 +66,7 @@ gem 'omniauth-bitbucket' gem 'chartkick' gem 'groupdate' gem 'turbolinks' -gem 'gitlab', git: 'https://github.com/librariesio/gitlab' +gem 'gitlab' gem 'bitbucket_rest_api', git: 'https://github.com/librariesio/bitbucket' gem 'github-markup', require: 'github/markup' gem 'commonmarker' diff --git a/Gemfile.lock b/Gemfile.lock index 72a9fbe44..533e28834 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -20,13 +20,6 @@ GIT nokogiri (>= 1.5.2) simple_oauth -GIT - remote: https://github.com/librariesio/gitlab - revision: 69b59cd43b29c316b4f380610bb961ec398b22ee - specs: - gitlab (4.0.0) - httparty - GIT remote: https://github.com/librariesio/payola revision: 7d4026856edf0cbafc8a188748ba96ced7bad99b @@ -242,6 +235,9 @@ GEM mime-types (>= 1.19) rugged (>= 0.25.1) github-markup (1.6.0) + gitlab (4.0.0) + httparty + terminal-table (= 1.7.1) globalid (0.4.0) activesupport (>= 4.2.0) groupdate (3.2.0) @@ -524,6 +520,8 @@ GEM stripe_event (1.6.0) activesupport (>= 3.1) stripe (>= 1.6, < 3.0) + terminal-table (1.7.1) + unicode-display_width (~> 1.1.1) thor (0.19.4) thread_safe (0.3.6) tilt (2.0.7) @@ -538,6 +536,7 @@ GEM thread_safe (~> 0.1) uglifier (3.2.0) execjs (>= 0.3.0, < 3) + unicode-display_width (1.1.3) websocket-driver (0.6.5) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.2) @@ -594,7 +593,7 @@ DEPENDENCIES gemoji github-linguist github-markup - gitlab! + gitlab groupdate highscore hiredis diff --git a/app/models/concerns/github_identity.rb b/app/models/concerns/github_identity.rb index fcc7ce747..a56407bff 100644 --- a/app/models/concerns/github_identity.rb +++ b/app/models/concerns/github_identity.rb @@ -75,14 +75,17 @@ def update_repo_permissions def download_self return unless github_identity - RepositoryUser.create_from_github(OpenStruct.new({id: github_identity.uid, login: github_identity.nickname, type: 'User', host_type: 'GitHub'})) - RepositoryUpdateUserWorker.perform_async(nickname) + repository_user = RepositoryUser.create_from_host('GitHub', {id: github_identity.uid, login: github_identity.nickname, type: 'User', host_type: 'GitHub'}) + if repository_user + github_identity.update_column(:repository_user_id, repository_user.id) + RepositoryUpdateUserWorker.perform_async('GitHub', nickname) + end end def download_orgs return unless token github_client.orgs.each do |org| - RepositoryCreateOrgWorker.perform_async(org.login) + RepositoryCreateOrgWorker.perform_async('GitHub', org.login) end rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS nil diff --git a/app/models/repository.rb b/app/models/repository.rb index c7b388942..fef1c5978 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -4,6 +4,9 @@ class Repository < ApplicationRecord include RepoManifests include RepositorySourceRank + # eager load this module to avoid clashing with Gitlab gem in development + RepositoryHost::Gitlab + STATUSES = ['Active', 'Deprecated', 'Unmaintained', 'Help Wanted', 'Removed'] API_FIELDS = [:full_name, :description, :fork, :created_at, :updated_at, :pushed_at, :homepage, @@ -302,7 +305,6 @@ def github_id end def repository_host - RepositoryHost::Gitlab @repository_host ||= RepositoryHost.const_get(host_type.capitalize).new(self) end end diff --git a/app/models/repository_host/base.rb b/app/models/repository_host/base.rb index 69a61bb26..28733cdba 100644 --- a/app/models/repository_host/base.rb +++ b/app/models/repository_host/base.rb @@ -78,6 +78,10 @@ def formatted_host self.class.format(repository.host_type) end + def repository_owner_class + RepositoryOwner.const_get(repository.host_type.capitalize) + end + def update_from_host(token = nil) begin r = self.class.fetch_repo(repository.id_or_name) diff --git a/app/models/repository_host/bitbucket.rb b/app/models/repository_host/bitbucket.rb index a74291022..b9c3359da 100644 --- a/app/models/repository_host/bitbucket.rb +++ b/app/models/repository_host/bitbucket.rb @@ -62,7 +62,25 @@ def download_forks(token = nil) end def download_owner - # not implemented yet + return if repository.owner && repository.repository_user_id && repository.owner.login == repository.owner_name + o = RepositoryOwner::Bitbucket.fetch_user(repository.owner_name) + if o.type == "team" + org = RepositoryOrganisation.create_from_host('Bitbucket', o) + if org + repository.repository_organisation_id = org.id + repository.repository_user_id = nil + repository.save + end + else + u = RepositoryUser.create_from_host('Bitbucket', o) + if u + repository.repository_user_id = u.id + repository.repository_organisation_id = nil + repository.save + end + end + rescue *IGNORABLE_EXCEPTIONS + nil end def create_webook(token = nil) diff --git a/app/models/repository_host/github.rb b/app/models/repository_host/github.rb index 15b386693..8167a6b8d 100644 --- a/app/models/repository_host/github.rb +++ b/app/models/repository_host/github.rb @@ -91,7 +91,7 @@ def download_contributions(token = nil) next unless c['id'] cont = existing_contributions.find{|cnt| cnt.repository_user.try(:uuid) == c.id } unless cont - user = RepositoryUser.create_from_github(c) + user = RepositoryUser.create_from_host('GitHub', c) cont = repository.contributions.find_or_create_by(repository_user: user) end @@ -127,14 +127,14 @@ def download_owner return if repository.owner && repository.repository_user_id && repository.owner.login == repository.owner_name o = api_client.user(repository.owner_name) if o.type == "Organization" - go = RepositoryOrganisation.create_from_github(o.id) + go = RepositoryOrganisation.create_from_host('GitHub', o) if go repository.repository_organisation_id = go.id repository.repository_user_id = nil repository.save end else - u = RepositoryUser.create_from_github(o) + u = RepositoryUser.create_from_host('GitHub', o) if u repository.repository_user_id = u.id repository.repository_organisation_id = nil diff --git a/app/models/repository_host/gitlab.rb b/app/models/repository_host/gitlab.rb index 27073cdb8..73bcd02f2 100644 --- a/app/models/repository_host/gitlab.rb +++ b/app/models/repository_host/gitlab.rb @@ -44,7 +44,28 @@ def download_forks(token = nil) end def download_owner - # not implemented yet + return if repository.owner && repository.repository_user_id && repository.owner.login == repository.owner_name + namespace = api_client.namespaces(search: repository.owner_name).try(:first) + return unless namespace + if namespace.kind == 'group' + o = RepositoryOwner::Gitlab.api_client.group(namespace.path) + org = RepositoryOrganisation.create_from_host('GitLab', o) + if org + repository.repository_organisation_id = org.id + repository.repository_user_id = nil + repository.save + end + elsif namespace.kind == 'user' + o = RepositoryOwner::Gitlab.fetch_user(namespace.path) + u = RepositoryUser.create_from_host('GitLab', o) + if u + repository.repository_user_id = u.id + repository.repository_organisation_id = nil + repository.save + end + end + rescue *IGNORABLE_EXCEPTIONS + nil end def create_webook(token = nil) diff --git a/app/models/repository_organisation.rb b/app/models/repository_organisation.rb index 3bbba0a00..331141fb4 100644 --- a/app/models/repository_organisation.rb +++ b/app/models/repository_organisation.rb @@ -10,6 +10,10 @@ class RepositoryOrganisation < ApplicationRecord has_many :contributors, -> { group('repository_users.id').order("sum(contributions.count) DESC") }, through: :open_source_repositories, source: :contributors has_many :projects, through: :open_source_repositories + # eager load this module to avoid clashing with Gitlab gem in development + RepositoryOwner::Gitlab + + validates :uuid, presence: true validates :login, uniqueness: {scope: :host_type}, if: lambda { self.login_changed? } validates :uuid, uniqueness: {scope: :host_type}, if: lambda { self.uuid_changed? } @@ -23,10 +27,10 @@ class RepositoryOrganisation < ApplicationRecord scope :host, lambda{ |host_type| where('lower(repository_organisations.host_type) = ?', host_type.try(:downcase)) } delegate :avatar_url, :repository_url, :top_favourite_projects, :top_contributors, - :to_s, :to_param, :github_id, to: :repository_owner + :to_s, :to_param, :github_id, :download_org_from_host, :download_orgs, + :download_org_from_host_by_login, :download_repos, to: :repository_owner def repository_owner - RepositoryOwner::Gitlab @repository_owner ||= RepositoryOwner.const_get(host_type.capitalize).new(self) end @@ -50,10 +54,6 @@ def company nil end - def github_client - AuthToken.client - end - def user_type 'Organisation' end @@ -66,46 +66,8 @@ def following 0 end - def self.create_from_github(login_or_id) - begin - r = AuthToken.client.org(login_or_id).to_hash - return false if r.blank? - - org = nil - org_by_id = RepositoryOrganisation.host('GitHub').find_by_uuid(r[:id]) - if r[:login].present? - org_by_login = RepositoryOrganisation.host('GitHub').where("lower(login) = ?", r[:login].downcase).first - else - org_by_login = nil - end - - if org_by_id # its fine - if org_by_id.login.try(:downcase) == r[:login].try(:downcase) - org = org_by_id - else - if org_by_login && !org_by_login.download_from_github - org_by_login.destroy - end - org_by_id.login = r[:login] - org_by_id.save! - org = org_by_id - end - elsif org_by_login # conflict - if org_by_login.download_from_github_by_login - org = org_by_login if org_by_login.github_id == r[:id] - end - org_by_login.destroy if org.nil? - end - if org.nil? - org = RepositoryOrganisation.create!(uuid: r[:id], login: r[:login], host_type: 'GitHub') - end - - org.assign_attributes r.slice(*RepositoryOrganisation::API_FIELDS) - org.save - org - rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS - false - end + def self.create_from_host(host_type, org_hash) + RepositoryOwner.const_get(host_type.capitalize).create_org(org_hash) end def async_sync @@ -115,31 +77,10 @@ def async_sync def sync download_from_github download_repos + download_members update_attributes(last_synced_at: Time.now) end - def download_from_github - download_from_github_by(github_id) - end - - def download_from_github_by_login - download_from_github_by(login) - end - - def download_from_github_by(id_or_login) - RepositoryOrganisation.create_from_github(github_client.org(id_or_login)) - rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS - nil - end - - def download_repos - github_client.org_repos(login).each do |repo| - CreateRepositoryWorker.perform_async('GitHub', repo.full_name) - end - rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS - nil - end - def find_repositories Repository.host(host_type).where('full_name ILIKE ?', "#{login}/%").update_all(repository_user_id: nil, repository_organisation_id: self.id) end diff --git a/app/models/repository_owner/base.rb b/app/models/repository_owner/base.rb index d6b2e3111..7bb1ac5d5 100644 --- a/app/models/repository_owner/base.rb +++ b/app/models/repository_owner/base.rb @@ -37,6 +37,69 @@ def github_id owner.uuid end + def download_orgs + raise NotImplementedError + end + + def download_repos + raise NotImplementedError + end + + def self.format(host_type) + case host_type.try(:downcase) + when 'github' + 'GitHub' + when 'gitlab' + 'GitLab' + when 'bitbucket' + 'Bitbucket' + end + end + + def formatted_host + self.class.format(repository.host_type) + end + + def download_user_from_host + download_user_from_host_by(owner.uuid) + end + + def download_user_from_host_by_login + download_user_from_host_by(owner.login) + end + + def download_user_from_host_by(id_or_login) + self.class.download_user_from_host(owner.host_type, id_or_login) + end + + def self.download_user_from_host(host_type, id_or_login) + RepositoryUser.create_from_host(host_type, self.fetch_user(id_or_login)) + end + + def download_org_from_host + download_org_from_host_by(owner.uuid) + end + + def download_org_from_host_by_login + download_org_from_host_by(owner.login) + end + + def download_org_from_host_by(id_or_login) + self.class.download_org_from_host(owner.host_type, id_or_login) + end + + def self.download_org_from_host(host_type, id_or_login) + RepositoryOrganisation.create_from_host(host_type, self.fetch_org(id_or_login)) + end + + def self.fetch_user(id_or_login) + raise NotImplementedError + end + + def self.fetch_org(id_or_login) + raise NotImplementedError + end + private def top_favourite_project_ids @@ -45,7 +108,6 @@ def top_favourite_project_ids end end - def top_contributor_ids Rails.cache.fetch [owner, "top_contributor_ids"], :expires_in => 1.week, race_condition_ttl: 2.minutes do owner.contributors.visible.limit(50).pluck(:id) diff --git a/app/models/repository_owner/bitbucket.rb b/app/models/repository_owner/bitbucket.rb index a73a9c93b..62c84d4f7 100644 --- a/app/models/repository_owner/bitbucket.rb +++ b/app/models/repository_owner/bitbucket.rb @@ -1,11 +1,151 @@ module RepositoryOwner class Bitbucket < Base def avatar_url(size = 60) - "https://bitbucket.org/account/#{owner.login}/avatar/256/?ts=#{owner.uuid}" + "https://bitbucket.org/account/#{owner.login}/avatar/256" end def repository_url "https://bitbucket.org/#{owner.login}" end + + def self.fetch_user(id_or_login) + begin + api_client.get_request "/2.0/users/#{URI.escape(id_or_login)}" + rescue BitBucket::Error::NotFound => error + if error.message.index('is a team account') + api_client.get_request "/2.0/teams/#{URI.escape(id_or_login)}" + end + end + rescue *RepositoryHost::BitBucket::IGNORABLE_EXCEPTIONS + nil + end + + def self.fetch_org(id_or_login) + api_client.get_request "/2.0/teams/#{URI.escape(id_or_login)}" + rescue *RepositoryHost::BitBucket::IGNORABLE_EXCEPTIONS + nil + end + + def self.api_client(token = nil) + BitBucket.new oauth_token: token || ENV['BITBUCKET_KEY'] + end + + def api_client(token = nil) + self.class.api_client(token) + end + + def download_orgs + return if owner.org? + # Bitbucket doesn't have an API to get a users public group memberships so we scrape it instead + groups_html = PackageManager::Base.get_html("https://bitbucket.org/#{owner.login}/profile/teams") + links = groups_html.css('li.team a.name').map{|l| l['title']} + + links.each do |org_name| + RepositoryCreateOrgWorker.perform_async('Bitbucket', org_name) + end + true + rescue *RepositoryHost::Bitbucket::IGNORABLE_EXCEPTIONS + nil + end + + def download_repos + api_client.get_request("/2.0/repositories/#{owner.login}")['values'].each do |repo| + CreateRepositoryWorker.perform_async('Bitbucket', repo.full_name) + end + true + rescue *RepositoryHost::Bitbucket::IGNORABLE_EXCEPTIONS + nil + end + + def download_members + return unless owner.org? + + api_client.teams.members(owner.login).each do |org| + RepositoryCreateUserWorker.perform_async('Bitbucket', org.login) + end + true + rescue *RepositoryHost::Bitbucket::IGNORABLE_EXCEPTIONS + nil + end + + def self.create_user(user_hash) + user_hash = user_hash.to_hash.with_indifferent_access + user_hash = { + id: user_hash[:uuid], + login: user_hash[:username], + name: user_hash[:display_name], + blog: user_hash[:website], + location: user_hash[:location], + type: user_hash[:type], + host_type: 'Bitbucket' + } + user = nil + user_by_id = RepositoryUser.host('Bitbucket').find_by_uuid(user_hash[:id]) + user_by_login = RepositoryUser.host('Bitbucket').where("lower(login) = ?", user_hash[:login].try(:downcase)).first + if user_by_id # its fine + if user_by_id.login.try(:downcase) == user_hash[:login].downcase && user_by_id.user_type == user_hash[:type] + user = user_by_id + else + if user_by_login && !user_by_login.download_user_from_host + user_by_login.destroy + end + user_by_id.login = user_hash[:login] + user_by_id.user_type = user_hash[:type] + user_by_id.save! + user = user_by_id + end + elsif user_by_login # conflict + if user_by_login.download_user_from_host_by_login + user = user_by_login if user_by_login.uuid == user_hash[:id] + end + user_by_login.destroy if user.nil? + end + if user.nil? + user = RepositoryUser.create!(uuid: user_hash[:id], login: user_hash[:login], user_type: user_hash[:type], host_type: 'Bitbucket') + end + + user.update(user_hash.slice(:name, :blog, :location)) + user + end + + def self.create_org(org_hash) + org_hash = org_hash.to_hash.with_indifferent_access + org_hash = { + id: org_hash[:uuid], + login: org_hash[:username], + name: org_hash[:display_name], + blog: org_hash[:website], + location: org_hash[:location], + type: org_hash[:type], + host_type: 'Bitbucket' + } + org = nil + org_by_id = RepositoryOrganisation.host('Bitbucket').find_by_uuid(org_hash[:id]) + org_by_login = RepositoryOrganisation.host('Bitbucket').where("lower(login) = ?", org_hash[:login].try(:downcase)).first + if org_by_id # its fine + if org_by_id.login.try(:downcase) == org_hash[:login].downcase && org_by_id.user_type == org_hash[:type] + org = org_by_id + else + if org_by_login && !org_by_login.download_org_from_host + org_by_login.destroy + end + org_by_id.login = org_hash[:login] + org_by_id.org_type = org_hash[:type] + org_by_id.save! + org = org_by_id + end + elsif org_by_login # conflict + if org_by_login.download_org_from_host_by_login + org = org_by_login if org_by_login.uuid == org_hash[:id] + end + org_by_login.destroy if org.nil? + end + if org.nil? + org = RepositoryOrganisation.create!(uuid: org_hash[:id], login: org_hash[:login], user_type: org_hash[:type], host_type: 'Bitbucket') + end + + org.update(org_hash.slice(:name, :blog, :location)) + org + end end end diff --git a/app/models/repository_owner/github.rb b/app/models/repository_owner/github.rb index ea59cec01..0427ec498 100644 --- a/app/models/repository_owner/github.rb +++ b/app/models/repository_owner/github.rb @@ -7,5 +7,105 @@ def avatar_url(size = 60) def repository_url "https://github.com/#{owner.login}" end + + def download_orgs + return if owner.org? + api_client.orgs(owner.login).each do |org| + RepositoryCreateOrgWorker.perform_async('GitHub', org.login) + end + true + rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS + nil + end + + def download_repos + api_client.search_repos("user:#{owner.login}").items.each do |repo| + CreateRepositoryWorker.perform_async('GitHub', repo.full_name) + end + true + rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS + nil + end + + def download_members + return unless owner.org? + + api_client.organization_members(owner.login).each do |org| + RepositoryCreateUserWorker.perform_async('GitHub', org.login) + end + true + rescue *RepositoryHost::Gitlab::IGNORABLE_EXCEPTIONS + nil + end + + def self.fetch_user(id_or_login) + api_client.user(id_or_login) + rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS + nil + end + + def self.api_client(token = nil) + AuthToken.fallback_client(token) + end + + def self.create_org(org_hash) + org_hash = org_hash.to_hash.with_indifferent_access + org = nil + org_by_id = RepositoryOrganisation.host('GitHub').find_by_uuid(org_hash[:id]) + org_by_login = RepositoryOrganisation.host('GitHub').where("lower(login) = ?", org_hash[:login].try(:downcase)).first + if org_by_id # its fine + if org_by_id.login.try(:downcase) == org_hash[:login].downcase && org_by_id.user_type == org_hash[:type] + org = org_by_id + else + if org_by_login && !org_by_login.download_org_from_host + org_by_login.destroy + end + org_by_id.login = org_hash[:login] + org_by_id.user_type = org_hash[:type] + org_by_id.save! + org = org_by_id + end + elsif org_by_login # conflict + if org_by_login.download_org_from_host_by_login + org = org_by_login if org_by_login.uuid == org_hash[:id] + end + org_by_login.destroy if org.nil? + end + if org.nil? + org = RepositoryOrganisation.create!(uuid: org_hash[:id], login: org_hash[:login], user_type: org_hash[:type], host_type: 'GitHub') + end + org.update(org_hash.slice(:name, :company, :blog, :location, :email, :bio)) + org + end + + def self.create_user(user_hash) + user_hash = user_hash.to_hash.with_indifferent_access + user = nil + user_by_id = RepositoryUser.host('GitHub').find_by_uuid(user_hash[:id]) + user_by_login = RepositoryUser.host('GitHub').where("lower(login) = ?", user_hash[:login].try(:downcase)).first + if user_by_id # its fine + if user_by_id.login.try(:downcase) == user_hash[:login].downcase && user_by_id.user_type == user_hash[:type] + user = user_by_id + else + if user_by_login && !user_by_login.download_user_from_host + user_by_login.destroy + end + user_by_id.login = user_hash[:login] + user_by_id.user_type = user_hash[:type] + user_by_id.save! + user = user_by_id + end + elsif user_by_login # conflict + if user_by_login.download_user_from_host_by_login + user = user_by_login if user_by_login.uuid == user_hash[:id] + end + user_by_login.destroy if user.nil? + end + if user.nil? + user = RepositoryUser.create!(uuid: user_hash[:id], login: user_hash[:login], user_type: user_hash[:type], host_type: 'GitHub') + end + user.update(user_hash.slice(:name, :company, :blog, :location, :email, :bio)) + user + end end end diff --git a/app/models/repository_owner/gitlab.rb b/app/models/repository_owner/gitlab.rb index 08ba3a0b9..5b94025f9 100644 --- a/app/models/repository_owner/gitlab.rb +++ b/app/models/repository_owner/gitlab.rb @@ -7,5 +7,154 @@ def avatar_url(size = 60) def repository_url "https://gitlab.com/#{owner.login}" end + + def self.fetch_user(id_or_login) + if id_or_login.to_i.zero? + api_client.get("/users?username=#{id_or_login}").first + else + api_client.user(id_or_login) + end + rescue *RepositoryHost::Gitlab::IGNORABLE_EXCEPTIONS + nil + end + + def self.fetch_org(id_or_login) + api_client.group(id_or_login) + rescue *RepositoryHost::Gitlab::IGNORABLE_EXCEPTIONS + nil + end + + def self.api_client(token = nil) + ::Gitlab.client(endpoint: 'https://gitlab.com/api/v3', private_token: token || ENV['GITLAB_KEY']) + end + + def api_client(token = nil) + self.class.api_client(token) + end + + def download_orgs + return if owner.org? + + # GitLab doesn't have an API to get a users public group memberships so we scrape it instead + groups_html = Nokogiri::HTML(PackageManager::Base.get_json("https://gitlab.com/users/#{owner.login}/groups")['html']) + links = groups_html.css('a.group-name').map{|l| l['href'][1..-1]} + + links.each do |org_login| + RepositoryCreateOrgWorker.perform_async('GitLab', org_login) + end + true + rescue *RepositoryHost::Gitlab::IGNORABLE_EXCEPTIONS + nil + end + + def download_repos + if owner.org? + repos = api_client.group_projects(owner.login).map(&:full_name) + else + # GitLab doesn't have an API to get a users public projects so we scrape it instead + projects_html = Nokogiri::HTML(PackageManager::Base.get_json("https://gitlab.com/users/#{owner.login}/projects")['html']) + links = projects_html.css('a.project').map{|l| l['href'][1..-1] }.uniq + end + + links.each do |repo_name| + CreateRepositoryWorker.perform_async('GitLab', repo_name) + end + true + rescue *RepositoryHost::Gitlab::IGNORABLE_EXCEPTIONS + nil + end + + def download_members + return unless owner.org? + + api_client.group_members(owner.login).each do |org| + RepositoryCreateUserWorker.perform_async('GitLab', org.login) + end + true + rescue *RepositoryHost::Gitlab::IGNORABLE_EXCEPTIONS + nil + end + + def self.create_user(user_hash) + user_hash = user_hash.to_hash.with_indifferent_access + user_hash = { + id: user_hash[:id], + login: user_hash[:username], + name: user_hash[:name], + blog: user_hash[:website_url], + location: user_hash[:location], + bio: user_hash[:bio], + type: 'User', + host_type: 'GitLab' + } + user = nil + user_by_id = RepositoryUser.host('GitLab').find_by_uuid(user_hash[:id]) + user_by_login = RepositoryUser.host('GitLab').where("lower(login) = ?", user_hash[:login].try(:downcase)).first + if user_by_id # its fine + if user_by_id.login.try(:downcase) == user_hash[:login].downcase && user_by_id.user_type == user_hash[:type] + user = user_by_id + else + if user_by_login && !user_by_login.download_user_from_host + user_by_login.destroy + end + user_by_id.login = user_hash[:login] + user_by_id.user_type = user_hash[:type] + user_by_id.save! + user = user_by_id + end + elsif user_by_login # conflict + if user_by_login.download_user_from_host_by_login + user = user_by_login if user_by_login.uuid == user_hash[:id] + end + user_by_login.destroy if user.nil? + end + if user.nil? + user = RepositoryUser.create!(uuid: user_hash[:id], login: user_hash[:login], user_type: user_hash[:type], host_type: 'GitLab') + end + + user.update(user_hash.slice(:name, :blog, :location)) + user + end + + def self.create_org(org_hash) + org_hash = org_hash.to_hash.with_indifferent_access + org_hash = { + id: org_hash[:id], + login: org_hash[:orgname], + name: org_hash[:name], + blog: org_hash[:website_url], + location: org_hash[:location], + bio: org_hash[:bio], + type: 'Organisation', + host_type: 'GitLab' + } + org = nil + org_by_id = RepositoryOrganisation.host('GitLab').find_by_uuid(org_hash[:id]) + org_by_login = RepositoryOrganisation.host('GitLab').where("lower(login) = ?", org_hash[:login].try(:downcase)).first + if org_by_id # its fine + if org_by_id.login.try(:downcase) == org_hash[:login].downcase && org_by_id.user_type == org_hash[:type] + org = org_by_id + else + if org_by_login && !org_by_login.download_org_from_host + org_by_login.destroy + end + org_by_id.login = org_hash[:login] + org_by_id.user_type = org_hash[:type] + org_by_id.save! + org = org_by_id + end + elsif org_by_login # conflict + if org_by_login.download_org_from_host_by_login + org = org_by_login if org_by_login.uuid == org_hash[:id] + end + org_by_login.destroy if org.nil? + end + if org.nil? + org = RepositoryOrganisation.create!(uuid: org_hash[:id], login: org_hash[:login], user_type: org_hash[:type], host_type: 'GitLab') + end + + org.update(org_hash.slice(:name, :blog, :location)) + org + end end end diff --git a/app/models/repository_user.rb b/app/models/repository_user.rb index bdbef4b3c..4909c88a2 100644 --- a/app/models/repository_user.rb +++ b/app/models/repository_user.rb @@ -12,6 +12,9 @@ class RepositoryUser < ApplicationRecord has_many :projects, through: :open_source_repositories has_many :identities + # eager load this module to avoid clashing with Gitlab gem in development + RepositoryOwner::Gitlab + has_many :issues, primary_key: :uuid validates :login, uniqueness: {scope: :host_type}, if: lambda { self.login_changed? } @@ -25,17 +28,13 @@ class RepositoryUser < ApplicationRecord scope :host, lambda{ |host_type| where('lower(repository_users.host_type) = ?', host_type.try(:downcase)) } delegate :avatar_url, :repository_url, :top_favourite_projects, :top_contributors, - :to_s, :to_param, :github_id, to: :repository_owner + :to_s, :to_param, :github_id, :download_user_from_host, :download_orgs, + :download_user_from_host_by_login, :download_repos, to: :repository_owner def repository_owner - RepositoryOwner::Gitlab @repository_owner ||= RepositoryOwner.const_get(host_type.capitalize).new(self) end - def github_id - uuid - end - def meta_tags { title: "#{self} on #{host_type}", @@ -52,80 +51,18 @@ def org? false end - def github_client - AuthToken.client - end - def async_sync - RepositoryUpdateUserWorker.perform_async(self.login) + RepositoryUpdateUserWorker.perform_async(host_type, self.login) end def sync - download_from_github + download_user_from_host download_orgs download_repos update_attributes(last_synced_at: Time.now) end - def download_from_github - download_from_github_by(uuid) - end - - def download_from_github_by_login - download_from_github_by(login) - end - - def download_from_github_by(id_or_login) - RepositoryUser.create_from_github(github_client.user(id_or_login)) - rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS - nil - end - - def download_orgs - github_client.orgs(login).each do |org| - RepositoryCreateOrgWorker.perform_async(org.login) - end - true - rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS - nil - end - - def download_repos - AuthToken.client.search_repos("user:#{login}").items.each do |repo| - Repository.create_from_hash repo.to_hash - end - - true - rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS - nil - end - - def self.create_from_github(repository_user) - user = nil - user_by_id = RepositoryUser.host('GitHub').find_by_uuid(repository_user.id) - user_by_login = RepositoryUser.host('GitHub').where("lower(login) = ?", repository_user.login.try(:downcase)).first - if user_by_id # its fine - if user_by_id.login.try(:downcase) == repository_user.login.downcase && user_by_id.user_type == repository_user.type - user = user_by_id - else - if user_by_login && !user_by_login.download_from_github - user_by_login.destroy - end - user_by_id.login = repository_user.login - user_by_id.user_type = repository_user.type - user_by_id.save! - user = user_by_id - end - elsif user_by_login # conflict - if user_by_login.download_from_github_by_login - user = user_by_login if user_by_login.uuid == repository_user.id - end - user_by_login.destroy if user.nil? - end - if user.nil? - user = RepositoryUser.create!(uuid: repository_user.id, login: repository_user.login, user_type: repository_user.type, host_type: 'GitHub') - end - user.update(repository_user.to_hash.slice(:name, :company, :blog, :location, :email, :bio)) - user + def self.create_from_host(host_type, user_hash) + RepositoryOwner.const_get(host_type.capitalize).create_user(user_hash) end end diff --git a/app/workers/repository_create_org_worker.rb b/app/workers/repository_create_org_worker.rb index a489595f7..374ed420d 100644 --- a/app/workers/repository_create_org_worker.rb +++ b/app/workers/repository_create_org_worker.rb @@ -2,7 +2,7 @@ class RepositoryCreateOrgWorker include Sidekiq::Worker sidekiq_options queue: :owners, unique: :until_executed - def perform(org_login) - RepositoryOrganisation.create_from_github(org_login) + def perform(host_type, org_login) + RepositoryOwner::Base.download_org_from_host(host_type, org_login) end end diff --git a/app/workers/repository_create_user_worker.rb b/app/workers/repository_create_user_worker.rb new file mode 100644 index 000000000..0b07cb07c --- /dev/null +++ b/app/workers/repository_create_user_worker.rb @@ -0,0 +1,8 @@ +class RepositoryCreateUserWorker + include Sidekiq::Worker + sidekiq_options queue: :owners, unique: :until_executed + + def perform(host_type, org_login) + RepositoryOwner::Base.download_user_from_host(host_type, org_login) + end +end diff --git a/app/workers/repository_update_user_worker.rb b/app/workers/repository_update_user_worker.rb index 2faf43ab1..3acef2a49 100644 --- a/app/workers/repository_update_user_worker.rb +++ b/app/workers/repository_update_user_worker.rb @@ -2,7 +2,7 @@ class RepositoryUpdateUserWorker include Sidekiq::Worker sidekiq_options queue: :owners, unique: :until_executed - def perform(login) - RepositoryUser.host('GitHub').find_by_login(login).try(:sync) + def perform(host_type, login) + RepositoryUser.host(host_type).find_by_login(login).try(:sync) end end diff --git a/db/migrate/20170411103819_add_host_to_owner_indexes.rb b/db/migrate/20170411103819_add_host_to_owner_indexes.rb new file mode 100644 index 000000000..0b7a0c58b --- /dev/null +++ b/db/migrate/20170411103819_add_host_to_owner_indexes.rb @@ -0,0 +1,22 @@ +class AddHostToOwnerIndexes < ActiveRecord::Migration[5.0] + def change + # drop indexes + remove_index :repository_users, name: :index_repository_users_on_login + remove_index :repository_users, :uuid + remove_index :repository_users, name: :github_users_lower_login + remove_index :repository_users, name: :index_github_users_on_lowercase_login + remove_index :repository_organisations, :uuid + remove_index :repository_organisations, name: :index_github_organisations_on_lowercase_login + + # change uuid to string + change_column :repository_users, :uuid, :string + change_column :repository_organisations, :uuid, :string + + # recreate indexes + add_index :repository_users, [:host_type, :login], unique: true, case_sensitive: false + add_index :repository_users, [:host_type, :uuid], unique: true + + add_index :repository_organisations, [:host_type, :login], unique: true, case_sensitive: false + add_index :repository_organisations, [:host_type, :uuid], unique: true + end +end diff --git a/db/schema.rb b/db/schema.rb index 38bd425ac..bcfbc6134 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -295,8 +295,8 @@ end create_table "repository_organisations", force: :cascade do |t| - t.string "login", :index=>{:name=>"index_github_organisations_on_lowercase_login", :unique=>true, :case_sensitive=>false} - t.integer "uuid", :index=>{:name=>"index_repository_organisations_on_uuid", :unique=>true} + t.string "login" + t.string "uuid" t.string "name" t.string "blog" t.string "email" @@ -306,7 +306,9 @@ t.datetime "updated_at", :null=>false t.boolean "hidden", :default=>false, :index=>{:name=>"index_repository_organisations_on_hidden"} t.datetime "last_synced_at" - t.string "host_type" + t.string "host_type", :index=>{:name=>"index_repository_organisations_on_host_type_and_login", :with=>["login"], :unique=>true, :case_sensitive=>false} + + t.index ["host_type", "uuid"], :name=>"index_repository_organisations_on_host_type_and_uuid", :unique=>true end create_table "repository_permissions", force: :cascade do |t| @@ -329,8 +331,8 @@ end create_table "repository_users", force: :cascade do |t| - t.integer "uuid", :index=>{:name=>"index_repository_users_on_uuid", :unique=>true} - t.string "login", :index=>{:name=>"github_users_lower_login", :case_sensitive=>false} + t.string "uuid" + t.string "login" t.string "user_type" t.datetime "created_at", :null=>false, :index=>{:name=>"index_repository_users_on_created_at"} t.datetime "updated_at", :null=>false @@ -344,10 +346,9 @@ t.string "bio" t.integer "followers" t.integer "following" - t.string "host_type" + t.string "host_type", :index=>{:name=>"index_repository_users_on_host_type_and_login", :with=>["login"], :unique=>true, :case_sensitive=>false} - t.index ["login"], :name=>"index_github_users_on_lowercase_login", :unique=>true, :case_sensitive=>false - t.index ["login"], :name=>"index_repository_users_on_login" + t.index ["host_type", "uuid"], :name=>"index_repository_users_on_host_type_and_uuid", :unique=>true end create_table "subscription_plans", force: :cascade do |t| diff --git a/lib/tasks/github.rake b/lib/tasks/github.rake index 07b710533..435913efb 100644 --- a/lib/tasks/github.rake +++ b/lib/tasks/github.rake @@ -50,7 +50,7 @@ namespace :github do u.login = o.login end else - RepositoryUser.create_from_github(o) + RepositoryUser.create_from_host('GitHub', o) end rescue nil diff --git a/spec/workers/repository_create_org_worker_spec.rb b/spec/workers/repository_create_org_worker_spec.rb index 81162c0d2..fb05eacf2 100644 --- a/spec/workers/repository_create_org_worker_spec.rb +++ b/spec/workers/repository_create_org_worker_spec.rb @@ -7,7 +7,8 @@ it "should create from github" do org_login = 'rails' - expect(RepositoryOrganisation).to receive(:create_from_github).with(org_login) - subject.perform(org_login) + host_type = 'GitHub' + expect(RepositoryOwner::Base).to receive(:download_org_from_host).with(host_type, org_login) + subject.perform(host_type, org_login) end end diff --git a/spec/workers/repository_create_user_worker_spec.rb b/spec/workers/repository_create_user_worker_spec.rb new file mode 100644 index 000000000..7e183420f --- /dev/null +++ b/spec/workers/repository_create_user_worker_spec.rb @@ -0,0 +1,14 @@ +require 'rails_helper' + +describe RepositoryCreateUserWorker do + it "should use the low priority queue" do + is_expected.to be_processed_in :owners + end + + it "should create from github" do + org_login = 'andrew' + host_type = 'GitHub' + expect(RepositoryOwner::Base).to receive(:download_user_from_host).with(host_type, org_login) + subject.perform(host_type, org_login) + end +end diff --git a/spec/workers/repository_update_user_worker_spec.rb b/spec/workers/repository_update_user_worker_spec.rb index bd988cf2e..9dbc93d05 100644 --- a/spec/workers/repository_update_user_worker_spec.rb +++ b/spec/workers/repository_update_user_worker_spec.rb @@ -9,6 +9,6 @@ user = create(:repository_user) expect(RepositoryUser).to receive(:find_by_login).with(user.login).and_return(user) expect(user).to receive(:sync) - subject.perform(user.login) + subject.perform(user.host_type, user.login) end end