From 2c588e012ecf62d0380b4a57a09ede77e4e5e438 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 10 Apr 2017 16:42:54 +0100 Subject: [PATCH 01/33] Allowing creation of non-GitHub users as repo owners --- app/models/concerns/github_identity.rb | 2 +- app/models/repository_host/base.rb | 9 ++++ app/models/repository_host/bitbucket.rb | 4 -- app/models/repository_host/github.rb | 18 +------- app/models/repository_host/gitlab.rb | 4 -- app/models/repository_owner/base.rb | 35 +++++++++++++++ app/models/repository_owner/bitbucket.rb | 56 +++++++++++++++++++++++- app/models/repository_owner/github.rb | 40 +++++++++++++++++ app/models/repository_owner/gitlab.rb | 56 ++++++++++++++++++++++++ app/models/repository_user.rb | 45 ++----------------- lib/tasks/github.rake | 2 +- 11 files changed, 201 insertions(+), 70 deletions(-) diff --git a/app/models/concerns/github_identity.rb b/app/models/concerns/github_identity.rb index fcc7ce747..0ad1fd21c 100644 --- a/app/models/concerns/github_identity.rb +++ b/app/models/concerns/github_identity.rb @@ -75,7 +75,7 @@ def update_repo_permissions def download_self return unless github_identity - RepositoryUser.create_from_github(OpenStruct.new({id: github_identity.uid, login: github_identity.nickname, type: 'User', host_type: 'GitHub'})) + RepositoryUser.create_from_host('GitHub', OpenStruct.new({id: github_identity.uid, login: github_identity.nickname, type: 'User', host_type: 'GitHub'})) RepositoryUpdateUserWorker.perform_async(nickname) end diff --git a/app/models/repository_host/base.rb b/app/models/repository_host/base.rb index 027f1a0a0..581cb0d5b 100644 --- a/app/models/repository_host/base.rb +++ b/app/models/repository_host/base.rb @@ -78,6 +78,15 @@ def formatted_host self.class.format(repository.host_type) end + def download_owner + # return if repository.owner && repository.owner.login == repository.owner_name + repository_owner_class.download_user_from_host(repository.host_type, repository.owner_name) + end + + def repository_owner_class + RepositoryOwner.const_get(repository.host_type.capitalize) + end + def update_from_host(token = nil) begin r = self.class.fetch_repo(repository.id_or_name) diff --git a/app/models/repository_host/bitbucket.rb b/app/models/repository_host/bitbucket.rb index 7b4a33a6f..6b375391e 100644 --- a/app/models/repository_host/bitbucket.rb +++ b/app/models/repository_host/bitbucket.rb @@ -60,10 +60,6 @@ def download_forks(token = nil) # not implemented yet end - def download_owner - # not implemented yet - end - def create_webook(token = nil) # not implemented yet end diff --git a/app/models/repository_host/github.rb b/app/models/repository_host/github.rb index da0f4d47b..f81816f34 100644 --- a/app/models/repository_host/github.rb +++ b/app/models/repository_host/github.rb @@ -91,7 +91,7 @@ def download_contributions(token = nil) next unless c['id'] cont = existing_contributions.find{|cnt| cnt.repository_user.try(:uuid) == c.id } unless cont - user = RepositoryUser.create_from_github(c) + user = RepositoryUser.create_from_host('GitHub', c) cont = repository.contributions.find_or_create_by(repository_user: user) end @@ -123,22 +123,6 @@ def download_issues(token = nil) nil end - def download_owner - return if repository.owner && repository.owner.login == repository.owner_name - o = api_client.user(repository.owner_name) - if o.type == "Organization" - go = RepositoryOrganisation.create_from_github(repository.owner_id.to_i) - if go - repository.repository_organisation_id = go.id - repository.save - end - else - RepositoryUser.create_from_github(o) - end - rescue *IGNORABLE_EXCEPTIONS - nil - end - def download_readme(token = nil) contents = { html_body: api_client(token).readme(repository.full_name, accept: 'application/vnd.github.V3.html') diff --git a/app/models/repository_host/gitlab.rb b/app/models/repository_host/gitlab.rb index d4222558f..449ed6722 100644 --- a/app/models/repository_host/gitlab.rb +++ b/app/models/repository_host/gitlab.rb @@ -43,10 +43,6 @@ def download_forks(token = nil) # not implemented yet end - def download_owner - # not implemented yet - end - def create_webook(token = nil) # not implemented yet end diff --git a/app/models/repository_owner/base.rb b/app/models/repository_owner/base.rb index 78f6fbf4e..ec4805a43 100644 --- a/app/models/repository_owner/base.rb +++ b/app/models/repository_owner/base.rb @@ -37,6 +37,41 @@ def github_id owner.uuid end + def self.format(host_type) + case host_type.try(:downcase) + when 'github' + 'GitHub' + when 'gitlab' + 'GitLab' + when 'bitbucket' + 'Bitbucket' + end + end + + def formatted_host + self.class.format(repository.host_type) + end + + def download_user_from_host + download_user_from_host_by(owner.uuid) + end + + def download_user_from_host_by_login + download_user_from_host_by(owner.login) + end + + def download_user_from_host_by(id_or_login) + self.class.download_user_from_host(owner.host_type, id_or_login) + end + + def self.download_user_from_host(host_type, id_or_login) + RepositoryUser.create_from_host(host_type, self.fetch_user(id_or_login)) + end + + def self.fetch_user(id_or_login) + raise NotImplementedError + end + private def top_favourite_project_ids diff --git a/app/models/repository_owner/bitbucket.rb b/app/models/repository_owner/bitbucket.rb index a73a9c93b..dc7e1dc72 100644 --- a/app/models/repository_owner/bitbucket.rb +++ b/app/models/repository_owner/bitbucket.rb @@ -1,11 +1,65 @@ module RepositoryOwner class Bitbucket < Base def avatar_url(size = 60) - "https://bitbucket.org/account/#{owner.login}/avatar/256/?ts=#{owner.uuid}" + "https://bitbucket.org/account/#{owner.login}/avatar/256" end def repository_url "https://bitbucket.org/#{owner.login}" end + + def self.fetch_user(id_or_login) + api_client.get_request "/2.0/users/#{id_or_login}" + rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS + nil + end + + def self.api_client(token = nil) + BitBucket.new oauth_token: token || ENV['BITBUCKET_KEY'] + end + + def api_client(token = nil) + self.class.api_client(token) + end + + def self.create_user(user_hash) + user_hash = user_hash.to_hash.with_indifferent_access + user_hash = { + id: user_hash[:uuid], + login: user_hash[:username], + name: user_hash[:display_name], + blog: user_hash[:website], + location: user_hash[:location], + type: user_hash[:type], + host_type: 'Bitbucket' + } + user = nil + user_by_id = RepositoryUser.host('Bitbucket').find_by_uuid(user_hash[:id]) + user_by_login = RepositoryUser.host('Bitbucket').where("lower(login) = ?", user_hash[:login].try(:downcase)).first + if user_by_id # its fine + if user_by_id.login.try(:downcase) == user_hash[:login].downcase && user_by_id.user_type == user_hash[:type] + user = user_by_id + else + if user_by_login && !user_by_login.download_user_from_host + user_by_login.destroy + end + user_by_id.login = user_hash[:login] + user_by_id.user_type = user_hash[:type] + user_by_id.save! + user = user_by_id + end + elsif user_by_login # conflict + if user_by_login.download_user_from_host_by_login + user = user_by_login if user_by_login.uuid == user_hash[:id] + end + user_by_login.destroy if user.nil? + end + if user.nil? + user = RepositoryUser.create!(uuid: user_hash[:id], login: user_hash[:login], user_type: user_hash[:type], host_type: 'Bitbucket') + end + + user.update(user_hash.slice(:name, :blog, :location)) + user + end end end diff --git a/app/models/repository_owner/github.rb b/app/models/repository_owner/github.rb index ea59cec01..ef72339cd 100644 --- a/app/models/repository_owner/github.rb +++ b/app/models/repository_owner/github.rb @@ -7,5 +7,45 @@ def avatar_url(size = 60) def repository_url "https://github.com/#{owner.login}" end + + def self.fetch_user(id_or_login) + api_client.user(id_or_login) + rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS + nil + end + + def self.api_client(token = nil) + AuthToken.fallback_client(token) + end + + def self.create_user(user_hash) + user_hash = user_hash.to_hash.with_indifferent_access + user = nil + user_by_id = RepositoryUser.host('GitHub').find_by_uuid(user_hash[:id]) + user_by_login = RepositoryUser.host('GitHub').where("lower(login) = ?", user_hash[:login].try(:downcase)).first + if user_by_id # its fine + if user_by_id.login.try(:downcase) == user_hash[:login].downcase && user_by_id.user_type == user_hash[:type] + user = user_by_id + else + if user_by_login && !user_by_login.download_user_from_host + user_by_login.destroy + end + user_by_id.login = user_hash[:login] + user_by_id.user_type = user_hash[:type] + user_by_id.save! + user = user_by_id + end + elsif user_by_login # conflict + if user_by_login.download_user_from_host_by_login + user = user_by_login if user_by_login.uuid == user_hash[:id] + end + user_by_login.destroy if user.nil? + end + if user.nil? + user = RepositoryUser.create!(uuid: user_hash[:id], login: user_hash[:login], user_type: user_hash[:type], host_type: 'GitHub') + end + user.update(user_hash.slice(:name, :company, :blog, :location, :email, :bio)) + user + end end end diff --git a/app/models/repository_owner/gitlab.rb b/app/models/repository_owner/gitlab.rb index 08ba3a0b9..617223ac1 100644 --- a/app/models/repository_owner/gitlab.rb +++ b/app/models/repository_owner/gitlab.rb @@ -7,5 +7,61 @@ def avatar_url(size = 60) def repository_url "https://gitlab.com/#{owner.login}" end + + def self.fetch_user(id_or_login) + api_client.user(id_or_login) + rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS + nil + end + + def self.api_client(token = nil) + ::Gitlab.client(endpoint: 'https://gitlab.com/api/v3', private_token: token || ENV['GITLAB_KEY']) + end + + def api_client(token = nil) + self.class.api_client(token) + end + + def self.create_user(user_hash) + user_hash = user_hash.to_hash.with_indifferent_access + user_hash = { + id: user_hash[:id], + login: user_hash[:username], + name: user_hash[:name], + blog: user_hash[:website_url], + location: user_hash[:location], + bio: user_hash[:bio], + type: 'User', + host_type: 'GitLab' + } + user = nil + user_by_id = RepositoryUser.host('GitLab').find_by_uuid(user_hash[:id]) + user_by_login = RepositoryUser.host('GitLab').where("lower(login) = ?", user_hash[:login].try(:downcase)).first + if user_by_id # its fine + if user_by_id.login.try(:downcase) == user_hash[:login].downcase && user_by_id.user_type == user_hash[:type] + user = user_by_id + else + if user_by_login && !user_by_login.download_user_from_host + user_by_login.destroy + end + user_by_id.login = user_hash[:login] + user_by_id.user_type = user_hash[:type] + user_by_id.save! + user = user_by_id + end + elsif user_by_login # conflict + if user_by_login.download_user_from_host_by_login + user = user_by_login if user_by_login.uuid == user_hash[:id] + end + user_by_login.destroy if user.nil? + end + if user.nil? + user = RepositoryUser.create!(uuid: user_hash[:id], login: user_hash[:login], user_type: user_hash[:type], host_type: 'GitLab') + end + + user.update(user_hash.slice(:name, :blog, :location)) + user + end + end end diff --git a/app/models/repository_user.rb b/app/models/repository_user.rb index e546c92da..24c42c7eb 100644 --- a/app/models/repository_user.rb +++ b/app/models/repository_user.rb @@ -24,7 +24,7 @@ class RepositoryUser < ApplicationRecord scope :host, lambda{ |host_type| where('lower(repository_users.host_type) = ?', host_type.try(:downcase)) } delegate :avatar_url, :repository_url, :top_favourite_projects, :top_contributors, - :to_s, :to_param, :github_id, to: :repository_owner + :to_s, :to_param, :github_id, :download_user_from_host, :download_user_from_host_by_login, to: :repository_owner def repository_owner RepositoryOwner::Gitlab @@ -66,20 +66,6 @@ def sync update_attributes(last_synced_at: Time.now) end - def download_from_github - download_from_github_by(uuid) - end - - def download_from_github_by_login - download_from_github_by(login) - end - - def download_from_github_by(id_or_login) - RepositoryUser.create_from_github(github_client.user(id_or_login)) - rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS - nil - end - def download_orgs github_client.orgs(login).each do |org| RepositoryCreateOrgWorker.perform_async(org.login) @@ -99,32 +85,7 @@ def download_repos nil end - def self.create_from_github(repository_user) - user = nil - user_by_id = RepositoryUser.host('GitHub').find_by_uuid(repository_user.id) - user_by_login = RepositoryUser.host('GitHub').where("lower(login) = ?", repository_user.login.try(:downcase)).first - if user_by_id # its fine - if user_by_id.login.try(:downcase) == repository_user.login.downcase && user_by_id.user_type == repository_user.type - user = user_by_id - else - if user_by_login && !user_by_login.download_from_github - user_by_login.destroy - end - user_by_id.login = repository_user.login - user_by_id.user_type = repository_user.type - user_by_id.save! - user = user_by_id - end - elsif user_by_login # conflict - if user_by_login.download_from_github_by_login - user = user_by_login if user_by_login.uuid == repository_user.id - end - user_by_login.destroy if user.nil? - end - if user.nil? - user = RepositoryUser.create!(uuid: repository_user.id, login: repository_user.login, user_type: repository_user.type, host_type: 'GitHub') - end - user.update(repository_user.to_hash.slice(:name, :company, :blog, :location, :email, :bio)) - user + def self.create_from_host(host_type, user_hash) + RepositoryOwner.const_get(host_type.capitalize).create_user(user_hash) end end diff --git a/lib/tasks/github.rake b/lib/tasks/github.rake index f9c5e52c0..0b788e55b 100644 --- a/lib/tasks/github.rake +++ b/lib/tasks/github.rake @@ -45,7 +45,7 @@ namespace :github do u.login = o.login end else - RepositoryUser.create_from_github(o) + RepositoryUser.create_from_host('GitHub', o) end rescue nil From 4a99e5ebfb6155fdf9428e19d6271f6b5b7a34e5 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Tue, 11 Apr 2017 14:35:54 +0100 Subject: [PATCH 02/33] Add schema_plus for better index configuration --- Gemfile | 1 + Gemfile.lock | 23 ++++++++++++- ...0170411103819_add_host_to_owner_indexes.rb | 34 +++++++++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20170411103819_add_host_to_owner_indexes.rb diff --git a/Gemfile b/Gemfile index ccfa0b759..5ae763f7b 100644 --- a/Gemfile +++ b/Gemfile @@ -82,6 +82,7 @@ gem 'github-linguist' gem 'appsignal' gem 'rack-cors', :require => 'rack/cors' gem 'active_model_serializers' +gem 'schema_plus_pg_indexes' group :development do gem 'spring' diff --git a/Gemfile.lock b/Gemfile.lock index 34c1f230b..6865f0bda 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -257,18 +257,20 @@ GEM indefinite_article (0.2.4) activesupport ipaddress (0.8.3) + its-it (1.3.0) jquery-rails (4.3.1) rails-dom-testing (>= 1, < 3) railties (>= 4.2.0) thor (>= 0.14, < 2.0) js_cookie_rails (2.1.4) railties (>= 3.1) - json (2.0.3) + json (2.0.4) json_spec (1.1.4) multi_json (~> 1.0) rspec (>= 2.0, < 4.0) jsonapi-renderer (0.1.2) jwt (1.5.6) + key_struct (0.4.2) librariesio-gem-parser (1.0.0) license-compatibility (2.0.0) listen (3.1.5) @@ -289,6 +291,8 @@ GEM mime-types-data (3.2016.0521) mini_portile2 (2.1.0) minitest (5.10.1) + modware (0.1.3) + key_struct (~> 0.4) multi_json (1.12.1) multi_xml (0.6.0) multipart-post (2.0.0) @@ -443,6 +447,22 @@ GEM sawyer (0.8.1) addressable (>= 2.3.5, < 2.6) faraday (~> 0.8, < 1.0) + schema_monkey (2.1.5) + activerecord (>= 4.2) + modware (~> 0.1) + schema_plus_core (2.0.1) + activerecord (~> 5.0) + its-it (~> 1.2) + schema_monkey (~> 2.1) + schema_plus_indexes (0.2.4) + activerecord (>= 4.2, < 5.1) + its-it (~> 1.2) + schema_plus_core + schema_plus_pg_indexes (0.2.1) + activerecord (~> 5.0, >= 5.0.1) + its-it (~> 1.2) + schema_plus_core (~> 2.0) + schema_plus_indexes (~> 0.2, >= 0.2.4) sdl4r (0.9.11) semantic (1.6.0) semantic_range (2.0.0) @@ -614,6 +634,7 @@ DEPENDENCIES rspec_junit_formatter sanitize-url sassc-rails + schema_plus_pg_indexes semantic semantic_range shoulda diff --git a/db/migrate/20170411103819_add_host_to_owner_indexes.rb b/db/migrate/20170411103819_add_host_to_owner_indexes.rb new file mode 100644 index 000000000..17b2a62a0 --- /dev/null +++ b/db/migrate/20170411103819_add_host_to_owner_indexes.rb @@ -0,0 +1,34 @@ +class AddHostToOwnerIndexes < ActiveRecord::Migration[5.0] + def change + # drop indexes + + # if index_exists?(:books, :created_at) + remove_index :repository_users, :login + remove_index :repository_users, :uuid + + # users + # t.index "lower((login)::text)", name: "github_users_lower_login", using: :btree + # t.index "lower((login)::text)", name: "index_github_users_on_lowercase_login", unique: true, using: :btree + # t.index ["login"], name: "index_repository_users_on_login", using: :btree + # t.index ["uuid"], name: "index_repository_users_on_uuid", unique: true, using: :btree + + # orgs + # t.index "lower((login)::text)", name: "index_github_organisations_on_lowercase_login", unique: true, using: :btree + # t.index ["uuid"], name: "index_repository_organisations_on_uuid", unique: true, using: :btree + + # change uuid to string + + change_column :repository_users, :uuid, :string + change_column :repository_organisations, :uuid, :string + + # recreate indexes + + # users + add_index :repository_organisations, [:host_type, :login], unique: true, index: { case_sensitive: false } + add_index :repository_users, [:host_type, :uuid] + + # orgs + add_index :repository_organisations, [:host_type, :login], unique: true, index: { case_sensitive: false } + add_index :repository_organisations, [:host_type, :uuid], unique: true + end +end From a801f483f30f3c6f44975f2e0efdda21b06125bc Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Wed, 12 Apr 2017 16:19:07 +0100 Subject: [PATCH 03/33] Update repo owner indexes --- ...0170411103819_add_host_to_owner_indexes.rb | 32 +-- db/schema.rb | 270 +++++++----------- 2 files changed, 120 insertions(+), 182 deletions(-) diff --git a/db/migrate/20170411103819_add_host_to_owner_indexes.rb b/db/migrate/20170411103819_add_host_to_owner_indexes.rb index 17b2a62a0..0b7a0c58b 100644 --- a/db/migrate/20170411103819_add_host_to_owner_indexes.rb +++ b/db/migrate/20170411103819_add_host_to_owner_indexes.rb @@ -1,34 +1,22 @@ class AddHostToOwnerIndexes < ActiveRecord::Migration[5.0] def change # drop indexes - - # if index_exists?(:books, :created_at) - remove_index :repository_users, :login - remove_index :repository_users, :uuid - - # users - # t.index "lower((login)::text)", name: "github_users_lower_login", using: :btree - # t.index "lower((login)::text)", name: "index_github_users_on_lowercase_login", unique: true, using: :btree - # t.index ["login"], name: "index_repository_users_on_login", using: :btree - # t.index ["uuid"], name: "index_repository_users_on_uuid", unique: true, using: :btree - - # orgs - # t.index "lower((login)::text)", name: "index_github_organisations_on_lowercase_login", unique: true, using: :btree - # t.index ["uuid"], name: "index_repository_organisations_on_uuid", unique: true, using: :btree + remove_index :repository_users, name: :index_repository_users_on_login + remove_index :repository_users, :uuid + remove_index :repository_users, name: :github_users_lower_login + remove_index :repository_users, name: :index_github_users_on_lowercase_login + remove_index :repository_organisations, :uuid + remove_index :repository_organisations, name: :index_github_organisations_on_lowercase_login # change uuid to string - change_column :repository_users, :uuid, :string change_column :repository_organisations, :uuid, :string # recreate indexes + add_index :repository_users, [:host_type, :login], unique: true, case_sensitive: false + add_index :repository_users, [:host_type, :uuid], unique: true - # users - add_index :repository_organisations, [:host_type, :login], unique: true, index: { case_sensitive: false } - add_index :repository_users, [:host_type, :uuid] - - # orgs - add_index :repository_organisations, [:host_type, :login], unique: true, index: { case_sensitive: false } - add_index :repository_organisations, [:host_type, :uuid], unique: true + add_index :repository_organisations, [:host_type, :login], unique: true, case_sensitive: false + add_index :repository_organisations, [:host_type, :uuid], unique: true end end diff --git a/db/schema.rb b/db/schema.rb index 3c0672ddc..2aaaebbf7 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,69 +10,62 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170411164230) do +ActiveRecord::Schema.define(version: 20170412143657) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" enable_extension "pg_stat_statements" create_table "api_keys", force: :cascade do |t| - t.string "access_token" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.integer "user_id" + t.string "access_token", :index=>{:name=>"index_api_keys_on_access_token"} + t.datetime "created_at", :null=>false + t.datetime "updated_at", :null=>false + t.integer "user_id", :index=>{:name=>"index_api_keys_on_user_id"} t.datetime "deleted_at" - t.integer "rate_limit", default: 60 - t.index ["access_token"], name: "index_api_keys_on_access_token", using: :btree - t.index ["user_id"], name: "index_api_keys_on_user_id", using: :btree + t.integer "rate_limit", :default=>60 end create_table "auth_tokens", force: :cascade do |t| t.string "login" t.string "token" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at", :null=>false + t.datetime "updated_at", :null=>false end create_table "contributions", force: :cascade do |t| - t.integer "repository_id" - t.integer "repository_user_id" + t.integer "repository_id", :index=>{:name=>"index_contributions_on_repository_id_and_user_id", :with=>["repository_user_id"]} + t.integer "repository_user_id", :index=>{:name=>"index_contributions_on_repository_user_id"} t.integer "count" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at", :null=>false + t.datetime "updated_at", :null=>false t.string "platform" - t.index ["repository_id", "repository_user_id"], name: "index_contributions_on_repository_id_and_user_id", using: :btree - t.index ["repository_user_id"], name: "index_contributions_on_repository_user_id", using: :btree end create_table "dependencies", force: :cascade do |t| - t.integer "version_id" - t.integer "project_id" + t.integer "version_id", :index=>{:name=>"index_dependencies_on_version_id"} + t.integer "project_id", :index=>{:name=>"index_dependencies_on_project_id"} t.string "project_name" t.string "platform" t.string "kind" - t.boolean "optional", default: false + t.boolean "optional", :default=>false t.string "requirements" t.datetime "created_at" t.datetime "updated_at" - t.index ["project_id"], name: "index_dependencies_on_project_id", using: :btree - t.index ["version_id"], name: "index_dependencies_on_version_id", using: :btree end create_table "identities", force: :cascade do |t| - t.string "uid" + t.string "uid", :index=>{:name=>"index_identities_on_uid"} t.string "provider" - t.integer "user_id" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.integer "user_id", :index=>{:name=>"index_identities_on_user_id"} + t.datetime "created_at", :null=>false + t.datetime "updated_at", :null=>false t.string "token" t.string "nickname" t.string "avatar_url" - t.index ["user_id"], name: "index_identities_on_user_id", using: :btree end create_table "issues", force: :cascade do |t| - t.integer "repository_id" + t.integer "repository_id", :index=>{:name=>"index_issues_on_repository_id"} t.integer "github_id" t.integer "number" t.string "state" @@ -82,25 +75,22 @@ t.boolean "locked" t.integer "comments_count" t.datetime "closed_at" - t.string "labels", default: [], array: true - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.string "labels", :default=>[], :array=>true + t.datetime "created_at", :null=>false + t.datetime "updated_at", :null=>false t.datetime "last_synced_at" t.boolean "pull_request" - t.index ["repository_id"], name: "index_issues_on_repository_id", using: :btree end create_table "manifests", force: :cascade do |t| - t.integer "repository_id" + t.integer "repository_id", :index=>{:name=>"index_manifests_on_repository_id"} t.string "platform" t.string "filepath" t.string "sha" t.string "branch" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at", :null=>false, :index=>{:name=>"index_manifests_on_created_at"} + t.datetime "updated_at", :null=>false t.string "kind" - t.index ["created_at"], name: "index_manifests_on_created_at", using: :btree - t.index ["repository_id"], name: "index_manifests_on_repository_id", using: :btree end create_table "payola_affiliates", force: :cascade do |t| @@ -116,14 +106,14 @@ t.integer "percent_off" t.datetime "created_at" t.datetime "updated_at" - t.boolean "active", default: true + t.boolean "active", :default=>true end create_table "payola_sales", force: :cascade do |t| - t.string "email", limit: 191 - t.string "guid", limit: 191 - t.integer "product_id" - t.string "product_type", limit: 100 + t.string "email", :limit=>191, :index=>{:name=>"index_payola_sales_on_email"} + t.string "guid", :limit=>191, :index=>{:name=>"index_payola_sales_on_guid"} + t.integer "product_id", :index=>{:name=>"index_payola_sales_on_product", :with=>["product_type"]} + t.string "product_type", :limit=>100 t.datetime "created_at" t.datetime "updated_at" t.string "state" @@ -135,23 +125,17 @@ t.text "error" t.integer "amount" t.integer "fee_amount" - t.integer "coupon_id" + t.integer "coupon_id", :index=>{:name=>"index_payola_sales_on_coupon_id"} t.boolean "opt_in" t.integer "download_count" t.integer "affiliate_id" t.text "customer_address" t.text "business_address" - t.string "stripe_customer_id", limit: 191 + t.string "stripe_customer_id", :limit=>191, :index=>{:name=>"index_payola_sales_on_stripe_customer_id"} t.string "currency" t.text "signed_custom_fields" - t.integer "owner_id" - t.string "owner_type", limit: 100 - t.index ["coupon_id"], name: "index_payola_sales_on_coupon_id", using: :btree - t.index ["email"], name: "index_payola_sales_on_email", using: :btree - t.index ["guid"], name: "index_payola_sales_on_guid", using: :btree - t.index ["owner_id", "owner_type"], name: "index_payola_sales_on_owner_id_and_owner_type", using: :btree - t.index ["product_id", "product_type"], name: "index_payola_sales_on_product", using: :btree - t.index ["stripe_customer_id"], name: "index_payola_sales_on_stripe_customer_id", using: :btree + t.integer "owner_id", :index=>{:name=>"index_payola_sales_on_owner_id_and_owner_type", :with=>["owner_type"]} + t.string "owner_type", :limit=>100 end create_table "payola_stripe_webhooks", force: :cascade do |t| @@ -188,7 +172,7 @@ t.datetime "updated_at" t.string "currency" t.integer "amount" - t.string "guid", limit: 191 + t.string "guid", :limit=>191, :index=>{:name=>"index_payola_subscriptions_on_guid"} t.string "stripe_status" t.integer "affiliate_id" t.string "coupon" @@ -196,16 +180,14 @@ t.text "customer_address" t.text "business_address" t.integer "setup_fee" - t.decimal "tax_percent", precision: 4, scale: 2 - t.index ["guid"], name: "index_payola_subscriptions_on_guid", using: :btree + t.decimal "tax_percent", :precision=>4, :scale=>2 end create_table "project_mutes", force: :cascade do |t| - t.integer "user_id", null: false - t.integer "project_id", null: false - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.index ["project_id", "user_id"], name: "index_project_mutes_on_project_id_and_user_id", unique: true, using: :btree + t.integer "user_id", :null=>false + t.integer "project_id", :null=>false, :index=>{:name=>"index_project_mutes_on_project_id_and_user_id", :with=>["user_id"], :unique=>true} + t.datetime "created_at", :null=>false + t.datetime "updated_at", :null=>false end create_table "project_suggestions", force: :cascade do |t| @@ -214,65 +196,57 @@ t.string "licenses" t.string "repository_url" t.text "notes" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at", :null=>false + t.datetime "updated_at", :null=>false t.string "status" end create_table "projects", force: :cascade do |t| - t.string "name" - t.string "platform" - t.datetime "created_at" - t.datetime "updated_at" + t.string "name", :index=>{:name=>"projects_lower_name", :case_sensitive=>false} + t.string "platform", :index=>{:name=>"projects_lower_platform", :case_sensitive=>false} + t.datetime "created_at", :index=>{:name=>"index_projects_on_created_at"} + t.datetime "updated_at", :index=>{:name=>"index_projects_on_updated_at"} t.text "description" t.text "keywords" t.string "homepage" t.string "licenses" t.string "repository_url" - t.integer "repository_id" - t.string "normalized_licenses", default: [], array: true - t.integer "versions_count", default: 0, null: false - t.integer "rank", default: 0 + t.integer "repository_id", :index=>{:name=>"index_projects_on_repository_id"} + t.string "normalized_licenses", :default=>[], :array=>true + t.integer "versions_count", :default=>0, :null=>false, :index=>{:name=>"index_projects_on_versions_count"} + t.integer "rank", :default=>0 t.datetime "latest_release_published_at" t.string "latest_release_number" t.integer "pm_id" - t.string "keywords_array", default: [], array: true - t.integer "dependents_count", default: 0, null: false + t.string "keywords_array", :default=>[], :array=>true, :index=>{:name=>"index_projects_on_keywords_array", :using=>:gin} + t.integer "dependents_count", :default=>0, :null=>false, :index=>{:name=>"index_projects_on_dependents_count"} t.string "language" t.string "status" t.datetime "last_synced_at" t.integer "dependent_repos_count" - t.index "lower((name)::text)", name: "projects_lower_name", using: :btree - t.index "lower((platform)::text)", name: "projects_lower_platform", using: :btree - t.index ["created_at"], name: "index_projects_on_created_at", using: :btree - t.index ["dependents_count"], name: "index_projects_on_dependents_count", using: :btree - t.index ["keywords_array"], name: "index_projects_on_keywords_array", using: :gin - t.index ["name", "platform"], name: "index_projects_on_name_and_platform", using: :btree - t.index ["repository_id"], name: "index_projects_on_repository_id", using: :btree - t.index ["updated_at"], name: "index_projects_on_updated_at", using: :btree - t.index ["versions_count"], name: "index_projects_on_versions_count", using: :btree + + t.index ["name", "platform"], :name=>"index_projects_on_name_and_platform" end create_table "readmes", force: :cascade do |t| - t.integer "repository_id" + t.integer "repository_id", :index=>{:name=>"index_readmes_on_repository_id", :unique=>true} t.text "html_body" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.index ["repository_id"], name: "index_readmes_on_repository_id", unique: true, using: :btree + t.datetime "created_at", :null=>false + t.datetime "updated_at", :null=>false end create_table "repositories", force: :cascade do |t| - t.string "full_name" - t.integer "owner_id" + t.string "full_name", :index=>{:name=>"index_github_repositories_on_lowercase_full_name", :unique=>true, :case_sensitive=>false} + t.integer "owner_id", :index=>{:name=>"index_repositories_on_owner_id"} t.string "description" t.boolean "fork" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at", :null=>false + t.datetime "updated_at", :null=>false t.datetime "pushed_at" t.string "homepage" t.integer "size" t.integer "stargazers_count" - t.string "language" + t.string "language", :index=>{:name=>"github_repositories_lower_language", :case_sensitive=>false} t.boolean "has_issues" t.boolean "has_wiki" t.boolean "has_pages" @@ -281,12 +255,12 @@ t.integer "open_issues_count" t.string "default_branch" t.integer "subscribers_count" - t.string "uuid" - t.string "source_name" + t.string "uuid", :index=>{:name=>"index_repositories_on_uuid", :unique=>true} + t.string "source_name", :index=>{:name=>"index_repositories_on_source_name"} t.string "license" - t.integer "repository_organisation_id" + t.integer "repository_organisation_id", :index=>{:name=>"index_repositories_on_repository_organisation_id"} t.boolean "private" - t.integer "contributions_count", default: 0, null: false + t.integer "contributions_count", :default=>0, :null=>false t.string "has_readme" t.string "has_changelog" t.string "has_contributing" @@ -294,7 +268,7 @@ t.string "has_coc" t.string "has_threat_model" t.string "has_audit" - t.string "status" + t.string "status", :index=>{:name=>"index_repositories_on_status"} t.datetime "last_synced_at" t.integer "rank" t.string "host_type" @@ -304,92 +278,75 @@ t.string "fork_policy" t.string "pull_requests_enabled" t.string "logo_url" - t.index "lower((full_name)::text)", name: "index_github_repositories_on_lowercase_full_name", unique: true, using: :btree - t.index "lower((language)::text)", name: "github_repositories_lower_language", using: :btree - t.index ["owner_id"], name: "index_repositories_on_owner_id", using: :btree - t.index ["repository_organisation_id"], name: "index_repositories_on_repository_organisation_id", using: :btree - t.index ["source_name"], name: "index_repositories_on_source_name", using: :btree - t.index ["status"], name: "index_repositories_on_status", using: :btree - t.index ["uuid"], name: "index_repositories_on_uuid", unique: true, using: :btree end create_table "repository_dependencies", force: :cascade do |t| - t.integer "project_id" - t.integer "manifest_id" + t.integer "project_id", :index=>{:name=>"index_repository_dependencies_on_project_id"} + t.integer "manifest_id", :index=>{:name=>"index_repository_dependencies_on_manifest_id"} t.boolean "optional" t.string "project_name" t.string "platform" t.string "requirements" t.string "kind" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.index ["manifest_id"], name: "index_repository_dependencies_on_manifest_id", using: :btree - t.index ["project_id"], name: "index_repository_dependencies_on_project_id", using: :btree + t.datetime "created_at", :null=>false + t.datetime "updated_at", :null=>false end create_table "repository_organisations", force: :cascade do |t| t.string "login" - t.integer "uuid" + t.string "uuid" t.string "name" t.string "blog" t.string "email" t.string "location" t.string "bio" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.boolean "hidden", default: false + t.datetime "created_at", :null=>false, :index=>{:name=>"index_repository_organisations_on_created_at"} + t.datetime "updated_at", :null=>false + t.boolean "hidden", :default=>false, :index=>{:name=>"index_repository_organisations_on_hidden"} t.datetime "last_synced_at" - t.string "host_type" - t.index "lower((login)::text)", name: "index_github_organisations_on_lowercase_login", unique: true, using: :btree - t.index ["created_at"], name: "index_repository_organisations_on_created_at", using: :btree - t.index ["hidden"], name: "index_repository_organisations_on_hidden", using: :btree - t.index ["uuid"], name: "index_repository_organisations_on_uuid", unique: true, using: :btree + t.string "host_type", :index=>{:name=>"index_repository_organisations_on_host_type_and_login", :with=>["login"], :unique=>true, :case_sensitive=>false} + + t.index ["host_type", "uuid"], :name=>"index_repository_organisations_on_host_type_and_uuid", :unique=>true end create_table "repository_permissions", force: :cascade do |t| - t.integer "user_id" + t.integer "user_id", :index=>{:name=>"user_repo_unique_repository_permissions", :with=>["repository_id"], :unique=>true} t.integer "repository_id" t.boolean "admin" t.boolean "push" t.boolean "pull" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.index ["user_id", "repository_id"], name: "user_repo_unique_repository_permissions", unique: true, using: :btree + t.datetime "created_at", :null=>false + t.datetime "updated_at", :null=>false end create_table "repository_subscriptions", force: :cascade do |t| t.integer "repository_id" t.integer "user_id" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at", :null=>false, :index=>{:name=>"index_repository_subscriptions_on_created_at"} + t.datetime "updated_at", :null=>false t.integer "hook_id" - t.boolean "include_prerelease", default: true - t.index ["created_at"], name: "index_repository_subscriptions_on_created_at", using: :btree + t.boolean "include_prerelease", :default=>true end create_table "repository_users", force: :cascade do |t| - t.integer "uuid" + t.string "uuid" t.string "login" t.string "user_type" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at", :null=>false, :index=>{:name=>"index_repository_users_on_created_at"} + t.datetime "updated_at", :null=>false t.string "name" t.string "company" t.string "blog" t.string "location" - t.boolean "hidden", default: false + t.boolean "hidden", :default=>false, :index=>{:name=>"index_repository_users_on_hidden"} t.datetime "last_synced_at" t.string "email" t.string "bio" t.integer "followers" t.integer "following" - t.string "host_type" - t.index "lower((login)::text)", name: "github_users_lower_login", using: :btree - t.index "lower((login)::text)", name: "index_github_users_on_lowercase_login", unique: true, using: :btree - t.index ["created_at"], name: "index_repository_users_on_created_at", using: :btree - t.index ["hidden"], name: "index_repository_users_on_hidden", using: :btree - t.index ["login"], name: "index_repository_users_on_login", using: :btree - t.index ["uuid"], name: "index_repository_users_on_uuid", unique: true, using: :btree + t.string "host_type", :index=>{:name=>"index_repository_users_on_host_type_and_login", :with=>["login"], :unique=>true, :case_sensitive=>false} + + t.index ["host_type", "uuid"], :name=>"index_repository_users_on_host_type_and_uuid", :unique=>true end create_table "subscription_plans", force: :cascade do |t| @@ -397,63 +354,56 @@ t.string "interval" t.string "stripe_id" t.string "name" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at", :null=>false + t.datetime "updated_at", :null=>false t.integer "repo_count" - t.boolean "hidden", default: false + t.boolean "hidden", :default=>false end create_table "subscriptions", force: :cascade do |t| - t.integer "project_id" - t.integer "user_id" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.integer "project_id", :index=>{:name=>"index_subscriptions_on_project_id"} + t.integer "user_id", :index=>{:name=>"index_subscriptions_on_user_id_and_project_id", :with=>["project_id"]} + t.datetime "created_at", :null=>false, :index=>{:name=>"index_subscriptions_on_created_at"} + t.datetime "updated_at", :null=>false t.integer "repository_subscription_id" - t.boolean "include_prerelease", default: true - t.index ["created_at"], name: "index_subscriptions_on_created_at", using: :btree - t.index ["project_id"], name: "index_subscriptions_on_project_id", using: :btree - t.index ["user_id", "project_id"], name: "index_subscriptions_on_user_id_and_project_id", using: :btree + t.boolean "include_prerelease", :default=>true end create_table "tags", force: :cascade do |t| - t.integer "repository_id" + t.integer "repository_id", :index=>{:name=>"index_tags_on_repository_id_and_name", :with=>["name"]} t.string "name" t.string "sha" t.string "kind" t.datetime "published_at" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.index ["repository_id", "name"], name: "index_tags_on_repository_id_and_name", using: :btree + t.datetime "created_at", :null=>false + t.datetime "updated_at", :null=>false end create_table "users", force: :cascade do |t| t.string "email" - t.datetime "created_at" + t.datetime "created_at", :index=>{:name=>"index_users_on_created_at"} t.datetime "updated_at" - t.boolean "currently_syncing", default: false, null: false + t.boolean "currently_syncing", :default=>false, :null=>false t.datetime "last_synced_at" - t.boolean "emails_enabled", default: true - t.index ["created_at"], name: "index_users_on_created_at", using: :btree + t.boolean "emails_enabled", :default=>true end create_table "versions", force: :cascade do |t| - t.integer "project_id" + t.integer "project_id", :index=>{:name=>"index_versions_on_project_id_and_number", :with=>["number"], :unique=>true} t.string "number" t.datetime "published_at" t.datetime "created_at" t.datetime "updated_at" - t.index ["project_id", "number"], name: "index_versions_on_project_id_and_number", unique: true, using: :btree end create_table "web_hooks", force: :cascade do |t| - t.integer "repository_id" + t.integer "repository_id", :index=>{:name=>"index_web_hooks_on_repository_id"} t.integer "user_id" t.string "url" t.string "last_response" t.datetime "last_sent_at" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.index ["repository_id"], name: "index_web_hooks_on_repository_id", using: :btree + t.datetime "created_at", :null=>false + t.datetime "updated_at", :null=>false end end From 2aa23fbefbcd41a01cda9b822fbbd8bae77916e9 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Tue, 18 Apr 2017 12:31:24 +0100 Subject: [PATCH 04/33] Remove duplicate gem --- Gemfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Gemfile b/Gemfile index 02fd0c367..3b29f7a75 100644 --- a/Gemfile +++ b/Gemfile @@ -82,7 +82,6 @@ gem 'github-linguist' gem 'appsignal' gem 'rack-cors', :require => 'rack/cors' gem 'active_model_serializers' -gem 'schema_plus_pg_indexes' gem 'pghero' gem 'pg_query' gem 'schema_plus_pg_indexes' From 3b4de0c9d5442fdc7cb57bf7bcb45c4f7392997a Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 12 May 2017 10:32:44 +0100 Subject: [PATCH 05/33] Fix downloading self --- app/models/concerns/github_identity.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/models/concerns/github_identity.rb b/app/models/concerns/github_identity.rb index 0ad1fd21c..1b1374250 100644 --- a/app/models/concerns/github_identity.rb +++ b/app/models/concerns/github_identity.rb @@ -75,8 +75,11 @@ def update_repo_permissions def download_self return unless github_identity - RepositoryUser.create_from_host('GitHub', OpenStruct.new({id: github_identity.uid, login: github_identity.nickname, type: 'User', host_type: 'GitHub'})) - RepositoryUpdateUserWorker.perform_async(nickname) + repository_user = RepositoryUser.create_from_host('GitHub', {id: github_identity.uid, login: github_identity.nickname, type: 'User', host_type: 'GitHub'}) + if repository_user + github_identity.update_column(:repository_user_id, repository_user.id) + RepositoryUpdateUserWorker.perform_async(nickname) + end end def download_orgs From f9c86c42fb047fd5f9aa239317cedab7c6e535ad Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 12 May 2017 10:43:17 +0100 Subject: [PATCH 06/33] Fix download_owner for user owned github repos --- app/models/repository_host/github.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/repository_host/github.rb b/app/models/repository_host/github.rb index 944852ea3..0cff1f7d9 100644 --- a/app/models/repository_host/github.rb +++ b/app/models/repository_host/github.rb @@ -134,7 +134,7 @@ def download_owner repository.save end else - u = RepositoryUser.create_from_github(o) + u = RepositoryUser.create_from_host('GitHub', o) if u repository.repository_user_id = u.id repository.repository_organisation_id = nil From 3b12d11e0ddc13c15de3f6e34dafa00f0b5a7686 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 12 May 2017 10:45:39 +0100 Subject: [PATCH 07/33] Ensure orgs have a uuid --- app/models/repository_organisation.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/models/repository_organisation.rb b/app/models/repository_organisation.rb index 3bbba0a00..fa5fe9bd6 100644 --- a/app/models/repository_organisation.rb +++ b/app/models/repository_organisation.rb @@ -10,6 +10,7 @@ class RepositoryOrganisation < ApplicationRecord has_many :contributors, -> { group('repository_users.id').order("sum(contributions.count) DESC") }, through: :open_source_repositories, source: :contributors has_many :projects, through: :open_source_repositories + validates :uuid, presence: true validates :login, uniqueness: {scope: :host_type}, if: lambda { self.login_changed? } validates :uuid, uniqueness: {scope: :host_type}, if: lambda { self.uuid_changed? } From 889d134b507b9998972c483370f7f3d08a6d3a25 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 12 May 2017 10:45:50 +0100 Subject: [PATCH 08/33] TODOs --- app/models/repository_host/base.rb | 7 ------- app/models/repository_host/bitbucket.rb | 4 ++++ app/models/repository_host/gitlab.rb | 4 ++++ 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/app/models/repository_host/base.rb b/app/models/repository_host/base.rb index 650363c1b..28733cdba 100644 --- a/app/models/repository_host/base.rb +++ b/app/models/repository_host/base.rb @@ -78,13 +78,6 @@ def formatted_host self.class.format(repository.host_type) end - def download_owner - # need to check if owner is org or user - repository_owner_class.download_user_from_host(repository.host_type, repository.owner_name) - # need to save owner id to correct field after creation - end - - def repository_owner_class RepositoryOwner.const_get(repository.host_type.capitalize) end diff --git a/app/models/repository_host/bitbucket.rb b/app/models/repository_host/bitbucket.rb index 395fd21e7..aea475ae5 100644 --- a/app/models/repository_host/bitbucket.rb +++ b/app/models/repository_host/bitbucket.rb @@ -61,6 +61,10 @@ def download_forks(token = nil) # not implemented yet end + def download_owner + # TODO + end + def create_webook(token = nil) # not implemented yet end diff --git a/app/models/repository_host/gitlab.rb b/app/models/repository_host/gitlab.rb index 449ed6722..15fd0b79c 100644 --- a/app/models/repository_host/gitlab.rb +++ b/app/models/repository_host/gitlab.rb @@ -43,6 +43,10 @@ def download_forks(token = nil) # not implemented yet end + def download_owner + # TODO + end + def create_webook(token = nil) # not implemented yet end From e5d1d385e4768377a096665790bab38fe7da8bef Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 12 May 2017 11:06:37 +0100 Subject: [PATCH 09/33] Eager load RepositoryHost::Gitlab in development --- app/models/repository.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/models/repository.rb b/app/models/repository.rb index c7b388942..fef1c5978 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -4,6 +4,9 @@ class Repository < ApplicationRecord include RepoManifests include RepositorySourceRank + # eager load this module to avoid clashing with Gitlab gem in development + RepositoryHost::Gitlab + STATUSES = ['Active', 'Deprecated', 'Unmaintained', 'Help Wanted', 'Removed'] API_FIELDS = [:full_name, :description, :fork, :created_at, :updated_at, :pushed_at, :homepage, @@ -302,7 +305,6 @@ def github_id end def repository_host - RepositoryHost::Gitlab @repository_host ||= RepositoryHost.const_get(host_type.capitalize).new(self) end end From e96ddb773ea72d0d7c2e4b9431591a322c961a6a Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 12 May 2017 11:06:57 +0100 Subject: [PATCH 10/33] Fix downloading of GitLab repos --- app/models/repository_host/gitlab.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/repository_host/gitlab.rb b/app/models/repository_host/gitlab.rb index 15fd0b79c..4b372998f 100644 --- a/app/models/repository_host/gitlab.rb +++ b/app/models/repository_host/gitlab.rb @@ -143,7 +143,7 @@ def api_client(token = nil) def self.fetch_repo(full_name, token = nil) client = api_client(token) - project = client.project(full_name.gsub('/','%2F')) + project = client.project(full_name) repo_hash = project.to_hash.with_indifferent_access.slice(:id, :description, :created_at, :name, :open_issues_count, :forks_count, :default_branch) repo_hash.merge!({ From b7a4ae3199c5bd12fc522b17cf84b82fbb75f50e Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 12 May 2017 11:44:17 +0100 Subject: [PATCH 11/33] Fixed syncing repository users --- app/models/repository_user.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/repository_user.rb b/app/models/repository_user.rb index c13c44abb..b0a7da54b 100644 --- a/app/models/repository_user.rb +++ b/app/models/repository_user.rb @@ -61,7 +61,7 @@ def async_sync end def sync - download_from_github + download_user_from_host download_orgs download_repos update_attributes(last_synced_at: Time.now) From 675930674dc0a80d1133e33cf4bbe50d2b58150f Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 12 May 2017 11:44:51 +0100 Subject: [PATCH 12/33] Downloading owner for gitlab repos --- app/models/repository_host/gitlab.rb | 20 +++++++++++++++++++- app/models/repository_owner/gitlab.rb | 6 +++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/app/models/repository_host/gitlab.rb b/app/models/repository_host/gitlab.rb index 4b372998f..fb385d849 100644 --- a/app/models/repository_host/gitlab.rb +++ b/app/models/repository_host/gitlab.rb @@ -44,7 +44,25 @@ def download_forks(token = nil) end def download_owner - # TODO + return if repository.owner && repository.repository_user_id && repository.owner.login == repository.owner_name + o = RepositoryOwner::Gitlab.fetch_user(repository.owner_name) + if o.type == "Organization" + go = RepositoryOrganisation.create_from_gitlab(o.id) + if go + repository.repository_organisation_id = go.id + repository.repository_user_id = nil + repository.save + end + else + u = RepositoryUser.create_from_host('GitLab', o) + if u + repository.repository_user_id = u.id + repository.repository_organisation_id = nil + repository.save + end + end + rescue *IGNORABLE_EXCEPTIONS + nil end def create_webook(token = nil) diff --git a/app/models/repository_owner/gitlab.rb b/app/models/repository_owner/gitlab.rb index 617223ac1..5eed2b98c 100644 --- a/app/models/repository_owner/gitlab.rb +++ b/app/models/repository_owner/gitlab.rb @@ -9,7 +9,11 @@ def repository_url end def self.fetch_user(id_or_login) - api_client.user(id_or_login) + if id_or_login.to_i.zero? + api_client.get(("/users?username=#{id_or_login}")).first + else + api_client.user(id_or_login) + end rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS nil end From a8cce6354946e70d087ff1128e55f0d624e2ee27 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 12 May 2017 11:48:43 +0100 Subject: [PATCH 13/33] Download owner for bitbucket repos --- app/models/repository_host/bitbucket.rb | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/app/models/repository_host/bitbucket.rb b/app/models/repository_host/bitbucket.rb index aea475ae5..fbf220c9b 100644 --- a/app/models/repository_host/bitbucket.rb +++ b/app/models/repository_host/bitbucket.rb @@ -62,7 +62,25 @@ def download_forks(token = nil) end def download_owner - # TODO + return if repository.owner && repository.repository_user_id && repository.owner.login == repository.owner_name + o = RepositoryOwner::Bitbucket.fetch_user(repository.owner_name) + if o.type == "Organization" + go = RepositoryOrganisation.create_from_gitlab(o.id) + if go + repository.repository_organisation_id = go.id + repository.repository_user_id = nil + repository.save + end + else + u = RepositoryUser.create_from_host('Bitbucket', o) + if u + repository.repository_user_id = u.id + repository.repository_organisation_id = nil + repository.save + end + end + rescue *IGNORABLE_EXCEPTIONS + nil end def create_webook(token = nil) From 3d1488061ef6892e0147fc938fda8a23df1d27bf Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 12 May 2017 11:49:26 +0100 Subject: [PATCH 14/33] escape bitbucket uuids before querying api --- app/models/repository_owner/bitbucket.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/repository_owner/bitbucket.rb b/app/models/repository_owner/bitbucket.rb index dc7e1dc72..4e49e1104 100644 --- a/app/models/repository_owner/bitbucket.rb +++ b/app/models/repository_owner/bitbucket.rb @@ -9,7 +9,7 @@ def repository_url end def self.fetch_user(id_or_login) - api_client.get_request "/2.0/users/#{id_or_login}" + api_client.get_request "/2.0/users/#{URI.escape(id_or_login)}" rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS nil end From 9349bc6edc50abcd07d724c29f72901d9d387d20 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 12 May 2017 15:55:19 +0100 Subject: [PATCH 15/33] Removed extra brackets --- app/models/repository_owner/gitlab.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/repository_owner/gitlab.rb b/app/models/repository_owner/gitlab.rb index 5eed2b98c..55840d4c0 100644 --- a/app/models/repository_owner/gitlab.rb +++ b/app/models/repository_owner/gitlab.rb @@ -10,7 +10,7 @@ def repository_url def self.fetch_user(id_or_login) if id_or_login.to_i.zero? - api_client.get(("/users?username=#{id_or_login}")).first + api_client.get("/users?username=#{id_or_login}").first else api_client.user(id_or_login) end From 06c7d04cadd300a50e6d78fe9e943382a2ef580b Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 12 May 2017 15:56:27 +0100 Subject: [PATCH 16/33] Handle the differences between users and orgs for gitlab and bitbucket --- app/models/repository_host/bitbucket.rb | 8 ++++---- app/models/repository_host/gitlab.rb | 15 +++++++++------ app/models/repository_owner/bitbucket.rb | 8 +++++++- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/app/models/repository_host/bitbucket.rb b/app/models/repository_host/bitbucket.rb index fbf220c9b..b9c3359da 100644 --- a/app/models/repository_host/bitbucket.rb +++ b/app/models/repository_host/bitbucket.rb @@ -64,10 +64,10 @@ def download_forks(token = nil) def download_owner return if repository.owner && repository.repository_user_id && repository.owner.login == repository.owner_name o = RepositoryOwner::Bitbucket.fetch_user(repository.owner_name) - if o.type == "Organization" - go = RepositoryOrganisation.create_from_gitlab(o.id) - if go - repository.repository_organisation_id = go.id + if o.type == "team" + org = RepositoryOrganisation.create_from_host('Bitbucket', o) + if org + repository.repository_organisation_id = org.id repository.repository_user_id = nil repository.save end diff --git a/app/models/repository_host/gitlab.rb b/app/models/repository_host/gitlab.rb index fb385d849..73bcd02f2 100644 --- a/app/models/repository_host/gitlab.rb +++ b/app/models/repository_host/gitlab.rb @@ -45,15 +45,18 @@ def download_forks(token = nil) def download_owner return if repository.owner && repository.repository_user_id && repository.owner.login == repository.owner_name - o = RepositoryOwner::Gitlab.fetch_user(repository.owner_name) - if o.type == "Organization" - go = RepositoryOrganisation.create_from_gitlab(o.id) - if go - repository.repository_organisation_id = go.id + namespace = api_client.namespaces(search: repository.owner_name).try(:first) + return unless namespace + if namespace.kind == 'group' + o = RepositoryOwner::Gitlab.api_client.group(namespace.path) + org = RepositoryOrganisation.create_from_host('GitLab', o) + if org + repository.repository_organisation_id = org.id repository.repository_user_id = nil repository.save end - else + elsif namespace.kind == 'user' + o = RepositoryOwner::Gitlab.fetch_user(namespace.path) u = RepositoryUser.create_from_host('GitLab', o) if u repository.repository_user_id = u.id diff --git a/app/models/repository_owner/bitbucket.rb b/app/models/repository_owner/bitbucket.rb index 4e49e1104..6a8c70824 100644 --- a/app/models/repository_owner/bitbucket.rb +++ b/app/models/repository_owner/bitbucket.rb @@ -9,7 +9,13 @@ def repository_url end def self.fetch_user(id_or_login) - api_client.get_request "/2.0/users/#{URI.escape(id_or_login)}" + begin + api_client.get_request "/2.0/users/#{URI.escape(id_or_login)}" + rescue BitBucket::Error::NotFound => error + if error.message.index('is a team account') + api_client.get_request "/2.0/teams/#{URI.escape(id_or_login)}" + end + end rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS nil end From 161e4fd1af18c322fcf6acdcb3b7ea845c901262 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 12 May 2017 16:22:47 +0100 Subject: [PATCH 17/33] Switch back to mainline gitlab gem --- Gemfile | 2 +- Gemfile.lock | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Gemfile b/Gemfile index 3b29f7a75..9d313ca3b 100644 --- a/Gemfile +++ b/Gemfile @@ -66,7 +66,7 @@ gem 'omniauth-bitbucket' gem 'chartkick' gem 'groupdate' gem 'turbolinks' -gem 'gitlab', git: 'https://github.com/librariesio/gitlab' +gem 'gitlab' gem 'bitbucket_rest_api', git: 'https://github.com/librariesio/bitbucket' gem 'github-markup', require: 'github/markup' gem 'commonmarker' diff --git a/Gemfile.lock b/Gemfile.lock index 7eb894873..0162452ab 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -20,13 +20,6 @@ GIT nokogiri (>= 1.5.2) simple_oauth -GIT - remote: https://github.com/librariesio/gitlab - revision: 69b59cd43b29c316b4f380610bb961ec398b22ee - specs: - gitlab (4.0.0) - httparty - GIT remote: https://github.com/librariesio/payola revision: 7d4026856edf0cbafc8a188748ba96ced7bad99b @@ -242,6 +235,9 @@ GEM mime-types (>= 1.19) rugged (>= 0.25.1) github-markup (1.6.0) + gitlab (4.0.0) + httparty + terminal-table (= 1.7.1) globalid (0.4.0) activesupport (>= 4.2.0) groupdate (3.2.0) @@ -524,6 +520,8 @@ GEM stripe_event (1.6.0) activesupport (>= 3.1) stripe (>= 1.6, < 3.0) + terminal-table (1.7.1) + unicode-display_width (~> 1.1.1) thor (0.19.4) thread_safe (0.3.6) tilt (2.0.7) @@ -538,6 +536,7 @@ GEM thread_safe (~> 0.1) uglifier (3.2.0) execjs (>= 0.3.0, < 3) + unicode-display_width (1.1.3) websocket-driver (0.6.5) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.2) @@ -594,7 +593,7 @@ DEPENDENCIES gemoji github-linguist github-markup - gitlab! + gitlab groupdate highscore hiredis From 09e7e2c435f6af0b857b814bc36c217d984691ff Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 12 May 2017 16:45:09 +0100 Subject: [PATCH 18/33] More eager loading of Gitlab modules for development --- app/models/repository_organisation.rb | 4 +++- app/models/repository_user.rb | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/app/models/repository_organisation.rb b/app/models/repository_organisation.rb index fa5fe9bd6..27f8321de 100644 --- a/app/models/repository_organisation.rb +++ b/app/models/repository_organisation.rb @@ -10,6 +10,9 @@ class RepositoryOrganisation < ApplicationRecord has_many :contributors, -> { group('repository_users.id').order("sum(contributions.count) DESC") }, through: :open_source_repositories, source: :contributors has_many :projects, through: :open_source_repositories + # eager load this module to avoid clashing with Gitlab gem in development + RepositoryOwner::Gitlab + validates :uuid, presence: true validates :login, uniqueness: {scope: :host_type}, if: lambda { self.login_changed? } validates :uuid, uniqueness: {scope: :host_type}, if: lambda { self.uuid_changed? } @@ -27,7 +30,6 @@ class RepositoryOrganisation < ApplicationRecord :to_s, :to_param, :github_id, to: :repository_owner def repository_owner - RepositoryOwner::Gitlab @repository_owner ||= RepositoryOwner.const_get(host_type.capitalize).new(self) end diff --git a/app/models/repository_user.rb b/app/models/repository_user.rb index b0a7da54b..bdd68507a 100644 --- a/app/models/repository_user.rb +++ b/app/models/repository_user.rb @@ -12,6 +12,9 @@ class RepositoryUser < ApplicationRecord has_many :projects, through: :open_source_repositories has_many :identities + # eager load this module to avoid clashing with Gitlab gem in development + RepositoryOwner::Gitlab + has_many :issues, primary_key: :uuid validates :login, uniqueness: {scope: :host_type}, if: lambda { self.login_changed? } @@ -28,7 +31,6 @@ class RepositoryUser < ApplicationRecord :to_s, :to_param, :github_id, :download_user_from_host, :download_user_from_host_by_login, to: :repository_owner def repository_owner - RepositoryOwner::Gitlab @repository_owner ||= RepositoryOwner.const_get(host_type.capitalize).new(self) end From bff5a9ed197f9c3d699bd323ba8702a788285938 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 12 May 2017 16:45:51 +0100 Subject: [PATCH 19/33] host agnostic create_org method --- app/models/repository_organisation.rb | 4 +++ app/models/repository_owner/base.rb | 21 ++++++++++++- app/models/repository_owner/bitbucket.rb | 40 ++++++++++++++++++++++++ app/models/repository_owner/github.rb | 30 ++++++++++++++++++ app/models/repository_owner/gitlab.rb | 40 ++++++++++++++++++++++++ 5 files changed, 134 insertions(+), 1 deletion(-) diff --git a/app/models/repository_organisation.rb b/app/models/repository_organisation.rb index 27f8321de..c7ca9c92b 100644 --- a/app/models/repository_organisation.rb +++ b/app/models/repository_organisation.rb @@ -69,6 +69,10 @@ def following 0 end + def self.create_from_host(host_type, org_hash) + RepositoryOwner.const_get(host_type.capitalize).create_org(org_hash) + end + def self.create_from_github(login_or_id) begin r = AuthToken.client.org(login_or_id).to_hash diff --git a/app/models/repository_owner/base.rb b/app/models/repository_owner/base.rb index 5cbda7a35..563178d9a 100644 --- a/app/models/repository_owner/base.rb +++ b/app/models/repository_owner/base.rb @@ -68,10 +68,30 @@ def self.download_user_from_host(host_type, id_or_login) RepositoryUser.create_from_host(host_type, self.fetch_user(id_or_login)) end + def download_org_from_host + download_org_from_host_by(owner.uuid) + end + + def download_org_from_host_by_login + download_org_from_host_by(owner.login) + end + + def download_org_from_host_by(id_or_login) + self.class.download_org_from_host(owner.host_type, id_or_login) + end + + def self.download_org_from_host(host_type, id_or_login) + RepositoryOrganisation.create_from_host(host_type, self.fetch_org(id_or_login)) + end + def self.fetch_user(id_or_login) raise NotImplementedError end + def self.fetch_org(id_or_login) + raise NotImplementedError + end + private def top_favourite_project_ids @@ -80,7 +100,6 @@ def top_favourite_project_ids end end - def top_contributor_ids Rails.cache.fetch [owner, "top_contributor_ids"], :expires_in => 1.week, race_condition_ttl: 2.minutes do owner.contributors.visible.limit(50).pluck(:id) diff --git a/app/models/repository_owner/bitbucket.rb b/app/models/repository_owner/bitbucket.rb index 6a8c70824..d748ed618 100644 --- a/app/models/repository_owner/bitbucket.rb +++ b/app/models/repository_owner/bitbucket.rb @@ -67,5 +67,45 @@ def self.create_user(user_hash) user.update(user_hash.slice(:name, :blog, :location)) user end + + def self.create_org(org_hash) + org_hash = org_hash.to_hash.with_indifferent_access + org_hash = { + id: org_hash[:uuid], + login: org_hash[:username], + name: org_hash[:display_name], + blog: org_hash[:website], + location: org_hash[:location], + type: org_hash[:type], + host_type: 'Bitbucket' + } + org = nil + org_by_id = RepositoryOrganisation.host('Bitbucket').find_by_uuid(org_hash[:id]) + org_by_login = RepositoryOrganisation.host('Bitbucket').where("lower(login) = ?", org_hash[:login].try(:downcase)).first + if org_by_id # its fine + if org_by_id.login.try(:downcase) == org_hash[:login].downcase && org_by_id.user_type == org_hash[:type] + org = org_by_id + else + if org_by_login && !org_by_login.download_org_from_host + org_by_login.destroy + end + org_by_id.login = org_hash[:login] + org_by_id.org_type = org_hash[:type] + org_by_id.save! + org = org_by_id + end + elsif org_by_login # conflict + if org_by_login.download_org_from_host_by_login + org = org_by_login if org_by_login.uuid == org_hash[:id] + end + org_by_login.destroy if org.nil? + end + if org.nil? + org = RepositoryOrganisation.create!(uuid: org_hash[:id], login: org_hash[:login], user_type: org_hash[:type], host_type: 'Bitbucket') + end + + org.update(org_hash.slice(:name, :blog, :location)) + org + end end end diff --git a/app/models/repository_owner/github.rb b/app/models/repository_owner/github.rb index ef72339cd..a4cca1cab 100644 --- a/app/models/repository_owner/github.rb +++ b/app/models/repository_owner/github.rb @@ -18,6 +18,36 @@ def self.api_client(token = nil) AuthToken.fallback_client(token) end + def self.create_org(org_hash) + org_hash = org_hash.to_hash.with_indifferent_access + org = nil + org_by_id = RepositoryOrganisation.host('GitHub').find_by_uuid(org_hash[:id]) + org_by_login = RepositoryOrganisation.host('GitHub').where("lower(login) = ?", org_hash[:login].try(:downcase)).first + if org_by_id # its fine + if org_by_id.login.try(:downcase) == org_hash[:login].downcase && org_by_id.user_type == org_hash[:type] + org = org_by_id + else + if org_by_login && !org_by_login.download_org_from_host + org_by_login.destroy + end + org_by_id.login = org_hash[:login] + org_by_id.user_type = org_hash[:type] + org_by_id.save! + org = org_by_id + end + elsif org_by_login # conflict + if org_by_login.download_org_from_host_by_login + org = org_by_login if org_by_login.uuid == org_hash[:id] + end + org_by_login.destroy if org.nil? + end + if org.nil? + org = RepositoryOrganisation.create!(uuid: org_hash[:id], login: org_hash[:login], user_type: org_hash[:type], host_type: 'GitHub') + end + org.update(org_hash.slice(:name, :company, :blog, :location, :email, :bio)) + org + end + def self.create_user(user_hash) user_hash = user_hash.to_hash.with_indifferent_access user = nil diff --git a/app/models/repository_owner/gitlab.rb b/app/models/repository_owner/gitlab.rb index 55840d4c0..b534b225f 100644 --- a/app/models/repository_owner/gitlab.rb +++ b/app/models/repository_owner/gitlab.rb @@ -67,5 +67,45 @@ def self.create_user(user_hash) user end + def self.create_org(org_hash) + org_hash = org_hash.to_hash.with_indifferent_access + org_hash = { + id: org_hash[:id], + login: org_hash[:orgname], + name: org_hash[:name], + blog: org_hash[:website_url], + location: org_hash[:location], + bio: org_hash[:bio], + type: 'Organisation', + host_type: 'GitLab' + } + org = nil + org_by_id = RepositoryOrganisation.host('GitLab').find_by_uuid(org_hash[:id]) + org_by_login = RepositoryOrganisation.host('GitLab').where("lower(login) = ?", org_hash[:login].try(:downcase)).first + if org_by_id # its fine + if org_by_id.login.try(:downcase) == org_hash[:login].downcase && org_by_id.user_type == org_hash[:type] + org = org_by_id + else + if org_by_login && !org_by_login.download_org_from_host + org_by_login.destroy + end + org_by_id.login = org_hash[:login] + org_by_id.user_type = org_hash[:type] + org_by_id.save! + org = org_by_id + end + elsif org_by_login # conflict + if org_by_login.download_org_from_host_by_login + org = org_by_login if org_by_login.uuid == org_hash[:id] + end + org_by_login.destroy if org.nil? + end + if org.nil? + org = RepositoryOrganisation.create!(uuid: org_hash[:id], login: org_hash[:login], user_type: org_hash[:type], host_type: 'GitLab') + end + + org.update(org_hash.slice(:name, :blog, :location)) + org + end end end From 90a01637c4eb432a3da7c0fdc691c9e20acfce9e Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 12 May 2017 16:57:49 +0100 Subject: [PATCH 20/33] Remove Github specific methods from RepoOrg and RepoUser classes --- app/models/repository_organisation.rb | 71 +-------------------------- app/models/repository_owner/base.rb | 8 +++ app/models/repository_owner/github.rb | 18 +++++++ app/models/repository_user.rb | 30 +---------- 4 files changed, 30 insertions(+), 97 deletions(-) diff --git a/app/models/repository_organisation.rb b/app/models/repository_organisation.rb index c7ca9c92b..45fb5bca9 100644 --- a/app/models/repository_organisation.rb +++ b/app/models/repository_organisation.rb @@ -27,7 +27,8 @@ class RepositoryOrganisation < ApplicationRecord scope :host, lambda{ |host_type| where('lower(repository_organisations.host_type) = ?', host_type.try(:downcase)) } delegate :avatar_url, :repository_url, :top_favourite_projects, :top_contributors, - :to_s, :to_param, :github_id, to: :repository_owner + :to_s, :to_param, :github_id, :download_org_from_host, :download_orgs, + :download_org_from_host_by_login, :download_repos, to: :repository_owner def repository_owner @repository_owner ||= RepositoryOwner.const_get(host_type.capitalize).new(self) @@ -53,10 +54,6 @@ def company nil end - def github_client - AuthToken.client - end - def user_type 'Organisation' end @@ -73,48 +70,6 @@ def self.create_from_host(host_type, org_hash) RepositoryOwner.const_get(host_type.capitalize).create_org(org_hash) end - def self.create_from_github(login_or_id) - begin - r = AuthToken.client.org(login_or_id).to_hash - return false if r.blank? - - org = nil - org_by_id = RepositoryOrganisation.host('GitHub').find_by_uuid(r[:id]) - if r[:login].present? - org_by_login = RepositoryOrganisation.host('GitHub').where("lower(login) = ?", r[:login].downcase).first - else - org_by_login = nil - end - - if org_by_id # its fine - if org_by_id.login.try(:downcase) == r[:login].try(:downcase) - org = org_by_id - else - if org_by_login && !org_by_login.download_from_github - org_by_login.destroy - end - org_by_id.login = r[:login] - org_by_id.save! - org = org_by_id - end - elsif org_by_login # conflict - if org_by_login.download_from_github_by_login - org = org_by_login if org_by_login.github_id == r[:id] - end - org_by_login.destroy if org.nil? - end - if org.nil? - org = RepositoryOrganisation.create!(uuid: r[:id], login: r[:login], host_type: 'GitHub') - end - - org.assign_attributes r.slice(*RepositoryOrganisation::API_FIELDS) - org.save - org - rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS - false - end - end - def async_sync RepositoryUpdateOrgWorker.perform_async(self.login) end @@ -125,28 +80,6 @@ def sync update_attributes(last_synced_at: Time.now) end - def download_from_github - download_from_github_by(github_id) - end - - def download_from_github_by_login - download_from_github_by(login) - end - - def download_from_github_by(id_or_login) - RepositoryOrganisation.create_from_github(github_client.org(id_or_login)) - rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS - nil - end - - def download_repos - github_client.org_repos(login).each do |repo| - CreateRepositoryWorker.perform_async('GitHub', repo.full_name) - end - rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS - nil - end - def find_repositories Repository.host(host_type).where('full_name ILIKE ?', "#{login}/%").update_all(repository_user_id: nil, repository_organisation_id: self.id) end diff --git a/app/models/repository_owner/base.rb b/app/models/repository_owner/base.rb index 563178d9a..7bb1ac5d5 100644 --- a/app/models/repository_owner/base.rb +++ b/app/models/repository_owner/base.rb @@ -37,6 +37,14 @@ def github_id owner.uuid end + def download_orgs + raise NotImplementedError + end + + def download_repos + raise NotImplementedError + end + def self.format(host_type) case host_type.try(:downcase) when 'github' diff --git a/app/models/repository_owner/github.rb b/app/models/repository_owner/github.rb index a4cca1cab..db268a39a 100644 --- a/app/models/repository_owner/github.rb +++ b/app/models/repository_owner/github.rb @@ -8,6 +8,24 @@ def repository_url "https://github.com/#{owner.login}" end + def download_orgs + api_client.orgs(owner.login).each do |org| + RepositoryCreateOrgWorker.perform_async(org.login) + end + true + rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS + nil + end + + def download_repos + api_client.search_repos("user:#{owner.login}").items.each do |repo| + CreateRepositoryWorker.perform_async('GitHub', repo.full_name) + end + true + rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS + nil + end + def self.fetch_user(id_or_login) api_client.user(id_or_login) rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS diff --git a/app/models/repository_user.rb b/app/models/repository_user.rb index bdd68507a..23f23fab3 100644 --- a/app/models/repository_user.rb +++ b/app/models/repository_user.rb @@ -28,16 +28,13 @@ class RepositoryUser < ApplicationRecord scope :host, lambda{ |host_type| where('lower(repository_users.host_type) = ?', host_type.try(:downcase)) } delegate :avatar_url, :repository_url, :top_favourite_projects, :top_contributors, - :to_s, :to_param, :github_id, :download_user_from_host, :download_user_from_host_by_login, to: :repository_owner + :to_s, :to_param, :github_id, :download_user_from_host, :download_orgs, + :download_user_from_host_by_login, :download_repos, to: :repository_owner def repository_owner @repository_owner ||= RepositoryOwner.const_get(host_type.capitalize).new(self) end - def github_id - uuid - end - def meta_tags { title: "#{self} on #{host_type}", @@ -54,10 +51,6 @@ def org? false end - def github_client - AuthToken.client - end - def async_sync RepositoryUpdateUserWorker.perform_async(self.login) end @@ -69,25 +62,6 @@ def sync update_attributes(last_synced_at: Time.now) end - def download_orgs - github_client.orgs(login).each do |org| - RepositoryCreateOrgWorker.perform_async(org.login) - end - true - rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS - nil - end - - def download_repos - AuthToken.client.search_repos("user:#{login}").items.each do |repo| - Repository.create_from_hash repo.to_hash - end - - true - rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS - nil - end - def self.create_from_host(host_type, user_hash) RepositoryOwner.const_get(host_type.capitalize).create_user(user_hash) end From 01793b555014e1cbed5e9b3a97a1e4c6d2038be9 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 15 May 2017 10:31:35 +0100 Subject: [PATCH 21/33] Fix spec --- app/models/repository_host/github.rb | 2 +- app/workers/repository_create_org_worker.rb | 2 +- spec/workers/repository_create_org_worker_spec.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/repository_host/github.rb b/app/models/repository_host/github.rb index 0cff1f7d9..8167a6b8d 100644 --- a/app/models/repository_host/github.rb +++ b/app/models/repository_host/github.rb @@ -127,7 +127,7 @@ def download_owner return if repository.owner && repository.repository_user_id && repository.owner.login == repository.owner_name o = api_client.user(repository.owner_name) if o.type == "Organization" - go = RepositoryOrganisation.create_from_github(o.id) + go = RepositoryOrganisation.create_from_host('GitHub', o) if go repository.repository_organisation_id = go.id repository.repository_user_id = nil diff --git a/app/workers/repository_create_org_worker.rb b/app/workers/repository_create_org_worker.rb index a489595f7..4aa78ed87 100644 --- a/app/workers/repository_create_org_worker.rb +++ b/app/workers/repository_create_org_worker.rb @@ -3,6 +3,6 @@ class RepositoryCreateOrgWorker sidekiq_options queue: :owners, unique: :until_executed def perform(org_login) - RepositoryOrganisation.create_from_github(org_login) + RepositoryOwner::Base.download_org_from_host('GitHub', org_login) end end diff --git a/spec/workers/repository_create_org_worker_spec.rb b/spec/workers/repository_create_org_worker_spec.rb index 81162c0d2..10d53a029 100644 --- a/spec/workers/repository_create_org_worker_spec.rb +++ b/spec/workers/repository_create_org_worker_spec.rb @@ -7,7 +7,7 @@ it "should create from github" do org_login = 'rails' - expect(RepositoryOrganisation).to receive(:create_from_github).with(org_login) + expect(RepositoryOwner::Base).to receive(:download_org_from_host).with('GitHub', org_login) subject.perform(org_login) end end From a13738b1b701ebf945a31d35b20fa84b381466ad Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 15 May 2017 10:46:24 +0100 Subject: [PATCH 22/33] Added TODOS --- app/models/repository_owner/bitbucket.rb | 12 ++++++++++++ app/models/repository_owner/gitlab.rb | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/app/models/repository_owner/bitbucket.rb b/app/models/repository_owner/bitbucket.rb index d748ed618..0a592c91e 100644 --- a/app/models/repository_owner/bitbucket.rb +++ b/app/models/repository_owner/bitbucket.rb @@ -20,6 +20,10 @@ def self.fetch_user(id_or_login) nil end + def self.fetch_org(id_or_login) + # TODO + end + def self.api_client(token = nil) BitBucket.new oauth_token: token || ENV['BITBUCKET_KEY'] end @@ -28,6 +32,14 @@ def api_client(token = nil) self.class.api_client(token) end + def download_orgs + # TODO + end + + def download_repos + # TODO + end + def self.create_user(user_hash) user_hash = user_hash.to_hash.with_indifferent_access user_hash = { diff --git a/app/models/repository_owner/gitlab.rb b/app/models/repository_owner/gitlab.rb index b534b225f..2ebd082e1 100644 --- a/app/models/repository_owner/gitlab.rb +++ b/app/models/repository_owner/gitlab.rb @@ -18,6 +18,10 @@ def self.fetch_user(id_or_login) nil end + def self.fetch_org(id_or_login) + # TODO + end + def self.api_client(token = nil) ::Gitlab.client(endpoint: 'https://gitlab.com/api/v3', private_token: token || ENV['GITLAB_KEY']) end @@ -26,6 +30,14 @@ def api_client(token = nil) self.class.api_client(token) end + def download_orgs + # TODO + end + + def download_repos + # TODO + end + def self.create_user(user_hash) user_hash = user_hash.to_hash.with_indifferent_access user_hash = { From 926643c4931bf71eb88cb6c0b8d6df91f69787ae Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 15 May 2017 12:06:50 +0100 Subject: [PATCH 23/33] Added worker for downloading users --- app/workers/repository_create_user_worker.rb | 8 ++++++++ spec/workers/repository_create_user_worker_spec.rb | 14 ++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 app/workers/repository_create_user_worker.rb create mode 100644 spec/workers/repository_create_user_worker_spec.rb diff --git a/app/workers/repository_create_user_worker.rb b/app/workers/repository_create_user_worker.rb new file mode 100644 index 000000000..0b07cb07c --- /dev/null +++ b/app/workers/repository_create_user_worker.rb @@ -0,0 +1,8 @@ +class RepositoryCreateUserWorker + include Sidekiq::Worker + sidekiq_options queue: :owners, unique: :until_executed + + def perform(host_type, org_login) + RepositoryOwner::Base.download_user_from_host(host_type, org_login) + end +end diff --git a/spec/workers/repository_create_user_worker_spec.rb b/spec/workers/repository_create_user_worker_spec.rb new file mode 100644 index 000000000..7e183420f --- /dev/null +++ b/spec/workers/repository_create_user_worker_spec.rb @@ -0,0 +1,14 @@ +require 'rails_helper' + +describe RepositoryCreateUserWorker do + it "should use the low priority queue" do + is_expected.to be_processed_in :owners + end + + it "should create from github" do + org_login = 'andrew' + host_type = 'GitHub' + expect(RepositoryOwner::Base).to receive(:download_user_from_host).with(host_type, org_login) + subject.perform(host_type, org_login) + end +end From eed427597843010f62d6059c6186cbadbf256756 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 15 May 2017 12:07:28 +0100 Subject: [PATCH 24/33] create org worker should take a host_type --- app/models/concerns/github_identity.rb | 2 +- app/workers/repository_create_org_worker.rb | 4 ++-- spec/workers/repository_create_org_worker_spec.rb | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app/models/concerns/github_identity.rb b/app/models/concerns/github_identity.rb index 1b1374250..74eda995e 100644 --- a/app/models/concerns/github_identity.rb +++ b/app/models/concerns/github_identity.rb @@ -85,7 +85,7 @@ def download_self def download_orgs return unless token github_client.orgs.each do |org| - RepositoryCreateOrgWorker.perform_async(org.login) + RepositoryCreateOrgWorker.perform_async('GitHub', org.login) end rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS nil diff --git a/app/workers/repository_create_org_worker.rb b/app/workers/repository_create_org_worker.rb index 4aa78ed87..374ed420d 100644 --- a/app/workers/repository_create_org_worker.rb +++ b/app/workers/repository_create_org_worker.rb @@ -2,7 +2,7 @@ class RepositoryCreateOrgWorker include Sidekiq::Worker sidekiq_options queue: :owners, unique: :until_executed - def perform(org_login) - RepositoryOwner::Base.download_org_from_host('GitHub', org_login) + def perform(host_type, org_login) + RepositoryOwner::Base.download_org_from_host(host_type, org_login) end end diff --git a/spec/workers/repository_create_org_worker_spec.rb b/spec/workers/repository_create_org_worker_spec.rb index 10d53a029..fb05eacf2 100644 --- a/spec/workers/repository_create_org_worker_spec.rb +++ b/spec/workers/repository_create_org_worker_spec.rb @@ -7,7 +7,8 @@ it "should create from github" do org_login = 'rails' - expect(RepositoryOwner::Base).to receive(:download_org_from_host).with('GitHub', org_login) - subject.perform(org_login) + host_type = 'GitHub' + expect(RepositoryOwner::Base).to receive(:download_org_from_host).with(host_type, org_login) + subject.perform(host_type, org_login) end end From 7cd6cf049fbee7ba77e68ef82bea6b8fc9137ee8 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 15 May 2017 12:09:00 +0100 Subject: [PATCH 25/33] Added extra syncing of members for orgs --- app/models/repository_organisation.rb | 1 + app/models/repository_owner/bitbucket.rb | 11 +++++++++++ app/models/repository_owner/github.rb | 11 +++++++++++ app/models/repository_owner/gitlab.rb | 11 +++++++++++ 4 files changed, 34 insertions(+) diff --git a/app/models/repository_organisation.rb b/app/models/repository_organisation.rb index 45fb5bca9..331141fb4 100644 --- a/app/models/repository_organisation.rb +++ b/app/models/repository_organisation.rb @@ -77,6 +77,7 @@ def async_sync def sync download_from_github download_repos + download_members update_attributes(last_synced_at: Time.now) end diff --git a/app/models/repository_owner/bitbucket.rb b/app/models/repository_owner/bitbucket.rb index 0a592c91e..84783994a 100644 --- a/app/models/repository_owner/bitbucket.rb +++ b/app/models/repository_owner/bitbucket.rb @@ -40,6 +40,17 @@ def download_repos # TODO end + def download_members + return unless owner.org? + + api_client.teams.members(owner.login).each do |org| + RepositoryCreateUserWorker.perform_async('Bitbucket', org.login) + end + true + rescue *RepositoryHost::Bitbucket::IGNORABLE_EXCEPTIONS + nil + end + def self.create_user(user_hash) user_hash = user_hash.to_hash.with_indifferent_access user_hash = { diff --git a/app/models/repository_owner/github.rb b/app/models/repository_owner/github.rb index db268a39a..151b10dd8 100644 --- a/app/models/repository_owner/github.rb +++ b/app/models/repository_owner/github.rb @@ -26,6 +26,17 @@ def download_repos nil end + def download_members + return unless owner.org? + + api_client.organization_members(owner.login).each do |org| + RepositoryCreateUserWorker.perform_async('GitHub', org.login) + end + true + rescue *RepositoryHost::Gitlab::IGNORABLE_EXCEPTIONS + nil + end + def self.fetch_user(id_or_login) api_client.user(id_or_login) rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS diff --git a/app/models/repository_owner/gitlab.rb b/app/models/repository_owner/gitlab.rb index 2ebd082e1..3ac4af126 100644 --- a/app/models/repository_owner/gitlab.rb +++ b/app/models/repository_owner/gitlab.rb @@ -38,6 +38,17 @@ def download_repos # TODO end + def download_members + return unless owner.org? + + api_client.group_members(owner.login).each do |org| + RepositoryCreateUserWorker.perform_async('GitLab', org.login) + end + true + rescue *RepositoryHost::Gitlab::IGNORABLE_EXCEPTIONS + nil + end + def self.create_user(user_hash) user_hash = user_hash.to_hash.with_indifferent_access user_hash = { From be14a21c3b9949ca16ddba915fe48619f7c1260a Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 15 May 2017 12:09:25 +0100 Subject: [PATCH 26/33] Fix syncing of orgs for github users --- app/models/repository_owner/github.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/repository_owner/github.rb b/app/models/repository_owner/github.rb index 151b10dd8..0427ec498 100644 --- a/app/models/repository_owner/github.rb +++ b/app/models/repository_owner/github.rb @@ -9,8 +9,9 @@ def repository_url end def download_orgs + return if owner.org? api_client.orgs(owner.login).each do |org| - RepositoryCreateOrgWorker.perform_async(org.login) + RepositoryCreateOrgWorker.perform_async('GitHub', org.login) end true rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS From bfa7006e4377c2591a074a5c06dc5d955e55c015 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 15 May 2017 12:09:58 +0100 Subject: [PATCH 27/33] download_repos method for bitbucket owners --- app/models/repository_owner/bitbucket.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/models/repository_owner/bitbucket.rb b/app/models/repository_owner/bitbucket.rb index 84783994a..dcfa61fe4 100644 --- a/app/models/repository_owner/bitbucket.rb +++ b/app/models/repository_owner/bitbucket.rb @@ -37,7 +37,12 @@ def download_orgs end def download_repos - # TODO + api_client.get_request("/2.0/repositories/#{owner.login}")['values'].each do |repo| + CreateRepositoryWorker.perform_async('Bitbucket', repo.full_name) + end + true + rescue *RepositoryHost::Bitbucket::IGNORABLE_EXCEPTIONS + nil end def download_members From c9b1e3a4c32382cf8ead1ffec0f79054a2892c3a Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 15 May 2017 12:34:16 +0100 Subject: [PATCH 28/33] Update user worker should take a host_type --- app/models/concerns/github_identity.rb | 2 +- app/models/repository_user.rb | 2 +- app/workers/repository_update_user_worker.rb | 4 ++-- spec/workers/repository_update_user_worker_spec.rb | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/models/concerns/github_identity.rb b/app/models/concerns/github_identity.rb index 74eda995e..a56407bff 100644 --- a/app/models/concerns/github_identity.rb +++ b/app/models/concerns/github_identity.rb @@ -78,7 +78,7 @@ def download_self repository_user = RepositoryUser.create_from_host('GitHub', {id: github_identity.uid, login: github_identity.nickname, type: 'User', host_type: 'GitHub'}) if repository_user github_identity.update_column(:repository_user_id, repository_user.id) - RepositoryUpdateUserWorker.perform_async(nickname) + RepositoryUpdateUserWorker.perform_async('GitHub', nickname) end end diff --git a/app/models/repository_user.rb b/app/models/repository_user.rb index 23f23fab3..4909c88a2 100644 --- a/app/models/repository_user.rb +++ b/app/models/repository_user.rb @@ -52,7 +52,7 @@ def org? end def async_sync - RepositoryUpdateUserWorker.perform_async(self.login) + RepositoryUpdateUserWorker.perform_async(host_type, self.login) end def sync diff --git a/app/workers/repository_update_user_worker.rb b/app/workers/repository_update_user_worker.rb index 2faf43ab1..3acef2a49 100644 --- a/app/workers/repository_update_user_worker.rb +++ b/app/workers/repository_update_user_worker.rb @@ -2,7 +2,7 @@ class RepositoryUpdateUserWorker include Sidekiq::Worker sidekiq_options queue: :owners, unique: :until_executed - def perform(login) - RepositoryUser.host('GitHub').find_by_login(login).try(:sync) + def perform(host_type, login) + RepositoryUser.host(host_type).find_by_login(login).try(:sync) end end diff --git a/spec/workers/repository_update_user_worker_spec.rb b/spec/workers/repository_update_user_worker_spec.rb index bd988cf2e..9dbc93d05 100644 --- a/spec/workers/repository_update_user_worker_spec.rb +++ b/spec/workers/repository_update_user_worker_spec.rb @@ -9,6 +9,6 @@ user = create(:repository_user) expect(RepositoryUser).to receive(:find_by_login).with(user.login).and_return(user) expect(user).to receive(:sync) - subject.perform(user.login) + subject.perform(user.host_type, user.login) end end From 657a3219cd378bed78e871e167f2d879e7a7b1a8 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 15 May 2017 15:15:52 +0100 Subject: [PATCH 29/33] Scraping of orgs and repos for gitlab users --- app/models/repository_owner/gitlab.rb | 28 +++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/app/models/repository_owner/gitlab.rb b/app/models/repository_owner/gitlab.rb index 3ac4af126..aa6113c87 100644 --- a/app/models/repository_owner/gitlab.rb +++ b/app/models/repository_owner/gitlab.rb @@ -31,11 +31,35 @@ def api_client(token = nil) end def download_orgs - # TODO + return if owner.org? + + # GitLab doesn't have an API to get a users public group memberships so we scrape it instead + groups_html = Nokogiri::HTML(PackageManager::Base.get_json("https://gitlab.com/users/#{owner.login}/groups")['html']) + links = groups_html.css('a.group-name').map{|l| l['href'][1..-1]} + + links.each do |org_login| + RepositoryCreateOrgWorker.perform_async('GitLab', org_login) + end + true + rescue *RepositoryHost::Gitlab::IGNORABLE_EXCEPTIONS + nil end def download_repos - # TODO + if owner.org? + repos = api_client.group_projects(owner.login).map(&:full_name) + else + # GitLab doesn't have an API to get a users public projects so we scrape it instead + projects_html = Nokogiri::HTML(PackageManager::Base.get_json("https://gitlab.com/users/#{owner.login}/projects")['html']) + links = projects_html.css('a.project').map{|l| l['href'][1..-1] }.uniq + end + + links.each do |repo_name| + CreateRepositoryWorker.perform_async('GitLab', repo_name) + end + true + rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS + nil end def download_members From b935e82b1cea399b0bc151f55ad3dd33dd2c1ca8 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 15 May 2017 15:31:15 +0100 Subject: [PATCH 30/33] Scrape bitbucket user orgs from profile pages --- app/models/repository_owner/bitbucket.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/app/models/repository_owner/bitbucket.rb b/app/models/repository_owner/bitbucket.rb index dcfa61fe4..3a0ed657e 100644 --- a/app/models/repository_owner/bitbucket.rb +++ b/app/models/repository_owner/bitbucket.rb @@ -33,7 +33,17 @@ def api_client(token = nil) end def download_orgs - # TODO + return if owner.org? + # Bitbucket doesn't have an API to get a users public group memberships so we scrape it instead + groups_html = PackageManager::Base.get_html("https://bitbucket.org/#{owner.login}/profile/teams") + links = groups_html.css('li.team a.name').map{|l| l['title']} + + links.each do |org_name| + RepositoryCreateOrgWorker.perform_async('Bitbucket', org_name) + end + true + rescue *RepositoryHost::Bitbucket::IGNORABLE_EXCEPTIONS + nil end def download_repos From 80933c680dbf16ec2a2a7c5d6bd86f2a32d661a3 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 15 May 2017 15:31:29 +0100 Subject: [PATCH 31/33] Fix copypasta error --- app/models/repository_owner/bitbucket.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/repository_owner/bitbucket.rb b/app/models/repository_owner/bitbucket.rb index 3a0ed657e..17853c1aa 100644 --- a/app/models/repository_owner/bitbucket.rb +++ b/app/models/repository_owner/bitbucket.rb @@ -16,7 +16,7 @@ def self.fetch_user(id_or_login) api_client.get_request "/2.0/teams/#{URI.escape(id_or_login)}" end end - rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS + rescue *RepositoryHost::BitBucket::IGNORABLE_EXCEPTIONS nil end From 0312d97e052920214b8d84ed0c09401cb2752812 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 15 May 2017 15:34:09 +0100 Subject: [PATCH 32/33] More copy paster --- app/models/repository_owner/gitlab.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/repository_owner/gitlab.rb b/app/models/repository_owner/gitlab.rb index aa6113c87..f99acbb16 100644 --- a/app/models/repository_owner/gitlab.rb +++ b/app/models/repository_owner/gitlab.rb @@ -14,7 +14,7 @@ def self.fetch_user(id_or_login) else api_client.user(id_or_login) end - rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS + rescue *RepositoryHost::Gitlab::IGNORABLE_EXCEPTIONS nil end @@ -58,7 +58,7 @@ def download_repos CreateRepositoryWorker.perform_async('GitLab', repo_name) end true - rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS + rescue *RepositoryHost::Gitlab::IGNORABLE_EXCEPTIONS nil end From 854b2c88f83e699d0e06921a8a6769ac55f55e56 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 15 May 2017 15:34:34 +0100 Subject: [PATCH 33/33] Fetch org methods for gitlab and bitbucket --- app/models/repository_owner/bitbucket.rb | 4 +++- app/models/repository_owner/gitlab.rb | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/app/models/repository_owner/bitbucket.rb b/app/models/repository_owner/bitbucket.rb index 17853c1aa..62c84d4f7 100644 --- a/app/models/repository_owner/bitbucket.rb +++ b/app/models/repository_owner/bitbucket.rb @@ -21,7 +21,9 @@ def self.fetch_user(id_or_login) end def self.fetch_org(id_or_login) - # TODO + api_client.get_request "/2.0/teams/#{URI.escape(id_or_login)}" + rescue *RepositoryHost::BitBucket::IGNORABLE_EXCEPTIONS + nil end def self.api_client(token = nil) diff --git a/app/models/repository_owner/gitlab.rb b/app/models/repository_owner/gitlab.rb index f99acbb16..5b94025f9 100644 --- a/app/models/repository_owner/gitlab.rb +++ b/app/models/repository_owner/gitlab.rb @@ -19,7 +19,9 @@ def self.fetch_user(id_or_login) end def self.fetch_org(id_or_login) - # TODO + api_client.group(id_or_login) + rescue *RepositoryHost::Gitlab::IGNORABLE_EXCEPTIONS + nil end def self.api_client(token = nil)