Make code nicer

- remove dead code
- use upsert method where necessary

README.md

@@ -40,7 +40,7 @@ For distributed mirroring you also need RabbitMQ >= 3.3
 
 
 ### 1. Install GHTorrent
-GHTorrent is written in Ruby (tested with 2.0). To install it as a Gem do:
+GHTorrent is written in Ruby (tested with Ruby > 2.0). To install it as a Gem do:
 
 <code>
 sudo gem install ghtorrent

lib/ghtorrent/commands/full_repo_retriever.rb

@@ -11,7 +11,7 @@ module FullRepoRetriever
 
       def stages
         %w(ensure_commits ensure_forks ensure_pull_requests
-       ensure_issues ensure_watchers ensure_labels ensure_languages) #ensure_project_members
+       ensure_issues ensure_watchers ensure_labels ensure_languages)
       end
 
       def settings

lib/ghtorrent/ghtorrent.rb

@@ -593,7 +593,7 @@ def ensure_repo(user, repo, recursive = false)
           repos.filter(:owner_id => curuser[:id], :name => repo).update(:forked_from => parent[:id])
           info "Repo #{user}/#{repo} is a fork of #{parent_owner}/#{parent_repo}"
 
-          forked_commit = ensure_fork_point(user, repo)
+          ensure_fork_point(user, repo)
         end
       end
 
@@ -813,71 +813,6 @@ def ensure_fork_point(owner, repo)
       db[:commits].where(:sha => forked_sha).first
     end
 
-    ##
-    # Make sure that a project has all the registered members defined
-    def ensure_project_members(user, repo, refresh = false)
-      currepo = ensure_repo(user, repo)
-      time = currepo[:created_at]
-
-      project_members = db.from(:project_members, :users).\
-          where(:project_members__user_id => :users__id).\
-          where(:project_members__repo_id => currepo[:id]).select(:login).all
-
-      retrieve_repo_collaborators(user, repo).reduce([]) do |acc, x|
-        if project_members.find {|y| y[:login] == x['login']}.nil?
-          acc << x
-        else
-          acc
-        end
-      end.map { |x| save{ensure_project_member(user, repo, x['login'], time) }}.select{|x| !x.nil?}
-    end
-
-    ##
-    # Make sure that a project member exists in a project
-    def ensure_project_member(owner, repo, new_member, date_added)
-      pr_members = db[:project_members]
-      project = ensure_repo(owner, repo)
-      new_user = ensure_user(new_member, false, false)
-
-      if project.nil? or new_user.nil?
-        warn "Could not find repo #{owner}/#{repo} or member #{new_member}"
-        return
-      end
-
-      memb_exist = pr_members.first(:user_id => new_user[:id],
-                                    :repo_id => project[:id])
-
-      if memb_exist.nil?
-        added = if date_added.nil?
-                  max(project[:created_at], new_user[:created_at])
-                else
-                  date_added
-                end
-        retrieved = retrieve_repo_collaborator(owner, repo, new_member)
-
-        if retrieved.nil?
-          warn "Could not retrieve member #{new_member} of #{owner}/#{repo}"
-          return
-        end
-
-        pr_members.insert(
-            :user_id => new_user[:id],
-            :repo_id => project[:id],
-            :created_at => date(added)
-        )
-        info "Added project_member #{repo} -> #{new_member}"
-      else
-        debug "Project member #{repo} -> #{new_member} exists"
-      end
-
-      unless date_added.nil?
-        pr_members.filter(:user_id => new_user[:id],
-                          :repo_id => project[:id])\
-                    .update(:created_at => date(date_added))
-        info "Updated project member #{repo} -> #{new_member}, created_at -> #{date(date_added)}"
-      end
-    end
-
     ##
     # Make sure that the organizations the user participates into exist
     #
@@ -1003,7 +938,7 @@ def ensure_commit_comment(owner, repo, sha, comment_id)
 
     ##
     # Make sure that all watchers exist for a repository
-    def ensure_watchers(owner, repo, refresh = false)
+    def ensure_watchers(owner, repo)
       currepo = ensure_repo(owner, repo)
 
       if currepo.nil?
@@ -1023,12 +958,12 @@ def ensure_watchers(owner, repo, refresh = false)
         else
           acc
         end
-      end.map { |x| save{ensure_watcher(owner, repo, x['login'], nil) }}.select{|x| !x.nil?}
+      end.map { |x| save{ensure_watcher(owner, repo, x['login']) }}.select{|x| !x.nil?}
     end
 
     ##
     # Make sure that a watcher/stargazer exists for a repository
-    def ensure_watcher(owner, repo, watcher, date_added = nil)
+    def ensure_watcher(owner, repo, watcher, date_added: nil)
       project = ensure_repo(owner, repo)
       new_watcher = ensure_user(watcher, false, false)
 
@@ -1448,23 +1383,19 @@ def ensure_fork(owner, repo, fork_id)
 
     ##
     # Make sure all issues exist for a project
-    def ensure_issues(owner, repo, refresh = false)
+    def ensure_issues(owner, repo)
       currepo = ensure_repo(owner, repo)
       if currepo.nil?
         warn "Could not find repo #{owner}/#{repo} for retrieving issues"
         return
       end
 
-      raw_issues = if refresh
-                     retrieve_issues(owner, repo, refresh = true)
-                   else
-                     issues = db[:issues].filter(:repo_id => currepo[:id]).all
-                     retrieve_issues(owner, repo).reduce([]) do |acc, x|
-                       if issues.find { |y| y[:issue_id] == x['number'] }.nil?
-                         acc << x
-                       else
+      issues = db[:issues].filter(:repo_id => currepo[:id]).all
+      raw_issues = retrieve_issues(owner, repo).reduce([]) do |acc, x|
+                     if issues.find { |y| y[:issue_id] == x['number'] }.nil?
+                       acc << x
+                     else
                          acc
-                       end
                      end
                    end
 
@@ -1719,7 +1650,7 @@ def ensure_issue_comment(owner, repo, issue_id, comment_id, pull_req_id = nil)
 
     ##
     # Retrieve repository issue labels
-    def ensure_labels(owner, repo, refresh = false)
+    def ensure_labels(owner, repo)
       currepo = ensure_repo(owner, repo)
 
       if currepo.nil?
@@ -1729,7 +1660,7 @@ def ensure_labels(owner, repo, refresh = false)
 
       repo_labels = db[:repo_labels].filter(:repo_id => currepo[:id]).all
 
-      retrieve_repo_labels(owner, repo, refresh).reduce([]) do |acc, x|
+      retrieve_repo_labels(owner, repo).reduce([]) do |acc, x|
         if repo_labels.find {|y| y[:name] == x['name']}.nil?
           acc << x
         else

lib/ghtorrent/retriever.rb

@@ -288,22 +288,6 @@ def retrieve_commit_comment(owner, repo, sha, id)
       end
     end
 
-    # Retrieve all collaborators for a repository
-    def retrieve_repo_collaborators(user, repo)
-      repo_bound_items(user, repo, :repo_collaborators,
-                       ["repos/#{user}/#{repo}/collaborators"],
-                       {'repo' => repo, 'owner' => user},
-                       'login', item = nil, refresh = false, order = :asc)
-    end
-
-    # Retrieve a single repository collaborator
-    def retrieve_repo_collaborator(user, repo, new_member)
-      repo_bound_item(user, repo, new_member, :repo_collaborators,
-                      ["repos/#{user}/#{repo}/collaborators"],
-                      {'repo' => repo, 'owner' => user},
-                      'login')
-    end
-
     # Retrieve all watchers for a repository
     def retrieve_watchers(user, repo)
       repo_bound_items(user, repo, :watchers,
@@ -654,8 +638,7 @@ def repo_bound_items(user, repo, entity, urls, selector, discriminator,
                        end
 
                   instance_selector = selector.merge({discriminator => id})
-                  persister.del(entity, instance_selector)
-                  persister.store(entity, x)
+                  persister.upsert(entity, instance_selector, x)
                   debug "Refreshing #{entity} #{user}/#{repo} -> #{x[discriminator]}"
                 end
               else

lib/ghtorrent/transacted_gh_torrent.rb

@@ -73,11 +73,6 @@ def ensure_issue_label(owner, repo, issue_id, name)
     end
   end
 
-  def ensure_project_member(owner, repo, new_member, date_added)
-    check_transaction do
-      super(owner, repo, new_member, date_added)
-    end
-  end
 
   def ensure_watcher(owner, repo, watcher, date_added = nil)
     check_transaction do