Merge pull request #182 from coinbase/github-resources

Add support for most GitHub resources

geoengineer.gemspec

@@ -34,4 +34,5 @@ Gem::Specification.new do |s|
   s.add_dependency 'commander',         '~> 4.4'
   s.add_dependency 'colorize',          '~> 0.7'
   s.add_dependency 'parallel',          '~> 1.10'
+  s.add_dependency 'octokit',           '~> 4.8'
 end

lib/geoengineer.rb

@@ -18,6 +18,7 @@ module GeoEngineer::Templates
 
 require 'aws-sdk'
 require 'json'
+require 'octokit'
 require 'ostruct'
 require 'uri'
 require 'securerandom'

lib/geoengineer/environment.rb

@@ -18,8 +18,6 @@ class GeoEngineer::Environment
 
   attr_reader :name
 
-  validate -> { validate_required_attributes([:region, :account_id]) }
-
   # Validate resources have unique attributes
   validate -> {
     resources_of_type_grouped_by(&:terraform_name).map do |klass, grouped_resources|
@@ -51,7 +49,7 @@ class GeoEngineer::Environment
   # Validate all resources
   validate -> { all_resources.map(&:errors).flatten }
 
-  before :validation, -> { self.region ||= ENV['AWS_REGION'] }
+  before :validation, -> { self.region ||= ENV['AWS_REGION'] if ENV['AWS_REGION'] }
 
   def initialize(name, &block)
     @name = name

lib/geoengineer/resources/github_issue_label.rb

@@ -0,0 +1,25 @@
+################################################################################
+# GithubIssueLabel is the +github_issue_label+ Terraform resource.
+#
+# {https://www.terraform.io/docs/providers/github/r/issue_label.html Terraform Docs}
+################################################################################
+class GeoEngineer::Resources::GithubIssueLabel < GeoEngineer::Resource
+  validate -> { validate_required_attributes([:repository, :name, :color]) }
+
+  after :initialize, -> { _terraform_id -> { "#{repository}:#{name}" } }
+  after :initialize, -> { _geo_id -> { "#{repository}:#{name}" } }
+
+  def self._fetch_remote_resources(provider)
+    repos = GithubClient.organization_repositories(provider.organization)
+
+    Parallel.map(repos, { in_threads: Parallel.processor_count * 3 }) do |repo|
+      labels = GithubClient.labels(repo[:full_name])
+      labels.each do |label|
+        label[:_terraform_id] = "#{repo[:name]}:#{label[:name]}"
+        label[:_geo_id] = collab[:_terraform_id]
+
+        label[:repository] = repo[:name]
+      end
+    end.flatten
+  end
+end

lib/geoengineer/resources/github_membership.rb

@@ -0,0 +1,28 @@
+################################################################################
+# GithubMembership is the +github_membership+ Terraform resource.
+#
+# {https://www.terraform.io/docs/providers/github/r/membership.html Terraform Docs}
+################################################################################
+class GeoEngineer::Resources::GithubMembership < GeoEngineer::Resource
+  validate -> { validate_required_attributes([:username]) }
+
+  after :initialize, -> { _terraform_id -> { "#{fetch_provider.organization}:#{username}" } }
+  after :initialize, -> { _geo_id -> { "#{fetch_provider.organization}:#{username}" } }
+
+  def self._fetch_remote_resources(provider)
+    # GitHub doesn't return the actual role of the member in the API, so we have
+    # to make requests for each role to assign each set the appropriate role
+    roles = %i[admin member]
+    Parallel.map(roles, { in_threads: Parallel.processor_count }) do |member_role|
+      members = GithubClient.organization_members(provider.organization, { role: member_role })
+
+      members.each do |member|
+        member[:_terraform_id] = "#{provider.organization}:#{member[:login]}"
+        member[:_geo_id] = member[:_terraform_id]
+
+        member[:username] = member[:login]
+        member[:role] = member_role
+      end
+    end.flatten
+  end
+end

lib/geoengineer/resources/github_repository.rb

@@ -0,0 +1,22 @@
+################################################################################
+# GithubRepository is the +github_repository+ Terraform resource.
+#
+# {https://www.terraform.io/docs/providers/github/r/repository.html Terraform Docs}
+################################################################################
+class GeoEngineer::Resources::GithubRepository < GeoEngineer::Resource
+  validate -> { validate_required_attributes([:name]) }
+
+  after :initialize, -> { _terraform_id -> { name } }
+  after :initialize, -> { _geo_id -> { name } }
+
+  def self._fetch_remote_resources(provider)
+    repos = GithubClient.organization_repositories(provider.organization)
+
+    repos.each do |repo|
+      repo[:_terraform_id] = repo[:name]
+      repo[:_geo_id] = repo[:name]
+      repo[:homepage_url] = repo[:homepage]
+      # TODO: Figure out how to get/set "allow_{rebase,squash,merge} commit
+    end
+  end
+end

lib/geoengineer/resources/github_repository_collaborator.rb

@@ -0,0 +1,39 @@
+################################################################################
+# GithubRepositoryCollaborator is the +github_repository_collaborator+ Terraform
+# resource.
+#
+# {https://www.terraform.io/docs/providers/github/r/repository_collaborator.html Terraform Docs}
+################################################################################
+class GeoEngineer::Resources::GithubRepositoryCollaborator < GeoEngineer::Resource
+  validate -> { validate_required_attributes([:repository, :username]) }
+
+  after :initialize, -> { _terraform_id -> { "#{repository}:#{username}" } }
+  after :initialize, -> { _geo_id -> { "#{repository}:#{username}" } }
+
+  def self._fetch_remote_resources(provider)
+    repos = GithubClient.organization_repositories(provider.organization)
+
+    Parallel.map(repos, { in_threads: Parallel.processor_count }) do |repo|
+      collaborators_for_repo(repo)
+    end.flatten
+  end
+
+  def self.collaborators_for_repo(repo) # rubocop:disable Metrics/AbcSize
+    GithubClient.repository_collaborators(repo[:full_name]).each do |collab|
+      collab[:_terraform_id] = "#{collab[:repository]}:#{collab[:login]}"
+      collab[:_geo_id] = collab[:_terraform_id]
+
+      collab[:username] = collab[:login]
+      collab[:repository] = repo[:name]
+
+      collab[:permission] =
+        if collab[:permissions][:admin]
+          'admin'
+        elsif collab[:permissions][:push]
+          'push'
+        elsif collab[:permissions][:pull]
+          'pull'
+        end
+    end
+  end
+end

lib/geoengineer/resources/github_team.rb

@@ -0,0 +1,19 @@
+################################################################################
+# GithubTeam is the +github_team+ Terraform resource.
+#
+# {https://www.terraform.io/docs/providers/github/r/team.html Terraform Docs}
+################################################################################
+class GeoEngineer::Resources::GithubTeam < GeoEngineer::Resource
+  validate -> { validate_required_attributes([:name]) }
+
+  after :initialize, -> { _terraform_id -> { NullObject.maybe(remote_resource)._terraform_id } }
+  after :initialize, -> { _geo_id -> { name } }
+
+  def self._fetch_remote_resources(provider)
+    GithubClient.organization_teams(provider.organization)
+                .each do |team|
+      team[:_terraform_id] = team[:id].to_s
+      team[:_geo_id] = team[:name]
+    end
+  end
+end

lib/geoengineer/resources/github_team_membership.rb

@@ -0,0 +1,31 @@
+################################################################################
+# GithubTeamMembership is the +github_team_membership+ Terraform resource.
+#
+# {https://www.terraform.io/docs/providers/github/r/team_membership.html Terraform Docs}
+################################################################################
+class GeoEngineer::Resources::GithubTeamMembership < GeoEngineer::Resource
+  validate -> { validate_required_attributes([:team_id, :username]) }
+
+  after :initialize, -> { _terraform_id -> { NullObject.maybe(remote_resource)._terraform_id } }
+  after :initialize, -> { _geo_id -> { "#{team_id}:#{username}" } }
+
+  def self._fetch_remote_resources(provider)
+    # There is no way to obtain all these resources in bulk in a single request,
+    # so we iterate over all teams and fetch their individual memberships.
+    # GitHub doesn't return the actual role of the member in the API, so we have
+    # to make calls with different filters to know which role the member has.
+    teams = GithubClient.organization_teams(provider.organization)
+    roles = %i[maintainer member]
+    jobs = teams.flat_map { |team| roles.map { |team_role| [team[:id], team_role] } }
+
+    Parallel.map(jobs, { in_threads: Parallel.processor_count }) do |team_id, team_role|
+      GithubClient.team_memberships(team_id, { role: team_role })
+                  .each do |team_membership|
+        team_membership[:_terraform_id] = "#{team_id}:#{team_membership[:login]}"
+        team_membership[:_geo_id] = team_membership[:_terraform_id]
+        team_membership[:username] = team_membership[:login]
+        team_membership[:role] = team_role
+      end
+    end.flatten
+  end
+end

lib/geoengineer/resources/github_team_repository.rb

@@ -0,0 +1,41 @@
+################################################################################
+# GithubTeamRepository is the +github_team_repository+ Terraform resource.
+#
+# {https://www.terraform.io/docs/providers/github/r/team_repository.html Terraform Docs}
+################################################################################
+class GeoEngineer::Resources::GithubTeamRepository < GeoEngineer::Resource
+  validate -> { validate_required_attributes([:team_id, :repository]) }
+
+  after :initialize, -> { _terraform_id -> { NullObject.maybe(remote_resource)._terraform_id } }
+  after :initialize, -> { _geo_id -> { "#{team_id}:#{repository}" } }
+
+  def self._fetch_remote_resources(provider)
+    # There is no way to obtain all these resources in bulk in a single request,
+    # so we iterate over all teams and fetch their individual repo permissions.
+    teams = GithubClient.organization_teams(provider.organization)
+
+    Parallel.map(teams, { in_threads: Parallel.processor_count }) do |team, team_role|
+      repos_for_team(team[:id])
+    end.flatten
+  end
+
+  def self.repos_for_team(team_id)
+    GithubClient.team_repositories(team_id)
+                .each do |team_repo|
+      team_repo[:_terraform_id] = "#{team_id}:#{team_repo[:name]}"
+      team_repo[:_geo_id] = team_repo[:_terraform_id]
+
+      team_repo[:team_id] = team_id
+      team_repo[:repository] = team_repo[:name]
+
+      team_repo[:permission] =
+        if team_repo[:permissions][:admin]
+          'admin'
+        elsif team_repo[:permissions][:push]
+          'push'
+        elsif team_repo[:permissions][:pull]
+          'pull'
+        end
+    end
+  end
+end

lib/geoengineer/utils/github_client.rb

@@ -0,0 +1,31 @@
+########################################################################
+# GithubClient exposes a set of API calls to fetch data from GitHub.
+# The primary reason for centralizing them here is testing and stubbing.
+########################################################################
+class GithubClient
+  Octokit.auto_paginate = true
+
+  def self.organization_members(*args)
+    Octokit.organization_members(*args).map(&:to_h)
+  end
+
+  def self.organization_repositories(*args)
+    Octokit.organization_repositories(*args).map(&:to_h)
+  end
+
+  def self.organization_teams(*args)
+    Octokit.organization_teams(*args).map(&:to_h)
+  end
+
+  def self.repository_collaborators(*args)
+    Octokit.collaborators(*args).map(&:to_h)
+  end
+
+  def self.team_memberships(*args)
+    Octokit.team_members(*args).map(&:to_h)
+  end
+
+  def self.team_repositories(*args)
+    Octokit.team_repositories(*args).map(&:to_h)
+  end
+end