Write job to extract Github project data and upload to database

.gitattributes

@@ -3,5 +3,8 @@
 # Mark the database schema as having been generated.
 db/schema.rb linguist-generated
 
+# Mark graphql schemas as having been generated
+db/*_schema.json linguist-generated
+
 # Mark any vendored files as having been vendored.
 vendor/* linguist-vendored

.gitignore

@@ -37,3 +37,5 @@
 .rubocop-https*
 
 /coverage
+
+.byebug_history

Gemfile

@@ -13,9 +13,11 @@ gem 'github_webhook', '~> 1.4'
 gem 'globalid'
 gem 'good_job', '~> 3.12'
 gem 'graphql'
+gem 'graphql-client'
 gem 'importmap-rails'
 gem 'jbuilder'
 gem 'kaminari'
+gem 'open-uri'
 gem 'pg', '~> 1.1'
 gem 'puma', '~> 5.6'
 gem 'rack-cors'

Gemfile.lock

@@ -96,6 +96,7 @@ GEM
     crack (0.4.5)
       rexml
     crass (1.0.6)
+    date (3.3.3)
     diff-lcs (1.5.0)
     docile (1.4.0)
     dotenv (2.8.1)
@@ -134,6 +135,9 @@ GEM
       thor (>= 0.14.1)
       webrick (>= 1.3)
     graphql (2.0.17)
+    graphql-client (0.18.0)
+      activesupport (>= 3.0)
+      graphql
     hashdiff (1.0.1)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
@@ -180,6 +184,10 @@ GEM
       racc (~> 1.4)
     nokogiri (1.14.1-x86_64-linux)
       racc (~> 1.4)
+    open-uri (0.3.0)
+      stringio
+      time
+      uri
     parallel (1.22.1)
     parser (3.1.3.0)
       ast (~> 2.4.1)
@@ -282,17 +290,21 @@ GEM
       concurrent-ruby (~> 1.0, >= 1.0.2)
     shoulda-matchers (5.2.0)
       activesupport (>= 5.2.0)
-    simplecov (0.21.2)
+    simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
+    stringio (3.0.5)
     thor (1.2.1)
+    time (0.2.1)
+      date
     timeout (0.3.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.3.0)
+    uri (0.12.0)
     webmock (3.18.1)
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
@@ -320,9 +332,11 @@ DEPENDENCIES
   globalid
   good_job (~> 3.12)
   graphql
+  graphql-client
   importmap-rails
   jbuilder
   kaminari
+  open-uri
   pg (~> 1.1)
   pry-byebug
   puma (~> 5.6)

app/controllers/github_webhooks_controller.rb

@@ -4,7 +4,7 @@ class GithubWebhooksController < ActionController::API
   include GithubWebhook::Processor
 
   def github_push(payload)
-    UploadJob.perform_later if payload[:ref] == ENV.fetch('GITHUB_WEBHOOK_REF') && edited_code?(payload)
+    UploadJob.perform_later(payload) if payload[:ref] == ENV.fetch('GITHUB_WEBHOOK_REF') && edited_code?(payload)
   end
 
   private

app/jobs/upload_job.rb

@@ -1,12 +1,87 @@
 # frozen_string_literal: true
 
+require 'open-uri'
+require 'project_importer'
+require 'github_api'
+
 class UploadJob < ApplicationJob
-  # Automatically retry jobs that encountered a deadlock
-  # retry_on ActiveRecord::Deadlocked
+  ProjectContentQuery = GithubApi::Client.parse <<-'GRAPHQL'
+    query($owner: String!, $repository: String!, $expression: String!) {
+      repository(owner: $owner, name: $repository) {
+        object(expression: $expression) {
+          ... on Tree {
+            entries {
+              name
+              object {
+                ... on Tree {
+                  entries {
+                    name
+                    extension
+                    object {
+                      ... on Blob {
+                        text
+                        commitResourcePath
+                        isBinary
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  GRAPHQL
+
+  def perform(payload)
+    repository = payload[:repository][:name]
+    owner = payload[:repository][:owner][:name]
+    projects_data = load_projects_data(repository, owner)
+
+    projects_data.data.repository.object.entries.each do |project_dir|
+      project = format_project(project_dir, repository, owner)
+      project_importer = ProjectImporter.new(**project)
+      project_importer.import!
+    end
+  end
+
+  private
+
+  def load_projects_data(repository, owner)
+    GithubApi::Client.query(
+      ProjectContentQuery,
+      variables: { repository:, owner:, expression: "#{ENV.fetch('GITHUB_WEBHOOK_REF')}:en/code" }
+    )
+  end
+
+  def format_project(project_dir, repository, owner)
+    components = []
+    images = []
+    project_dir.object.entries.each do |file|
+      if file.name == 'project_config.yml'
+        @proj_config = YAML.safe_load(file.object.text, symbolize_names: true)
+      elsif file.object.text
+        components << component(file)
+      else
+        images << image(file, project_dir, repository, owner)
+      end
+    end
+    { **@proj_config, components:, images: }
+  end
+
+  def component(file)
+    name = file.name.chomp(file.extension)
+    extension = file.extension[1..]
+    content = file.object.text
+    default = file.name == 'main.py'
+    { name:, extension:, content:, default: }
+  end
 
-  # Most jobs are safe to ignore if the underlying records are no longer available
-  # discard_on ActiveJob::DeserializationError
-  def perform
-    # puts 'hello world'
+  def image(file, project_dir, repository, owner)
+    filename = file.name
+    directory = project_dir.name
+    url = "https://github.com/#{owner}/#{repository}/raw/#{ENV.fetch('GITHUB_WEBHOOK_REF')}/en/code/#{directory}/#{filename}"
+    { filename:, io: URI.parse(url).open }
   end
 end

config/initializers/github_api.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'github_api'
+require 'graphql/client/log_subscriber'
+
+Rails.application.configure do
+  config.github_api = ActiveSupport::OrderedOptions.new
+  config.github_api.client = GithubApi::Client
+
+  GraphQL::Client::LogSubscriber.attach_to :github_api
+end