From b4e9d6e5b99af2eebe3e188372d11974932a7490 Mon Sep 17 00:00:00 2001 From: loiswells97 Date: Thu, 9 Feb 2023 16:42:53 +0000 Subject: [PATCH 1/5] Create draft PR for #116 From ba335bde96de42588fb5bf6cae43bd8e41a928b0 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Fri, 10 Feb 2023 09:25:56 +0000 Subject: [PATCH 2/5] Add in good job --- Gemfile | 2 + Gemfile.lock | 16 +++++ Procfile | 3 +- app/controllers/github_webhooks_controller.rb | 6 +- app/jobs/upload_job.rb | 12 ++++ config/application.rb | 2 + db/migrate/20230209171350_create_good_jobs.rb | 66 +++++++++++++++++++ db/schema.rb | 61 ++++++++++++++++- release_process.sh | 2 + 9 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 app/jobs/upload_job.rb create mode 100644 db/migrate/20230209171350_create_good_jobs.rb create mode 100644 release_process.sh diff --git a/Gemfile b/Gemfile index 27ac6b5a4..904692e2a 100644 --- a/Gemfile +++ b/Gemfile @@ -37,3 +37,5 @@ group :test do gem 'shoulda-matchers', '~> 5.0' gem 'webmock' end + +gem "good_job", "~> 3.12" diff --git a/Gemfile.lock b/Gemfile.lock index 8988f4aa8..e3e750ee7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -102,6 +102,8 @@ GEM dotenv (= 2.8.1) railties (>= 3.2) erubi (1.11.0) + et-orbi (1.2.7) + tzinfo factory_bot (6.2.1) activesupport (>= 5.0.0) factory_bot_rails (6.2.0) @@ -113,12 +115,23 @@ GEM faraday-net_http (>= 2.0, < 3.1) ruby2_keywords (>= 0.0.4) faraday-net_http (3.0.2) + fugit (1.8.1) + et-orbi (~> 1, >= 1.2.7) + raabro (~> 1.4) github_webhook (1.4.2) activesupport (>= 4) rack (>= 1.3) railties (>= 4) globalid (1.0.0) activesupport (>= 5.0) + good_job (3.12.1) + activejob (>= 6.0.0) + activerecord (>= 6.0.0) + concurrent-ruby (>= 1.0.2) + fugit (>= 1.1) + railties (>= 6.0.0) + thor (>= 0.14.1) + webrick (>= 1.3) hashdiff (1.0.1) i18n (1.12.0) concurrent-ruby (~> 1.0) @@ -178,6 +191,7 @@ GEM public_suffix (5.0.0) puma (5.6.5) nio4r (~> 2.0) + raabro (1.4.0) racc (1.6.1) rack (2.2.4) rack-cors (1.1.1) @@ -279,6 +293,7 @@ GEM addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) + webrick (1.8.1) websocket-driver (0.7.5) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) @@ -297,6 +312,7 @@ DEPENDENCIES faker faraday github_webhook (~> 1.4) + good_job (~> 3.12) importmap-rails jbuilder kaminari diff --git a/Procfile b/Procfile index 77fe2d7d8..6c4842bae 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,3 @@ web: bundle exec puma -C config/puma.rb -release: bundle exec rails projects:create_starter +release: ./release_process.sh +worker: bundle exec good_job start --max-threads=8 diff --git a/app/controllers/github_webhooks_controller.rb b/app/controllers/github_webhooks_controller.rb index 5d5422623..08cadf26d 100644 --- a/app/controllers/github_webhooks_controller.rb +++ b/app/controllers/github_webhooks_controller.rb @@ -3,8 +3,10 @@ class GithubWebhooksController < ActionController::API include GithubWebhook::Processor - def github_push(_payload) - # TODO: handle push webhook + def github_push(payload) + if payload['ref'] == ENV.fetch('GITHUB_WEBHOOK_REF') + UploadJob.perform_later + end head :ok end diff --git a/app/jobs/upload_job.rb b/app/jobs/upload_job.rb new file mode 100644 index 000000000..c53d7dc10 --- /dev/null +++ b/app/jobs/upload_job.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +class UploadJob < ApplicationJob + # Automatically retry jobs that encountered a deadlock + # retry_on ActiveRecord::Deadlocked + + # Most jobs are safe to ignore if the underlying records are no longer available + # discard_on ActiveJob::DeserializationError + def perform + puts 'hello world' + end +end diff --git a/config/application.rb b/config/application.rb index 6c82f7953..f329af37a 100644 --- a/config/application.rb +++ b/config/application.rb @@ -43,5 +43,7 @@ class Application < Rails::Application config.generators do |g| g.orm :active_record, primary_key_type: :uuid end + + config.active_job.queue_adapter = :good_job end end diff --git a/db/migrate/20230209171350_create_good_jobs.rb b/db/migrate/20230209171350_create_good_jobs.rb new file mode 100644 index 000000000..a0b60d888 --- /dev/null +++ b/db/migrate/20230209171350_create_good_jobs.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true +class CreateGoodJobs < ActiveRecord::Migration[7.0] + def change + enable_extension 'pgcrypto' + + create_table :good_jobs, id: :uuid do |t| + t.text :queue_name + t.integer :priority + t.jsonb :serialized_params + t.datetime :scheduled_at + t.datetime :performed_at + t.datetime :finished_at + t.text :error + + t.timestamps + + t.uuid :active_job_id + t.text :concurrency_key + t.text :cron_key + t.uuid :retried_good_job_id + t.datetime :cron_at + + t.uuid :batch_id + t.uuid :batch_callback_id + end + + create_table :good_job_batches, id: :uuid do |t| + t.timestamps + t.text :description + t.jsonb :serialized_properties + t.text :on_finish + t.text :on_success + t.text :on_discard + t.text :callback_queue_name + t.integer :callback_priority + t.datetime :enqueued_at + t.datetime :discarded_at + t.datetime :finished_at + end + + create_table :good_job_processes, id: :uuid do |t| + t.timestamps + t.jsonb :state + end + + create_table :good_job_settings, id: :uuid do |t| + t.timestamps + t.text :key + t.jsonb :value + t.index :key, unique: true + end + + add_index :good_jobs, :scheduled_at, where: "(finished_at IS NULL)", name: "index_good_jobs_on_scheduled_at" + add_index :good_jobs, [:queue_name, :scheduled_at], where: "(finished_at IS NULL)", name: :index_good_jobs_on_queue_name_and_scheduled_at + add_index :good_jobs, [:active_job_id, :created_at], name: :index_good_jobs_on_active_job_id_and_created_at + add_index :good_jobs, :concurrency_key, where: "(finished_at IS NULL)", name: :index_good_jobs_on_concurrency_key_when_unfinished + add_index :good_jobs, [:cron_key, :created_at], name: :index_good_jobs_on_cron_key_and_created_at + add_index :good_jobs, [:cron_key, :cron_at], name: :index_good_jobs_on_cron_key_and_cron_at, unique: true + add_index :good_jobs, [:active_job_id], name: :index_good_jobs_on_active_job_id + add_index :good_jobs, [:finished_at], where: "retried_good_job_id IS NULL AND finished_at IS NOT NULL", name: :index_good_jobs_jobs_on_finished_at + add_index :good_jobs, [:priority, :created_at], order: { priority: "DESC NULLS LAST", created_at: :asc }, + where: "finished_at IS NULL", name: :index_good_jobs_jobs_on_priority_created_at_when_unfinished + add_index :good_jobs, [:batch_id], where: "batch_id IS NOT NULL" + add_index :good_jobs, [:batch_callback_id], where: "batch_callback_id IS NOT NULL" + end +end diff --git a/db/schema.rb b/db/schema.rb index 468f8e243..0f2b14b8a 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2022_03_11_121518) do +ActiveRecord::Schema[7.0].define(version: 2023_02_09_171350) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -56,6 +56,65 @@ t.index ["project_id"], name: "index_components_on_project_id" end + create_table "good_job_batches", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.text "description" + t.jsonb "serialized_properties" + t.text "on_finish" + t.text "on_success" + t.text "on_discard" + t.text "callback_queue_name" + t.integer "callback_priority" + t.datetime "enqueued_at" + t.datetime "discarded_at" + t.datetime "finished_at" + end + + create_table "good_job_processes", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.jsonb "state" + end + + create_table "good_job_settings", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.text "key" + t.jsonb "value" + t.index ["key"], name: "index_good_job_settings_on_key", unique: true + end + + create_table "good_jobs", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.text "queue_name" + t.integer "priority" + t.jsonb "serialized_params" + t.datetime "scheduled_at" + t.datetime "performed_at" + t.datetime "finished_at" + t.text "error" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.uuid "active_job_id" + t.text "concurrency_key" + t.text "cron_key" + t.uuid "retried_good_job_id" + t.datetime "cron_at" + t.uuid "batch_id" + t.uuid "batch_callback_id" + t.index ["active_job_id", "created_at"], name: "index_good_jobs_on_active_job_id_and_created_at" + t.index ["active_job_id"], name: "index_good_jobs_on_active_job_id" + t.index ["batch_callback_id"], name: "index_good_jobs_on_batch_callback_id", where: "(batch_callback_id IS NOT NULL)" + t.index ["batch_id"], name: "index_good_jobs_on_batch_id", where: "(batch_id IS NOT NULL)" + t.index ["concurrency_key"], name: "index_good_jobs_on_concurrency_key_when_unfinished", where: "(finished_at IS NULL)" + t.index ["cron_key", "created_at"], name: "index_good_jobs_on_cron_key_and_created_at" + t.index ["cron_key", "cron_at"], name: "index_good_jobs_on_cron_key_and_cron_at", unique: true + t.index ["finished_at"], name: "index_good_jobs_jobs_on_finished_at", where: "((retried_good_job_id IS NULL) AND (finished_at IS NOT NULL))" + t.index ["priority", "created_at"], name: "index_good_jobs_jobs_on_priority_created_at_when_unfinished", order: { priority: "DESC NULLS LAST" }, where: "(finished_at IS NULL)" + t.index ["queue_name", "scheduled_at"], name: "index_good_jobs_on_queue_name_and_scheduled_at", where: "(finished_at IS NULL)" + t.index ["scheduled_at"], name: "index_good_jobs_on_scheduled_at", where: "(finished_at IS NULL)" + end + create_table "projects", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.uuid "user_id" t.string "name" diff --git a/release_process.sh b/release_process.sh new file mode 100644 index 000000000..14739e3a7 --- /dev/null +++ b/release_process.sh @@ -0,0 +1,2 @@ +bundle exec rails db:migrate:with_data +bundle exec rails projects:create_starter From e955b09677cf0fcdec06d7024266a07b65ee9f95 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Fri, 10 Feb 2023 09:30:59 +0000 Subject: [PATCH 3/5] fixing rubocop --- Gemfile | 2 +- app/controllers/github_webhooks_controller.rb | 4 +--- app/jobs/upload_job.rb | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Gemfile b/Gemfile index 904692e2a..adb5f433d 100644 --- a/Gemfile +++ b/Gemfile @@ -38,4 +38,4 @@ group :test do gem 'webmock' end -gem "good_job", "~> 3.12" +gem 'good_job', '~> 3.12' diff --git a/app/controllers/github_webhooks_controller.rb b/app/controllers/github_webhooks_controller.rb index 08cadf26d..0a71e0b7d 100644 --- a/app/controllers/github_webhooks_controller.rb +++ b/app/controllers/github_webhooks_controller.rb @@ -4,9 +4,7 @@ class GithubWebhooksController < ActionController::API include GithubWebhook::Processor def github_push(payload) - if payload['ref'] == ENV.fetch('GITHUB_WEBHOOK_REF') - UploadJob.perform_later - end + UploadJob.perform_later if payload['ref'] == ENV.fetch('GITHUB_WEBHOOK_REF') head :ok end diff --git a/app/jobs/upload_job.rb b/app/jobs/upload_job.rb index c53d7dc10..bef0f069b 100644 --- a/app/jobs/upload_job.rb +++ b/app/jobs/upload_job.rb @@ -7,6 +7,6 @@ class UploadJob < ApplicationJob # Most jobs are safe to ignore if the underlying records are no longer available # discard_on ActiveJob::DeserializationError def perform - puts 'hello world' + # puts 'hello world' end end From 7176fb01aa2d6bb1d035a283092a17537962c2ce Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Fri, 10 Feb 2023 10:48:14 +0000 Subject: [PATCH 4/5] fixing procfile --- Procfile | 2 +- release_process.sh | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) delete mode 100644 release_process.sh diff --git a/Procfile b/Procfile index 6c4842bae..2da4ddc90 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,3 @@ web: bundle exec puma -C config/puma.rb -release: ./release_process.sh +release: bundle exec rails db:migrate:with_data projects:create_starter worker: bundle exec good_job start --max-threads=8 diff --git a/release_process.sh b/release_process.sh deleted file mode 100644 index 14739e3a7..000000000 --- a/release_process.sh +++ /dev/null @@ -1,2 +0,0 @@ -bundle exec rails db:migrate:with_data -bundle exec rails projects:create_starter From 679e780f4fde51645eaa00bb61bf20ce846f2bc5 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Fri, 10 Feb 2023 10:55:30 +0000 Subject: [PATCH 5/5] trying to get the procfile working --- Procfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Procfile b/Procfile index 2da4ddc90..493abea56 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,3 @@ web: bundle exec puma -C config/puma.rb -release: bundle exec rails db:migrate:with_data projects:create_starter +release: bundle exec rails db:migrate projects:create_starter worker: bundle exec good_job start --max-threads=8