Merge pull request #74 from eagletmt/503-on-maint

Return 503 in maintenance mode when mysql2 error occurs

app/controllers/barbeque/api/job_executions_controller.rb

@@ -1,3 +1,5 @@
+require 'barbeque/maintenance'
+
 class Barbeque::Api::JobExecutionsController < Barbeque::Api::ApplicationController
   include Garage::RestfulActions
 
@@ -22,6 +24,13 @@ def require_resources
   def require_resource
     model = Barbeque::JobExecution.find_or_initialize_by(message_id: params[:message_id])
     @resource = Barbeque::Api::JobExecutionResource.new(model, url_options)
+  rescue ActiveRecord::StatementInvalid, Mysql2::Error::ConnectionError => e
+    if Barbeque::Maintenance.database_maintenance_mode?
+      Barbeque::ExceptionHandler.handle_exception(e)
+      @resource = Barbeque::Api::DatabaseMaintenanceResource.new(e)
+    else
+      raise e
+    end
   end
 
   def create_resource
@@ -49,4 +58,12 @@ def location
       nil
     end
   end
+
+  def respond_with_resource_options
+    if @resource.is_a?(Barbeque::Api::DatabaseMaintenanceResource)
+      super.merge(status: 503)
+    else
+      super
+    end
+  end
 end

app/models/barbeque/api/database_maintenance_resource.rb

@@ -0,0 +1,11 @@
+class Barbeque::Api::DatabaseMaintenanceResource
+  include Garage::Representer
+
+  property :message
+
+  delegate :message, to: :@exception
+
+  def initialize(exception)
+    @exception = exception
+  end
+end

app/models/barbeque/job_queue.rb

@@ -1,3 +1,5 @@
+require 'barbeque/maintenance'
+
 class Barbeque::JobQueue < Barbeque::ApplicationRecord
   SQS_NAME_PREFIX = ENV['BARBEQUE_SQS_NAME_PREFIX'] || 'Barbeque-'
   SQS_NAME_MAX_LENGTH = 80
@@ -26,15 +28,10 @@ def sqs_queue_name
   # @param name [String] queue name in Barbeque
   # @return [String] queue URL of SQS
   def self.queue_url_from_name(name)
-    if database_maintenance_mode?
+    if Barbeque::Maintenance.database_maintenance_mode?
       "https://sqs.#{ENV.fetch('AWS_REGION')}.amazonaws.com/#{ENV.fetch('AWS_ACCOUNT_ID')}/#{SQS_NAME_PREFIX}#{name}"
     else
       select(:queue_url).find_by!(name: name).queue_url
     end
   end
-
-  def self.database_maintenance_mode?
-    ENV['BARBEQUE_DATABASE_MAINTENANCE'] == '1' && ENV['AWS_REGION'].present? && ENV['AWS_ACCOUNT_ID'].present?
-  end
-  private_class_method :database_maintenance_mode?
 end

doc/api/job_executions.md

@@ -5,7 +5,7 @@ Shows a status of a job_execution.
 
 #### Request
 ```
-GET /v1/job_executions/44b34792-99c2-4165-b0a7-97ba8cb67701 HTTP/1.1
+GET /v1/job_executions/a05ba839-aaab-4aeb-9bb6-fa7af3d870a9 HTTP/1.1
 Accept: application/json
 Content-Length: 0
 Content-Type: application/json
@@ -18,14 +18,14 @@ HTTP/1.1 200
 Cache-Control: max-age=0, private, must-revalidate
 Content-Length: 81
 Content-Type: application/json; charset=utf-8
-ETag: W/"162d4526406d7cc92778f7c9a77770b5"
-X-Request-Id: 752ecffe-9fa0-471b-9c18-9da21f689b28
-X-Runtime: 0.012975
+ETag: W/"a6c4db4cdaf2feeb7f02f7842c36a2e2"
+X-Request-Id: a120cc28-4dbd-408a-872c-e6a4b6a8b7b9
+X-Runtime: 0.011375
 
 {
-  "message_id": "44b34792-99c2-4165-b0a7-97ba8cb67701",
+  "message_id": "a05ba839-aaab-4aeb-9bb6-fa7af3d870a9",
   "status": "success",
-  "id": 683
+  "id": 305
 }
 ```
 
@@ -36,7 +36,7 @@ Shows url to job_execution.
 
 #### Request
 ```
-GET /v1/job_executions/d54860cb-f374-4ef2-b19e-1662ce714c61?fields=__default__,html_url HTTP/1.1
+GET /v1/job_executions/d8f3aca9-eb94-42f1-88be-c5c1372c98f1?fields=__default__,html_url HTTP/1.1
 Accept: application/json
 Content-Length: 0
 Content-Type: application/json
@@ -49,15 +49,15 @@ HTTP/1.1 200
 Cache-Control: max-age=0, private, must-revalidate
 Content-Length: 169
 Content-Type: application/json; charset=utf-8
-ETag: W/"83d62488e2a41e9531c68de74c956ab0"
-X-Request-Id: 2edf1ac6-cea2-42b7-a47a-b033c9981ff2
-X-Runtime: 0.002832
+ETag: W/"ed43007e8be846b3ae14a156b436f74c"
+X-Request-Id: f14551e7-dda9-48e0-8273-b8b95c64437b
+X-Runtime: 0.002641
 
 {
-  "message_id": "d54860cb-f374-4ef2-b19e-1662ce714c61",
+  "message_id": "d8f3aca9-eb94-42f1-88be-c5c1372c98f1",
   "status": "success",
-  "id": 684,
-  "html_url": "http://www.example.com/job_executions/d54860cb-f374-4ef2-b19e-1662ce714c61"
+  "id": 306,
+  "html_url": "http://www.example.com/job_executions/d8f3aca9-eb94-42f1-88be-c5c1372c98f1"
 }
 ```
 
@@ -68,7 +68,7 @@ Returns message of the job_execution.
 
 #### Request
 ```
-GET /v1/job_executions/ece0e578-a9ac-4eb5-ac8d-5a02270629c1?fields=__default__,message HTTP/1.1
+GET /v1/job_executions/d369e2ab-7795-4583-b395-9b1011ba92eb?fields=__default__,message HTTP/1.1
 Accept: application/json
 Content-Length: 0
 Content-Type: application/json
@@ -81,20 +81,48 @@ HTTP/1.1 200
 Cache-Control: max-age=0, private, must-revalidate
 Content-Length: 111
 Content-Type: application/json; charset=utf-8
-ETag: W/"03c274fb6c9afa0dc90cd8eec1e77dcf"
-X-Request-Id: fd15b690-9a0d-457c-b672-9721b28d5bca
-X-Runtime: 0.005220
+ETag: W/"a7ab894eb65c3315a838124226eb3975"
+X-Request-Id: e95da0fb-ef37-4ea9-9d08-3e512d47e482
+X-Runtime: 0.002996
 
 {
-  "message_id": "ece0e578-a9ac-4eb5-ac8d-5a02270629c1",
+  "message_id": "d369e2ab-7795-4583-b395-9b1011ba92eb",
   "status": "success",
-  "id": 685,
+  "id": 307,
   "message": {
     "recipe_id": 12345
   }
 }
 ```
 
+## GET /v1/job_executions/:message_id
+Returns error message.
+
+### Example
+
+#### Request
+```
+GET /v1/job_executions/b46e9fd7-e2dd-461b-9476-1bb37e643644 HTTP/1.1
+Accept: application/json
+Content-Length: 0
+Content-Type: application/json
+Host: www.example.com
+```
+
+#### Response
+```
+HTTP/1.1 503
+Cache-Control: no-cache
+Content-Length: 237
+Content-Type: application/json; charset=utf-8
+X-Request-Id: 05c0ece4-ea68-4489-a0f5-18d63049418b
+X-Runtime: 0.001956
+
+{
+  "message": "Mysql2::Error::ConnectionError: Can't connect to MySQL server: SELECT  `barbeque_job_executions`.* FROM `barbeque_job_executions` WHERE `barbeque_job_executions`.`message_id` = 'b46e9fd7-e2dd-461b-9476-1bb37e643644' LIMIT 1"
+}
+```
+
 ## POST /v2/job_executions
 Enqueues a job execution.
 
@@ -118,7 +146,7 @@ Host: www.example.com
 {
   "application": "blog",
   "job": "NotifyAuthor",
-  "queue": "queue-102",
+  "queue": "queue-104",
   "message": {
     "recipe_id": 1
   }
@@ -131,12 +159,12 @@ HTTP/1.1 201
 Cache-Control: max-age=0, private, must-revalidate
 Content-Length: 82
 Content-Type: application/json; charset=utf-8
-ETag: W/"413ca32999e1a97bebde87741a53d078"
-X-Request-Id: 8bc996f6-2b20-4bf6-a7f7-0980f796f0aa
-X-Runtime: 0.002134
+ETag: W/"35bd09a003bb4f7fb8b0e2b755fb76b0"
+X-Request-Id: d01d9be1-de8f-4818-aa86-f6bd8e7da8c7
+X-Runtime: 0.001690
 
 {
-  "message_id": "e91b56d8-30a9-4331-93f6-1296216c7407",
+  "message_id": "ac34af9b-f26a-4674-983d-69ddb29451e6",
   "status": "pending",
   "id": null
 }
@@ -165,7 +193,7 @@ Host: www.example.com
 {
   "application": "blog",
   "job": "NotifyAuthor",
-  "queue": "queue-105",
+  "queue": "queue-107",
   "message": {
     "recipe_id": 1
   },
@@ -179,12 +207,12 @@ HTTP/1.1 201
 Cache-Control: max-age=0, private, must-revalidate
 Content-Length: 82
 Content-Type: application/json; charset=utf-8
-ETag: W/"dabbb98b9040a53f74a93682f6a83ccd"
-X-Request-Id: b58503df-873d-4544-868a-021ffdcde58a
-X-Runtime: 0.004408
+ETag: W/"b369de6c5fcf19825ef21650cc312f49"
+X-Request-Id: 4840803e-2cce-4818-a9b3-a6c97a469b8e
+X-Runtime: 0.001685
 
 {
-  "message_id": "8f5cbe66-8dee-433e-ba90-30cbf36eddfe",
+  "message_id": "44ac17fa-6df7-4af9-9e1b-670b1d585b0e",
   "status": "pending",
   "id": null
 }

doc/api/job_retries.md

@@ -8,7 +8,7 @@ Enqueues a message to retry a specified message.
 
 #### Request
 ```
-POST /v1/job_executions/7d659a22-b53e-4137-8f70-6416a1ce1c34/retries HTTP/1.1
+POST /v1/job_executions/8b861a99-a45b-4720-851a-805c1bc70287/retries HTTP/1.1
 Accept: application/json
 Content-Length: 0
 Content-Type: application/json
@@ -21,12 +21,12 @@ HTTP/1.1 201
 Cache-Control: max-age=0, private, must-revalidate
 Content-Length: 72
 Content-Type: application/json; charset=utf-8
-ETag: W/"352714e6aa30cf531412bbaf085e21fb"
-X-Request-Id: 14df67d9-6d10-46e8-b86d-0e8748387035
-X-Runtime: 0.006126
+ETag: W/"e03d4f08be4c8b4dfd18839dd48708e8"
+X-Request-Id: 9c2a3b6f-3888-47da-bc2a-d75137af73bc
+X-Runtime: 0.004713
 
 {
-  "message_id": "e5dec945-952a-4256-afe4-b728430a9b75",
+  "message_id": "af18541e-9668-4eb2-ac5b-1899dbbe7e1b",
   "status": "pending"
 }
 ```

doc/api/revision_locks.md

@@ -8,7 +8,7 @@ Updates a tag of docker_image.
 
 #### Request
 ```
-POST /v1/apps/app-105/revision_lock HTTP/1.1
+POST /v1/apps/app-107/revision_lock HTTP/1.1
 Accept: application/json
 Content-Length: 55
 Content-Type: application/json
@@ -26,8 +26,8 @@ Cache-Control: max-age=0, private, must-revalidate
 Content-Length: 55
 Content-Type: application/json; charset=utf-8
 ETag: W/"a3f0b3d8c32ee1e318357b5276e50a3c"
-X-Request-Id: aec3ef82-1987-45c3-9f00-1b400bcaa171
-X-Runtime: 0.009411
+X-Request-Id: 0a4f9e4b-d80e-4788-ad77-666152308941
+X-Runtime: 0.008506
 
 {
   "revision": "798926db1e623cd51245b70b1f1acb40d780ddc1"
@@ -41,7 +41,7 @@ Updates a tag of docker_image.
 
 #### Request
 ```
-DELETE /v1/apps/app-107/revision_lock HTTP/1.1
+DELETE /v1/apps/app-109/revision_lock HTTP/1.1
 Accept: application/json
 Content-Length: 0
 Content-Type: application/json
@@ -52,6 +52,6 @@ Host: www.example.com
 ```
 HTTP/1.1 204
 Cache-Control: no-cache
-X-Request-Id: 016e5584-500a-4e9f-aa08-d5d8e711dce3
-X-Runtime: 0.010075
+X-Request-Id: 29434e92-0539-40cc-b38d-011e4d0c6347
+X-Runtime: 0.006990
 ```

doc/toc.md

@@ -3,6 +3,7 @@
  * [GET /v1/job_executions/:message_id](api/job_executions.md#get-v1job_executionsmessage_id)
  * [GET /v1/job_executions/:message_id](api/job_executions.md#get-v1job_executionsmessage_id-1)
  * [GET /v1/job_executions/:message_id](api/job_executions.md#get-v1job_executionsmessage_id-2)
+ * [GET /v1/job_executions/:message_id](api/job_executions.md#get-v1job_executionsmessage_id-3)
  * [POST /v2/job_executions](api/job_executions.md#post-v2job_executions)
  * [POST /v2/job_executions](api/job_executions.md#post-v2job_executions-1)
 * [api/job_retries.md](api/job_retries.md)

lib/barbeque/maintenance.rb

@@ -0,0 +1,7 @@
+module Barbeque
+  module Maintenance
+    def self.database_maintenance_mode?
+      ENV['BARBEQUE_DATABASE_MAINTENANCE'] == '1' && ENV['AWS_REGION'].present? && ENV['AWS_ACCOUNT_ID'].present?
+    end
+  end
+end

spec/requests/api/job_executions_spec.rb

@@ -90,6 +90,45 @@
         end
       end
     end
+
+    context 'when database maintenance mode' do
+      around do |example|
+        env = ENV.to_h
+        ENV['BARBEQUE_DATABASE_MAINTENANCE'] = '1'
+        ENV['AWS_REGION'] = 'ap-northeast-1'
+        ENV['AWS_ACCOUNT_ID'] = '123456789012'
+        example.run
+        ENV.replace(env)
+      end
+
+      let!(:job_execution) { FactoryBot.create(:job_execution) }
+
+      context 'when database is available' do
+        it 'returns execution status' do
+          get "/v1/job_executions/#{job_execution.message_id}", env: env
+          expect(response).to have_http_status(200)
+          expect(result).to eq(
+            'message_id' => job_execution.message_id,
+            'status' => job_execution.status,
+            'id' => job_execution.id,
+          )
+        end
+      end
+
+      context 'when database is unavailable', :autodoc do
+        before do
+          allow_any_instance_of(Mysql2::Client).to receive(:query).and_raise(Mysql2::Error::ConnectionError.new("Can't connect to MySQL server"))
+        end
+
+        it 'returns error message' do
+          get "/v1/job_executions/#{job_execution.message_id}", env: env
+          expect(response).to have_http_status(503)
+          expect(result).to match(
+            'message' => a_kind_of(String),
+          )
+        end
+      end
+    end
   end
 
   describe 'POST /v2/job_executions' do