This repository has been archived by the owner. It is now read-only.
Permalink
Browse files

Upgrade all FastlaneBuildRunner -> RemoteRunner (#1093)

this PR migrates all of the existing FastlaneBuildRunners to use the RemoteRunner.

The RemoteRunner is a distributed build agent that uses GRPC to server requests.
The responses get fanned out to websocket clients via `FastlaneCI::Notifications`
  • Loading branch information...
snatchev committed Jul 25, 2018
1 parent 41ae8c7 commit db06b98f6f9403ffe1a68a7dd873a7a04a79bda8
View
@@ -156,7 +156,7 @@ GEM
mime-types (~> 3.0)
representable (~> 3.0)
retriable (>= 2.0, < 4.0)
google-protobuf (3.5.1.2)
google-protobuf (3.6.0)
googleapis-common-protos-types (1.0.1)
google-protobuf (~> 3.0)
googleauth (0.6.2)
@@ -167,10 +167,9 @@ GEM
multi_json (~> 1.11)
os (~> 0.9)
signet (~> 0.7)
grpc (1.12.0)
grpc (1.13.0)
google-protobuf (~> 3.1)
googleapis-common-protos-types (~> 1.0.0)
googleauth (>= 0.5.1, < 0.7)
grpc-tools (1.12.0)
hashdiff (0.3.7)
highline (1.7.10)
@@ -98,6 +98,10 @@ def jwt
authorization = request.env["HTTP_AUTHORIZATION"]
bearer_token = authorization && authorization.slice(7..-1) # strip off the `Bearer `
# give the option to pass the bearer token as a query param.
# this is used when making websocket connections which do not allow us to set http headers
bearer_token ||= request.params["bearer_token"]
payload, _header = JWT.decode(
bearer_token,
settings.jwt_secret,
@@ -1,14 +1,16 @@
require_relative "api_controller"
require_relative "../features/build_runner/remote_runner"
require_relative "./view_models/build_summary_view_model"
require_relative "./view_models/build_view_model"
require "faye/websocket"
Faye::WebSocket.load_adapter("thin")
module FastlaneCI
# Controller for providing all data relating to builds
class BuildJSONController < APIController
HOME = "/data/projects/:project_id/build"
use(FastlaneCI::BuildWebsocketBackend)
def self.build_url(project_id:, build_number:)
return "/project/#{project_id}/build/#{build_number}"
end
@@ -84,33 +86,79 @@ def self.build_url(project_id:, build_number:)
json(build_summary_view_model)
end
get "#{HOME}/:build_number/logs" do |project_id, build_number|
# `current_build_runner` is only defined if the build was just run a while back
# if the server was restarted, we're gonna end here in this code block
build_log_artifact = current_build.artifacts.find do |current_artifact|
# We can improve the detection in the future, to actually mark an artifact as "default output"
current_artifact.type.include?("log") && current_artifact.reference.end_with?("fastlane.log")
get "#{HOME}/:build_number/log.ws" do |project_id, build_number|
halt(415, "unsupported media type") unless Faye::WebSocket.websocket?(request.env)
ws = Faye::WebSocket.new(request.env, nil, { ping: 30 })
ws.on(:open) do |event|
logger.debug([:open, ws.object_id])
## handle the case that the build completes, and runner.log is saved.
current_project = FastlaneCI::Services.project_service.project_by_id(project_id)
current_build = current_project.builds.find { |b| b.number == build_number.to_i }
build_log_artifact = current_build.artifacts.find do |current_artifact|
# We can improve the detection in the future, to actually mark an artifact as "default output"
current_artifact.type.include?("log") && current_artifact.reference.end_with?("runner.log")
end
if build_log_artifact
logger.debug("streaming back artifact: #{build_log_artifact.reference}")
File.open(build_log_artifact.reference, "r") do |file|
file.each_line do |line|
ws.send(convert_ansi_to_plain_text(line.chomp))
end
end
ws.close(1000, "runner complete.")
next
end
## if we have no runner.log, then check to see if the build_runner is still working.
current_build_runner = Services.build_runner_service.find_build_runner(
project_id: project_id,
build_number: build_number.to_i
)
if current_build_runner.nil?
ws.close(1000, "no runner found for project #{project_id} and build #{build_number}.")
next
end
# if the build runner has already completed, we can close the connection, and do not proceed
if current_build_runner.completed?
ws.close(1000, "runner complete.")
next
end
# once the build runner completes, close the websocket connection.
current_build_runner.on_complete do
ws.close(1000, "runner complete.")
end
# subscribe the current socket to events from the remote_runner
# as soon as a subscriber is returned, they will receive all historical items as well.
@subscriber = current_build_runner.subscribe do |_topic, payload|
ws.send(convert_ansi_to_plain_text(JSON.dump(payload)))
end
end
if build_log_artifact
# TODO: This only works for local storage. Add External storage support (ex. Google Cloud Storage)
artifact_file_content = File.read(build_log_artifact.provider.retrieve!(artifact: build_log_artifact))
else
json_error!(
error_message: "Logs file missing for build #{build_number}",
error_key: "Build.LogsMissing",
error_code: 404
ws.on(:close) do |event|
logger.debug([:close, ws.object_id, event.code, event.reason])
current_build_runner = Services.build_runner_service.find_build_runner(
project_id: project_id,
build_number: build_number.to_i
)
end
next if current_build_runner.nil?
current_build_runner.unsubscribe(@subscriber)
plain_artifact_file_content = convert_ansi_to_plain_text(artifact_file_content)
log_array = plain_artifact_file_content.split("\n").collect do |log_line|
{
message: log_line
}
Services.build_runner_service.remove_build_runner(build_runner: current_build_runner)
end
return json(log_array)
# Return async Rack response
return ws.rack_response
end
def current_build
@@ -57,8 +57,6 @@ class BuildViewModel
attr_reader :artifacts
def initialize(build:)
raise "Incorrect object type. Expected Build, got #{build.class}" unless build.kind_of?(Build)
@project_id = build.project.id
@number = build.number
@status = build.status
Oops, something went wrong.

0 comments on commit db06b98

Please sign in to comment.