Worker agent updates (#987)

Gemfile

@@ -64,6 +64,9 @@ gem "xcode-install", ">= 2.4.0", "< 3.0.0"
 # Interprocess communication
 gem "grpc", ">= 1.11.0", "< 2.0.0"
 
+# state machine for ruby objects
+gem "micromachine", ">= 3.0.0", "< 4.0.0"
+
 # Internal projects
 gem "fastfile-parser", git: "https://github.com/fastlane/fastfile-parser", require: false
 gem "fastlane", git: "https://github.com/fastlane/fastlane"
@@ -80,5 +83,6 @@ group :test, :development do
   gem "rack-test", require: "rack/test"
   gem "rake"
   gem "rspec"
+  gem "timecop", ">= 0.9.1", "< 1.0.0"
   gem "webmock", ">= 3.4.1", "< 3.5.0"
 end

Gemfile.lock

@@ -179,6 +179,7 @@ GEM
     memoizable (0.4.2)
       thread_safe (~> 0.3, >= 0.3.1)
     method_source (0.9.0)
+    micromachine (3.0.0)
     mime-types (3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2016.0521)
@@ -291,6 +292,7 @@ GEM
     thor (0.19.4)
     thread_safe (0.3.6)
     tilt (2.0.8)
+    timecop (0.9.1)
     tins (1.16.3)
     tty-color (0.4.2)
     tty-command (0.8.1)
@@ -353,6 +355,7 @@ DEPENDENCIES
   grpc (>= 1.11.0, < 2.0.0)
   grpc-tools
   jwt (>= 2.1.0, < 3.0.0)
+  micromachine (>= 3.0.0, < 4.0.0)
   octokit (>= 4.8.0, < 5.0.0)
   pry
   pry-byebug
@@ -366,6 +369,7 @@ DEPENDENCIES
   sinatra-flash
   taskqueue!
   thin (>= 1.7.2, < 2.0.0)
+  timecop (>= 0.9.1, < 1.0.0)
   tty-command (>= 0.7.0, < 1.0.0)
   webmock (>= 3.4.1, < 3.5.0)
   xcode-install (>= 2.4.0, < 3.0.0)

agent/agent.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "logger"
 require "grpc"
 # put ./protos in the load path. this is required because they are auto-generated and have specific `require` paths
 proto_path = File.expand_path("../protos", File.dirname(__FILE__))
@@ -15,6 +16,20 @@ module Agent
     VERSION = "0.0.0-alpha"
     HOST = "0.0.0.0"
     PORT = "8089"
+    NULL_CHAR = "\0"
     EOT_CHAR = "\4" # end-of-transmission character.
+
+    ##
+    # Logging module to expose the logger.
+    module Logging
+      def logger
+        return @logger if defined?(@logger)
+
+        @logger = Logger.new(STDOUT)
+        @logger.level = Logger::DEBUG
+
+        @logger
+      end
+    end
   end
 end

agent/client.rb

@@ -3,16 +3,45 @@
 module FastlaneCI
   module Agent
     ##
-    # A sample client that can be used to make a request to the server.
+    # A sample client that can be used to make a request to the service.
     class Client
       def initialize(host)
-        @stub = Stub.new("#{host}:#{PORT}", :this_channel_is_insecure)
+        @stub = Proto::Agent::Stub.new("#{host}:#{PORT}", :this_channel_is_insecure)
       end
 
       def request_spawn(bin, *params, env: {})
-        command = Command.new(bin: bin, parameters: params, env: env)
+        command = Proto::Command.new(bin: bin, parameters: params, env: env)
         @stub.spawn(command)
       end
+
+      def request_run_fastlane(bin, *params, env: {})
+        command = Proto::Command.new(bin: bin, parameters: params, env: env)
+        @stub.run_fastlane(Proto::InvocationRequest.new(command: command))
+      end
     end
   end
 end
+
+if $0 == __FILE__
+  client = FastlaneCI::Agent::Client.new("localhost")
+  env = {
+    "FASTLANE_CI_ARTIFACTS" => "artifacts",
+    "GIT_URL" => "https://github.com/snatchev/themoji-ios"
+  }
+  response = client.request_run_fastlane("actions", env: env)
+
+  @file = nil
+  response.each do |r|
+    puts("Log: #{r.log.message}") if r.log
+
+    puts("State: #{r.state}") if r.state != :PENDING
+
+    puts("Error: #{r.error.description} #{r.error.stacktrace}") if r.error
+
+    next unless r.artifact
+    puts("Chunk: writing to #{r.artifact.filename}")
+    @file ||= File.new(r.artifact.filename, "wb")
+    @file.write(r.artifact.chunk)
+  end
+  @file && @file.close
+end

agent/invocation.rb

@@ -0,0 +1,153 @@
+require_relative "agent"
+require_relative "invocation/recipes"
+require_relative "invocation/state_machine"
+
+module FastlaneCI::Agent
+  ##
+  # the Invocation models a fastlane invocation.
+  # It has states and state transitions
+  # streams update to state, logs, and artifacts via the `yielder`
+  #
+  #
+  #
+  class Invocation
+    include Logging
+    prepend StateMachine
+
+    def initialize(invocation_request, yielder)
+      @invocation_request = invocation_request
+      @yielder = yielder
+      @output_queue = Queue.new
+      Recipes.output_queue = @output_queue
+    end
+
+    ## 
+    # StateMachine actions
+    # These methods run as hooks whenever a transition was successful.
+    ##
+
+    def run
+      # send logs that get put on the output queue.
+      # this needs to be on a separate thread since Queue is a threadsafe blocking queue.
+      Thread.new do
+        send_log(@output_queue.pop) while state == "running"
+      end
+
+      git_url = command_env(:GIT_URL)
+
+      Recipes.setup_repo(git_url)
+
+      # TODO: ensure we are able to satisfy the request
+      # unless has_required_xcode_version?
+      #   reject(RuntimeError.new("Does not have required xcode version!. This is hardcode to be random."))
+      #   return
+      # end
+
+      if Recipes.run_fastlane(@invocation_request.command.env.to_h)
+        finish
+      else
+        # fail is a keyword, so we must call self.
+        # rubocop:disable Style/RedundantSelf
+        self.fail
+        # rubocop:enable Style/RedundantSelf
+      end
+    end
+
+    def finish
+      artifact_path = command_env(:FASTLANE_CI_ARTIFACTS)
+
+      file_path = Recipes.archive_artifacts(artifact_path)
+      send_file(file_path)
+      succeed
+    end
+
+    # the `succeed` method has no special action
+    # the StateMachine requires it's implemented as it's called after a successful transition
+    def succeed
+      # no-op
+    end
+
+    # the `fail` method has no special action
+    # the StateMachine requires it's implemented as it's called after a successful transition
+    def fail
+      # no-op
+    end
+
+    def throw(exception)
+      logger.error("Caught Error: #{exception}")
+
+      error = FastlaneCI::Proto::InvocationResponse::Error.new
+      error.stacktrace = exception.backtrace.join("\n")
+      error.description = exception.message
+
+      @yielder << FastlaneCI::Proto::InvocationResponse.new(error: error)
+    end
+
+    def reject(exception)
+      logger.info("Invocation rejected: #{exception}")
+
+      error = FastlaneCI::Proto::InvocationResponse::Error.new
+      error.description = exception.message
+
+      @yielder << FastlaneCI::Proto::InvocationResponse.new(error: error)
+    end
+
+    ## state machine transition guards
+
+    def has_required_xcode_version?
+      # TODO: bring in from build_runner
+      rand(10) > 3 # 1 in 3 chance of failure
+    end
+
+    # responder methods
+
+    def send_state(event, _payload)
+      logger.debug("State changed. Event `#{event}` => #{state}")
+
+      @yielder << FastlaneCI::Proto::InvocationResponse.new(state: state.to_s.upcase.to_sym)
+    end
+
+    ##
+    # TODO: parse the line, using parse_log_line to figure out the severity and timestamp
+    def send_log(line, level = :DEBUG)
+      log = FastlaneCI::Proto::Log.new(message: line, timestamp: Time.now.to_i, level: level)
+      @yielder << FastlaneCI::Proto::InvocationResponse.new(log: log)
+    end
+
+    def send_file(file_path, chunk_size: 1024 * 1024)
+      unless File.exist?(file_path)
+        logger.warn("No file found at #{file_path}. Skipping sending the file.")
+        return
+      end
+
+      file = File.open(file_path, "rb")
+
+      until file.eof?
+        artifact = FastlaneCI::Proto::InvocationResponse::Artifact.new
+        artifact.chunk = file.read(chunk_size)
+        artifact.filename = File.basename(file_path)
+
+        @yielder << FastlaneCI::Proto::InvocationResponse.new(artifact: artifact)
+      end
+    end
+
+    private
+
+    def command_env(key)
+      key = key.to_s
+      env = @invocation_request.command.env.to_h
+      if env.key?(key)
+        env[key]
+      else
+        raise NameError, "`#{env}` does not have a key `#{key}`"
+      end
+    end
+
+    def parse_log_line(line)
+      re = /^[A-Z], \[([0-9:T\.-]+) #(\d+)\] (\w+) -- (\w*?): (.*)$/
+      if (match_data = re.match(line))
+        return match_data.captures
+      end
+    end
+  end
+end

agent/invocation/recipes.rb

@@ -0,0 +1,81 @@
+require "tmpdir"
+require_relative "../agent"
+
+module FastlaneCI::Agent
+  ##
+  # stateless invocation recipes go here
+  module Recipes
+    extend Logging
+
+    # all method below this are module functions, callable on the module directly.
+    module_function
+
+    ##
+    # set up a Queue that is used to push output from stdout/err from commands that shell out.
+    def output_queue=(value)
+      @output_queue = value
+    end
+
+    def setup_repo(git_url)
+      dir = Dir.mktmpdir("fastlane-ci")
+      Dir.chdir(dir)
+      logger.debug("Changing into working directory #{dir}.")
+
+      sh("git clone --depth 1 #{git_url} repo")
+
+      Dir.chdir("repo")
+      sh("gem install bundler --no-doc")
+      sh("bundle install --deployment")
+
+      sh("gem install cocoapods --no-doc")
+      sh("pod install")
+    end
+
+    def run_fastlane(env)
+      logger.debug("invoking fastlane.")
+      # TODO: send the env to fastlane.
+      sh("bundle exec fastlane actions")
+
+      true
+    end
+
+    ##
+    # archive a directory using tar/gz
+    #
+    # @return String path to the archive that was created or nil if there was an error.
+    def archive_artifacts(artifact_path)
+      unless Dir.exist?(artifact_path)
+        logger.debug("No artifacts found in #{File.expand_path(artifact_path)}.")
+        return
+      end
+      logger.debug("Archiving directory #{artifact_path}")
+
+      Dir.chdir(artifact_path) do
+        sh("tar -cvzf Archive.tgz .")
+      end
+
+      return File.join(artifact_path, "Archive.tgz")
+    end
+
+    ##
+    # use this to execute shell commands so the output can be streamed back as a response.
+    #
+    # this command will either execute successfully or raise an exception.
+    def sh(*params, env: {})
+      @output_queue.push(params.join(" "))
+      stdin, stdouterr, thread = Open3.popen2e(*params)
+      stdin.close
+
+      # `gets` on a pipe will block until the pipe is closed, then returns nil.
+      while (line = stdouterr.gets)
+        logger.debug(line)
+        @output_queue.push(line)
+      end
+
+      exit_status = thread.value.exitstatus
+      if exit_status != 0
+        raise SystemCallError.new(line, exit_status)
+      end
+    end
+  end
+end