Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Allow Harbormaster build targets to wait for messages
Summary:
This hooks up all the pieces of the build pipeline so `harbormaster.sendmessage` actually works. Particularly:

  - Candidate build steps (i.e., those which interact with external systems) can now "Wait for Message". This pauses them indefinitely when they complete, until something calls `harbormaster.sendmessage`.
  - After processing a target, we check if we should move it to PASSED or WAITING.
  - Before updating a build, we move WAITING targets with pending messages to either PASSED or FAILED.
  - I added an explicit "Building" state, which doesn't affect workflows but communicates more information to human users.

A big part of this is avoiding races. I believe we get the correct behavior no matter which order events occur in:

  - We update builds after targets complete and after we receive messages, so we're guaranteed to update once both these conditions are true. This means messages can't be lost (even if they arrive before a build completes).
  - The minor changes to the build engine logic mean that firing additional build updates is always safe, no matter what the current state of the build is.
  - The build itself is protected by a lock in the build engine.
  - The target is not covered by an explicit lock, but for all states only the engine (waiting) //or// the worker (all other states) can interact with it. All of the interactions also move the target state forward to the same destination and have no other side effects.
  - Messages are only consumed inside the engine lock, so they don't need an explicit lock.

Test Plan:
  - Made an HTTP request wait after completion, then ran a pile of builds through it using `bin/harbormaster build` and the web UI.
  - Passed and failed message-awaiting builds with `harbormaster.sendmessage`.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley, zeeg

Differential Revision: https://secure.phabricator.com/D8788
  • Loading branch information
epriestley committed Apr 16, 2014
1 parent 803c50c commit 78bf266
Show file tree
Hide file tree
Showing 9 changed files with 204 additions and 16 deletions.
Expand Up @@ -43,6 +43,13 @@ protected function execute(ConduitAPIRequest $request) {
->setType($message_type)
->save();

// If the build has completely paused because all steps are blocked on
// waiting targets, this will resume it.
id(new HarbormasterBuildEngine())
->setViewer($viewer)
->setBuild($build_target->getBuild())
->continueBuild();

return null;
}

Expand Down
Expand Up @@ -251,15 +251,27 @@ private function buildBuildList(HarbormasterBuildable $buildable) {
switch ($target->getTargetStatus()) {
case HarbormasterBuildTarget::STATUS_PENDING:
$icon = 'time-green';
$status_name = pht('Pending');
break;
case HarbormasterBuildTarget::STATUS_BUILDING:
$icon = 'right-green';
$status_name = pht('Building');
break;
case HarbormasterBuildTarget::STATUS_WAITING:
$icon = 'time-orange';
$status_name = pht('Waiting');
break;
case HarbormasterBuildTarget::STATUS_PASSED:
$icon = 'accept-green';
$status_name = pht('Passed');
break;
case HarbormasterBuildTarget::STATUS_FAILED:
$icon = 'reject-red';
$status_name = pht('Failed');
break;
default:
$icon = 'question';
$status_name = pht('Unknown');
break;
}

Expand All @@ -272,7 +284,7 @@ private function buildBuildList(HarbormasterBuildable $buildable) {

$target_list->addItem(
id(new PHUIStatusItemView())
->setIcon($icon)
->setIcon($icon, $status_name)
->setTarget(pht('Target %d', $target->getID()))
->setNote($name));
}
Expand Down
Expand Up @@ -9,7 +9,41 @@ public function getStandardCustomFieldNamespace() {
}

public function createFields($object) {
$specs = $object->getStepImplementation()->getFieldSpecifications();
$impl = $object->getStepImplementation();
$specs = $impl->getFieldSpecifications();

if ($impl->supportsWaitForMessage()) {
$specs['builtin.next-steps-header'] = array(
'type' => 'header',
'name' => pht('Next Steps'),
);

$specs['builtin.wait-for-message'] = array(
'type' => 'select',
'name' => pht('When Complete'),
'instructions' => pht(
'After completing this build step Harbormaster can continue the '.
'build normally, or it can pause the build and wait for a message. '.
'If you are using this build step to trigger some work in an '.
'external system, you may want to have Phabricator wait for that '.
'system to perform the work and report results back.'.
"\n\n".
'If you select **Continue Build Normally**, the build plan will '.
'proceed once this step finishes.'.
"\n\n".
'If you select **Wait For Message**, the build plan will pause '.
'indefinitely once this step finishes. To resume the build, an '.
'external system must call `harbormaster.sendmessage` with the '.
'build target PHID, and either `"pass"` or `"fail"` to indicate '.
'the result for this step. After the result is recorded, the build '.
'plan will resume.'),
'options' => array(
'' => pht('Continue Build Normally'),
'wait' => pht('Wait For Message'),
),
);
}

return PhabricatorStandardCustomField::buildStandardFields($this, $specs);
}

Expand Down
109 changes: 97 additions & 12 deletions src/applications/harbormaster/engine/HarbormasterBuildEngine.php
Expand Up @@ -127,22 +127,45 @@ private function updateBuildSteps(HarbormasterBuild $build) {
->setViewer($this->getViewer())
->withBuildPHIDs(array($build->getPHID()))
->execute();

$this->updateWaitingTargets($targets);

$targets = mgroup($targets, 'getBuildStepPHID');

$steps = id(new HarbormasterBuildStepQuery())
->setViewer($this->getViewer())
->withBuildPlanPHIDs(array($build->getBuildPlan()->getPHID()))
->execute();

// Identify steps which are complete.
// Identify steps which are in various states.

$queued = array();
$underway = array();
$waiting = array();
$complete = array();
$failed = array();
$waiting = array();
foreach ($steps as $step) {
$step_targets = idx($targets, $step->getPHID(), array());

if ($step_targets) {
$is_queued = false;

$is_underway = false;
foreach ($step_targets as $target) {
if ($target->isUnderway()) {
$is_underway = true;
break;
}
}

$is_waiting = false;
foreach ($step_targets as $target) {
if ($target->isWaiting()) {
$is_waiting = true;
break;
}
}

$is_complete = true;
foreach ($step_targets as $target) {
if (!$target->isComplete()) {
Expand All @@ -158,25 +181,33 @@ private function updateBuildSteps(HarbormasterBuild $build) {
break;
}
}

$is_waiting = false;
} else {
$is_queued = true;
$is_underway = false;
$is_waiting = false;
$is_complete = false;
$is_failed = false;
$is_waiting = true;
}

if ($is_complete) {
$complete[$step->getPHID()] = true;
if ($is_queued) {
$queued[$step->getPHID()] = true;
}

if ($is_failed) {
$failed[$step->getPHID()] = true;
if ($is_underway) {
$underway[$step->getPHID()] = true;
}

if ($is_waiting) {
$waiting[$step->getPHID()] = true;
}

if ($is_complete) {
$complete[$step->getPHID()] = true;
}

if ($is_failed) {
$failed[$step->getPHID()] = true;
}
}

// If any step failed, fail the whole build, then bail.
Expand Down Expand Up @@ -209,7 +240,7 @@ private function updateBuildSteps(HarbormasterBuild $build) {
$dependencies = array();
}

if (isset($waiting[$step->getPHID()])) {
if (isset($queued[$step->getPHID()])) {
$can_run = true;
foreach ($dependencies as $dependency) {
if (empty($complete[$dependency->getPHID()])) {
Expand All @@ -226,9 +257,9 @@ private function updateBuildSteps(HarbormasterBuild $build) {
$previous_step = $step;
}

if (!$runnable) {
if (!$runnable && !$waiting && !$underway) {
// TODO: This means the build is deadlocked, probably? It should not
// normally be possible, but we should communicate it more clearly.
// normally be possible yet, but we should communicate it more clearly.
$build->setBuildStatus(HarbormasterBuild::STATUS_FAILED);
$build->save();
return;
Expand All @@ -243,6 +274,60 @@ private function updateBuildSteps(HarbormasterBuild $build) {

$this->queueNewBuildTarget($target);
}

}


/**
* Process messages which were sent to these targets, kicking applicable
* targets out of "Waiting" and into either "Passed" or "Failed".
*
* @param list<HarbormasterBuildTarget> List of targets to process.
* @return void
*/
private function updateWaitingTargets(array $targets) {
assert_instances_of($targets, 'HarbormasterBuildTarget');

// We only care about messages for targets which are actually in a waiting
// state.
$waiting_targets = array();
foreach ($targets as $target) {
if ($target->isWaiting()) {
$waiting_targets[$target->getPHID()] = $target;
}
}

if (!$waiting_targets) {
return;
}

$messages = id(new HarbormasterBuildMessageQuery())
->setViewer($this->getViewer())
->withBuildTargetPHIDs(array_keys($waiting_targets))
->withConsumed(false)
->execute();

foreach ($messages as $message) {
$target = $waiting_targets[$message->getBuildTargetPHID()];

$new_status = null;
switch ($message->getType()) {
case 'pass':
$new_status = HarbormasterBuildTarget::STATUS_PASSED;
break;
case 'fail':
$new_status = HarbormasterBuildTarget::STATUS_FAILED;
break;
}

if ($new_status !== null) {
$message->setIsConsumed(true);
$message->save();

$target->setTargetStatus($new_status);
$target->save();
}
}
}

}
Expand Up @@ -83,7 +83,7 @@ private function buildWhereClause(AphrontDatabaseConnection $conn_r) {
$where[] = qsprintf(
$conn_r,
'isConsumed = %d',
(int)$this->isConsumed);
(int)$this->consumed);
}

$where[] = $this->buildPagingClause($conn_r);
Expand Down
Expand Up @@ -196,4 +196,16 @@ protected function formatValueForDescription($value) {
}
}

public function supportsWaitForMessage() {
return false;
}

public function shouldWaitForMessage(HarbormasterBuildTarget $target) {
if (!$this->supportsWaitForMessage()) {
return false;
}

return (bool)$target->getDetail('builtin.wait-for-message');
}

}
Expand Up @@ -99,4 +99,8 @@ public function getFieldSpecifications() {
);
}

public function supportsWaitForMessage() {
return true;
}

}
Expand Up @@ -11,6 +11,8 @@ final class HarbormasterBuildTarget extends HarbormasterDAO
protected $targetStatus;

const STATUS_PENDING = 'target/pending';
const STATUS_BUILDING = 'target/building';
const STATUS_WAITING = 'target/waiting';
const STATUS_PASSED = 'target/passed';
const STATUS_FAILED = 'target/failed';

Expand Down Expand Up @@ -128,6 +130,26 @@ public function isFailed() {
}


public function isWaiting() {
switch ($this->getTargetStatus()) {
case self::STATUS_WAITING:
return true;
}

return false;
}

public function isUnderway() {
switch ($this->getTargetStatus()) {
case self::STATUS_PENDING:
case self::STATUS_BUILDING:
return true;
}

return false;
}


/* -( PhabricatorPolicyInterface )----------------------------------------- */


Expand Down
Expand Up @@ -36,9 +36,21 @@ public function doWork() {
$viewer = $this->getViewer();

try {
$status_pending = HarbormasterBuildTarget::STATUS_PENDING;
if ($target->getTargetStatus() == $status_pending) {
$target->setTargetStatus(HarbormasterBuildTarget::STATUS_BUILDING);
$target->save();
}

$implementation = $target->getImplementation();
$implementation->execute($build, $target);
$target->setTargetStatus(HarbormasterBuildTarget::STATUS_PASSED);

$next_status = HarbormasterBuildTarget::STATUS_PASSED;
if ($implementation->shouldWaitForMessage($target)) {
$next_status = HarbormasterBuildTarget::STATUS_WAITING;
}

$target->setTargetStatus($next_status);
$target->save();
} catch (Exception $ex) {
phlog($ex);
Expand Down

0 comments on commit 78bf266

Please sign in to comment.