Skip to content
Permalink
Browse files
Merge pull request #179 from syapse/spot-ssh-rebase
Poll for spot instances instead of JNLP launcher JENKINS-27529 JENKINS-19059
  • Loading branch information
Francis Upton IV committed Mar 31, 2016
2 parents cc23601 + ae7e53a commit e3bfb31528faba3dd23bef940d1a4a8fa2a227c3
Showing 8 changed files with 53 additions and 97 deletions.
@@ -46,7 +46,6 @@
import net.sf.json.JSONObject;

import org.apache.commons.lang.StringUtils;
import org.kohsuke.stapler.DataBoundConstructor;
import org.kohsuke.stapler.QueryParameter;
import org.kohsuke.stapler.StaplerRequest;

@@ -121,7 +120,6 @@ public abstract class EC2AbstractSlave extends Slave {

public static final String TEST_ZONE = "testZone";

@DataBoundConstructor
public EC2AbstractSlave(String name, String instanceId, String description, String remoteFS, int numExecutors, Mode mode, String labelString, ComputerLauncher launcher, RetentionStrategy<EC2Computer> retentionStrategy, String initScript, String tmpDir, List<? extends NodeProperty<?>> nodeProperties, String remoteAdmin, String jvmopts, boolean stopOnTerminate, String idleTerminationMinutes, List<EC2Tag> tags, String cloudName, boolean usePrivateDnsName, boolean useDedicatedTenancy, int launchTimeout, AMITypeData amiType)
throws FormException, IOException {

@@ -348,7 +348,7 @@ private int countCurrentEC2Slaves(SlaveTemplate template) throws AmazonClientExc
if (!(node instanceof EC2SpotSlave))
continue;
EC2SpotSlave ec2Slave = (EC2SpotSlave) node;
SpotInstanceRequest sir = ec2Slave.getSpotRequest(ec2Slave.getSpotInstanceRequestId());
SpotInstanceRequest sir = ec2Slave.getSpotRequest();
if (sir == null) {
LOGGER.log(Level.FINE, "Found spot node without request: " + ec2Slave.getSpotInstanceRequestId());
n++;
@@ -51,6 +51,26 @@ public abstract class EC2ComputerLauncher extends ComputerLauncher {
public void launch(SlaveComputer _computer, TaskListener listener) {
try {
EC2Computer computer = (EC2Computer) _computer;

while (true) {
String instanceId = computer.getInstanceId();
if (instanceId != null && !instanceId.equals("")) {
break;
}
// Only spot slaves can have no instance id.
EC2SpotSlave ec2Slave = (EC2SpotSlave) computer.getNode();
if (ec2Slave.isSpotRequestDead()) {
// Terminate launch
return;
}
final String msg = "Node " + computer.getName() + "(SpotRequest " + computer.getSpotInstanceRequestId() +
") still requesting the instance, waiting 10s";
// report to system log and console
((EC2Computer) _computer).getCloud().log(LOGGER, Level.FINEST, listener, msg);
// check every 10 seconds if in spot request phase
Thread.sleep(10000);
}

final String baseMsg = "Node " + computer.getName() + "(" + computer.getInstanceId() + ")";
String msg;

@@ -91,10 +111,10 @@ public void launch(SlaveComputer _computer, TaskListener listener) {
break;
}

// report to system log and console
((EC2Computer) _computer).getCloud().log(LOGGER, Level.FINEST, listener, msg);
// check every 5 secs
Thread.sleep(5000);
// and report to system log and console
((EC2Computer) _computer).getCloud().log(LOGGER, Level.FINEST, listener, msg);
}

launch(computer, listener, computer.describeInstance());

This file was deleted.

This file was deleted.

@@ -15,11 +15,14 @@
import com.amazonaws.services.ec2.model.DescribeSpotInstanceRequestsRequest;
import com.amazonaws.services.ec2.model.DescribeSpotInstanceRequestsResult;
import com.amazonaws.services.ec2.model.SpotInstanceRequest;
import com.amazonaws.services.ec2.model.SpotInstanceState;
import com.amazonaws.services.ec2.model.TerminateInstancesRequest;

import hudson.Extension;
import hudson.model.Hudson;
import hudson.model.Descriptor.FormException;
import hudson.plugins.ec2.ssh.EC2UnixLauncher;
import hudson.plugins.ec2.win.EC2WindowsLauncher;
import hudson.slaves.NodeProperty;

public final class EC2SpotSlave extends EC2AbstractSlave {
@@ -28,18 +31,25 @@ public final class EC2SpotSlave extends EC2AbstractSlave {

public EC2SpotSlave(String name, String spotInstanceRequestId, String description, String remoteFS, int numExecutors, Mode mode, String initScript, String tmpDir, String labelString, String remoteAdmin, String jvmopts, String idleTerminationMinutes, List<EC2Tag> tags, String cloudName, boolean usePrivateDnsName, int launchTimeout, AMITypeData amiType)
throws FormException, IOException {
this(name, spotInstanceRequestId, description, remoteFS, numExecutors, mode, initScript, tmpDir, labelString, Collections.<NodeProperty<?>> emptyList(), remoteAdmin, jvmopts, idleTerminationMinutes, tags, cloudName, usePrivateDnsName, launchTimeout, amiType);
this(description + " (" + name + ")", spotInstanceRequestId, description, remoteFS, numExecutors, mode, initScript, tmpDir, labelString, Collections.<NodeProperty<?>> emptyList(), remoteAdmin, jvmopts, idleTerminationMinutes, tags, cloudName, usePrivateDnsName, launchTimeout, amiType);
}

@DataBoundConstructor
public EC2SpotSlave(String name, String spotInstanceRequestId, String description, String remoteFS, int numExecutors, Mode mode, String initScript, String tmpDir, String labelString, List<? extends NodeProperty<?>> nodeProperties, String remoteAdmin, String jvmopts, String idleTerminationMinutes, List<EC2Tag> tags, String cloudName, boolean usePrivateDnsName, int launchTimeout, AMITypeData amiType)
throws FormException, IOException {

super(name, "", description, remoteFS, numExecutors, mode, labelString, new EC2SpotComputerLauncher(), new EC2SpotRetentionStrategy(idleTerminationMinutes), initScript, tmpDir, nodeProperties, remoteAdmin, jvmopts, false, idleTerminationMinutes, tags, cloudName, usePrivateDnsName, false, launchTimeout, amiType);
super(name, "", description, remoteFS, numExecutors, mode, labelString, amiType.isWindows() ? new EC2WindowsLauncher() :
new EC2UnixLauncher(), new EC2RetentionStrategy(idleTerminationMinutes), initScript, tmpDir, nodeProperties, remoteAdmin, jvmopts, false, idleTerminationMinutes, tags, cloudName, usePrivateDnsName, false, launchTimeout, amiType);

this.name = name;
this.spotInstanceRequestId = spotInstanceRequestId;
}

@Override
protected boolean isAlive(boolean force) {
return super.isAlive(force) || !this.isSpotRequestDead();
}

/**
* Cancel the spot request for the instance. Terminate the instance if it is up. Remove the slave from Jenkins.
*/
@@ -58,7 +68,7 @@ public void terminate() {

// Terminate the slave if it is running
if (instanceId != null && !instanceId.equals("")) {
if (!isAlive(true)) {
if (!super.isAlive(true)) {
/*
* The node has been killed externally, so we've nothing to do here
*/
@@ -95,14 +105,13 @@ public void terminate() {

/**
* Retrieve the SpotRequest for a requestId
*
* @param spotRequestId
* @return SpotInstanceRequest object for the requestId, or null
*
* @return SpotInstanceRequest object for this slave, or null
*/
SpotInstanceRequest getSpotRequest(String spotRequestId) {
SpotInstanceRequest getSpotRequest() {
AmazonEC2 ec2 = getCloud().connect();

DescribeSpotInstanceRequestsRequest dsirRequest = new DescribeSpotInstanceRequestsRequest().withSpotInstanceRequestIds(spotRequestId);
DescribeSpotInstanceRequestsRequest dsirRequest = new DescribeSpotInstanceRequestsRequest().withSpotInstanceRequestIds(this.spotInstanceRequestId);
DescribeSpotInstanceRequestsResult dsirResult = null;
List<SpotInstanceRequest> siRequests = null;

@@ -112,10 +121,10 @@ SpotInstanceRequest getSpotRequest(String spotRequestId) {

} catch (AmazonServiceException e) {
// Spot request is no longer valid
LOGGER.log(Level.WARNING, "Failed to fetch spot instance request for requestId: " + spotRequestId);
LOGGER.log(Level.WARNING, "Failed to fetch spot instance request for requestId: " + this.spotInstanceRequestId);
} catch (AmazonClientException e) {
// Spot request is no longer valid
LOGGER.log(Level.WARNING, "Failed to fetch spot instance request for requestId: " + spotRequestId);
LOGGER.log(Level.WARNING, "Failed to fetch spot instance request for requestId: " + this.spotInstanceRequestId);
}

if (dsirResult == null || siRequests.isEmpty()) {
@@ -124,6 +133,13 @@ SpotInstanceRequest getSpotRequest(String spotRequestId) {
return siRequests.get(0);
}

public boolean isSpotRequestDead() {
SpotInstanceState requestState = SpotInstanceState.fromValue(this.getSpotRequest().getState());
return requestState == SpotInstanceState.Cancelled
|| requestState == SpotInstanceState.Closed
|| requestState == SpotInstanceState.Failed;
}

/**
* Accessor for the spotInstanceRequestId
*/
@@ -134,7 +150,7 @@ public String getSpotInstanceRequestId() {
@Override
public String getInstanceId() {
if (instanceId == null || instanceId.equals("")) {
SpotInstanceRequest sr = getSpotRequest(spotInstanceRequestId);
SpotInstanceRequest sr = this.getSpotRequest();
if (sr != null)
instanceId = sr.getInstanceId();
}
@@ -161,7 +177,7 @@ public String getDisplayName() {

@Override
public String getEc2Type() {
String spotMaxBidPrice = this.getSpotRequest(spotInstanceRequestId).getSpotPrice();
String spotMaxBidPrice = this.getSpotRequest().getSpotPrice();
return Messages.EC2SpotSlave_Spot1() + spotMaxBidPrice.substring(0, spotMaxBidPrice.length() - 3)
+ Messages.EC2SpotSlave_Spot2();
}
@@ -725,43 +725,7 @@ private EC2AbstractSlave provisionSpot(TaskListener listener) throws AmazonClien
}
}

// The slave must know the Jenkins server to register with as well
// as the name of the node in Jenkins it should register as. The
// only
// way to give information to the Spot slaves is through the ec2
// user data
String jenkinsUrl = Hudson.getInstance().getRootUrl();
// We must provide a unique node name for the slave to connect to
// Jenkins.
// We don't have the EC2 generated instance ID, or the Spot request
// ID
// until after the instance is requested, which is then too late to
// set the
// user-data for the request. Instead we generate a unique name from
// UUID
// so that the slave has a unique name within Jenkins to register
// to.
String slaveName = UUID.randomUUID().toString();
String newUserData = "";

// We want to allow node configuration with cloud-init and
// user-data,
// while maintaining backward compatibility with old ami's
// The 'new' way is triggered by the presence of '${SLAVE_NAME}'' in
// the user data
// (which is not too much to ask)
if (userData.contains("${SLAVE_NAME}")) {
// The cloud-init compatible way
newUserData = new String(userData);
newUserData = newUserData.replace("${SLAVE_NAME}", slaveName);
newUserData = newUserData.replace("${JENKINS_URL}", jenkinsUrl);
} else {
// The 'old' way - maitain full backward compatibility
newUserData = "JENKINS_URL=" + jenkinsUrl + "&SLAVE_NAME=" + slaveName + "&USER_DATA="
+ Base64.encodeBase64String(userData.getBytes());
}

String userDataString = Base64.encodeBase64String(newUserData.getBytes());
String userDataString = Base64.encodeBase64String(userData.getBytes());

launchSpecification.setUserData(userDataString);
launchSpecification.setKeyName(keyPair.getKeyName());
@@ -808,6 +772,7 @@ private EC2AbstractSlave provisionSpot(TaskListener listener) throws AmazonClien
if (spotInstReq == null) {
throw new AmazonClientException("Spot instance request is null");
}
String slaveName = spotInstReq.getSpotInstanceRequestId();

/* Now that we have our Spot request, we can set tags on it */
if (inst_tags != null) {
@@ -50,13 +50,6 @@ THE SOFTWARE.
</f:entry>

<f:optionalBlock name="spotConfig" title="Use Spot Instance" checked="${instance.spotConfig != null}">
<f:description>Be aware, AMIs used for Spot slaves must be configured to callback to Jenkins when
a Spot request has been fulfilled and the instance has become available. The call back script can
be found by clicking the following link. <a href="${resURL}/plugin/ec2/AMI-Scripts/ubuntu-ami-setup.sh">Ubuntu-ami-setup</a>
</f:description>
<f:description>Slaves designated as Spot slaves will initially show up as disconnected. The state
will change to connecting when a Spot request has been fulfilled. </f:description>

<f:validateButton title="${%Check Current Spot Price}" progress="${%Checking...}" method="currentSpotPrice" with="useInstanceProfileForCredentials,accessId,secretKey,region,type,zone" />

<f:entry title="${%Spot Max Bid Price}" field="spotMaxBidPrice">

0 comments on commit e3bfb31

Please sign in to comment.