Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

JIRA-14884 Resume Stopped Ec2 Instances

  • Loading branch information...
commit d5154c60f1e3622a3248223a00d0774ac368c3b9 1 parent 3a16a1e
Francis Upton IV authored
View
3  src/main/java/hudson/plugins/ec2/EC2Cloud.java
@@ -218,7 +218,8 @@ public void doProvision(StaplerRequest req, StaplerResponse rsp, @QueryParameter
public Collection<PlannedNode> provision(Label label, int excessWorkload) {
try {
- final SlaveTemplate t = getTemplate(label);
+
+ final SlaveTemplate t = getTemplate(label);
List<PlannedNode> r = new ArrayList<PlannedNode>();
for( ; excessWorkload>0; excessWorkload-- ) {
View
3  src/main/java/hudson/plugins/ec2/EC2Computer.java
@@ -103,7 +103,8 @@ private Instance _describeInstance() throws AmazonClientException {
@Override
public HttpResponse doDoDelete() throws IOException {
checkPermission(DELETE);
- getNode().terminate();
+ if (getNode() != null)
+ getNode().terminate();
return new HttpRedirect("..");
}
View
15 src/main/java/hudson/plugins/ec2/EC2ComputerLauncher.java
@@ -6,9 +6,14 @@
import java.io.IOException;
import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.List;
import com.amazonaws.AmazonClientException;
+import com.amazonaws.services.ec2.AmazonEC2;
import com.amazonaws.services.ec2.model.Instance;
+import com.amazonaws.services.ec2.model.StartInstancesRequest;
+import com.amazonaws.services.ec2.model.StartInstancesResult;
/**
* {@link ComputerLauncher} for EC2 that waits for the instance to really come up before proceeding to
@@ -27,10 +32,20 @@ public void launch(SlaveComputer _computer, TaskListener listener) {
while(true) {
switch (computer.getState()) {
case PENDING:
+ case STOPPING:
Thread.sleep(5000); // check every 5 secs
continue OUTER;
case RUNNING:
break OUTER;
+ case STOPPED:
+ AmazonEC2 ec2 = EC2Cloud.get().connect();
+ List<String> instances = new ArrayList<String>();
+ instances.add(computer.getInstanceId());
+
+ StartInstancesRequest siRequest = new StartInstancesRequest(instances);
+ StartInstancesResult siResult = ec2.startInstances(siRequest);
+ logger.println("Starting existing instance: "+computer.getInstanceId()+ " result:"+siResult);
+ continue OUTER;
case SHUTTING_DOWN:
case TERMINATED:
// abort
View
10 src/main/java/hudson/plugins/ec2/EC2RetentionStrategy.java
@@ -40,13 +40,15 @@ public synchronized long check(EC2Computer c) {
/* If we've been told never to terminate, then we're done. */
if (idleTerminationMinutes == 0) return 1;
-
- if (c.isIdle() && !disabled) {
+ final long idleMilliseconds1 = System.currentTimeMillis() - c.getIdleStartMilliseconds();
+ System.out.println(c.getName() + " idle: " + idleMilliseconds1);
+
+ if (c.isIdle() && c.isOnline() && !disabled) {
// TODO: really think about the right strategy here
final long idleMilliseconds = System.currentTimeMillis() - c.getIdleStartMilliseconds();
if (idleMilliseconds > TimeUnit2.MINUTES.toMillis(idleTerminationMinutes)) {
- LOGGER.info("Disconnecting "+c.getName());
- c.getNode().terminate();
+ LOGGER.info("Idle timeout: "+c.getName());
+ c.getNode().idleTimeout();
}
}
return 1;
View
68 src/main/java/hudson/plugins/ec2/EC2Slave.java
@@ -29,6 +29,7 @@
import com.amazonaws.AmazonClientException;
import com.amazonaws.services.ec2.AmazonEC2;
import com.amazonaws.services.ec2.model.*;
+
import net.sf.json.JSONObject;
/**
@@ -137,6 +138,23 @@ public Computer createComputer() {
return new EC2Computer(this);
}
+ public static Instance getInstance(String instanceId) {
+ DescribeInstancesRequest request = new DescribeInstancesRequest();
+ request.setInstanceIds(Collections.<String>singletonList(instanceId));
+ EC2Cloud cloudInstance = EC2Cloud.get();
+ if (cloudInstance == null)
+ return null;
+ AmazonEC2 ec2 = cloudInstance.connect();
+ List<Reservation> reservations = ec2.describeInstances(request).getReservations();
+ Instance i = null;
+ if (reservations.size() > 0) {
+ List<Instance> instances = reservations.get(0).getInstances();
+ if (instances.size() > 0)
+ i = instances.get(0);
+ }
+ return i;
+ }
+
/**
* Terminates the instance in EC2.
*/
@@ -144,20 +162,12 @@ public void terminate() {
try {
if (!isAlive(true)) {
/* The node has been killed externally, so we've nothing to do here */
- LOGGER.info("EC2 instance already terminated: " + getInstanceId());
+ LOGGER.info("EC2 instance already terminated: "+getInstanceId());
} else {
- /* The node is still alive - do the appropriate thing */
-
AmazonEC2 ec2 = EC2Cloud.get().connect();
- if (stopOnTerminate) {
- StopInstancesRequest request = new StopInstancesRequest(Collections.singletonList(getInstanceId()));
- ec2.stopInstances(request);
- LOGGER.info("Terminated EC2 instance (stopped): "+getInstanceId());
- } else {
- TerminateInstancesRequest request = new TerminateInstancesRequest(Collections.singletonList(getInstanceId()));
- ec2.terminateInstances(request);
- LOGGER.info("Terminated EC2 instance (terminated): "+getInstanceId());
- }
+ TerminateInstancesRequest request = new TerminateInstancesRequest(Collections.singletonList(getInstanceId()));
+ ec2.terminateInstances(request);
+ LOGGER.info("Terminated EC2 instance (terminated): "+getInstanceId());
}
Hudson.getInstance().removeNode(this);
} catch (AmazonClientException e) {
@@ -167,6 +177,26 @@ public void terminate() {
}
}
+ void idleTimeout() {
+ LOGGER.info("EC2 instance idle time expired: "+getInstanceId());
+ if (!stopOnTerminate) {
+ terminate();
+ return;
+ }
+
+ try {
+ AmazonEC2 ec2 = EC2Cloud.get().connect();
+ StopInstancesRequest request = new StopInstancesRequest(
+ Collections.singletonList(getInstanceId()));
+ ec2.stopInstances(request);
+ toComputer().disconnect(null);
+ } catch (AmazonClientException e) {
+ Instance i = getInstance(getNodeName());
+ LOGGER.log(Level.WARNING, "Failed to terminate EC2 instance: "+getInstanceId() + " info: "+((i != null)?i:"") , e);
+ }
+ LOGGER.info("EC2 instance stopped: " + getInstanceId());
+ }
+
String getRemoteAdmin() {
if (remoteAdmin == null || remoteAdmin.length() == 0)
return "root";
@@ -207,12 +237,12 @@ private void fetchLiveInstanceData( boolean force ) throws AmazonClientException
return;
}
- DescribeInstancesRequest request = new DescribeInstancesRequest();
- request.setInstanceIds(Collections.<String>singletonList(getNodeName()));
- Instance i = EC2Cloud.get().connect().describeInstances(request).getReservations().get(0).getInstances().get(0);
+ Instance i = getInstance(getNodeName());
lastFetchTime = now;
lastFetchInstance = i;
+ if (i == null)
+ return;
publicDNS = i.getPublicDnsName();
privateDNS = i.getPrivateIpAddress();
@@ -226,9 +256,7 @@ private void fetchLiveInstanceData( boolean force ) throws AmazonClientException
/* Clears all existing tag data so that we can force the instance into a known state */
private void clearLiveInstancedata() throws AmazonClientException {
- DescribeInstancesRequest request = new DescribeInstancesRequest();
- request.setInstanceIds( Collections.<String>singletonList(getNodeName()));
- Instance inst = EC2Cloud.get().connect().describeInstances(request).getReservations().get(0).getInstances().get(0);
+ Instance inst = getInstance(getNodeName());
/* Now that we have our instance, we can clear the tags on it */
if (!tags.isEmpty()) {
@@ -247,9 +275,7 @@ private void clearLiveInstancedata() throws AmazonClientException {
/* Sets tags on an instance. This will not clear existing tag data, so call clearLiveInstancedata if needed */
private void pushLiveInstancedata() throws AmazonClientException {
- DescribeInstancesRequest request = new DescribeInstancesRequest();
- request.setInstanceIds(Collections.<String>singletonList(getNodeName()));
- Instance inst = EC2Cloud.get().connect().describeInstances(request).getReservations().get(0).getInstances().get(0);
+ Instance inst = getInstance(getNodeName());
/* Now that we have our instance, we can set tags on it */
if (tags != null && !tags.isEmpty()) {
View
94 src/main/java/hudson/plugins/ec2/SlaveTemplate.java
@@ -160,7 +160,7 @@ public String getidleTerminationMinutes() {
return idleTerminationMinutes;
}
- public Set getLabelSet(){
+ public Set<LabelAtom> getLabelSet(){
return labelSet;
}
@@ -202,15 +202,20 @@ public EC2Slave provision(TaskListener listener) throws AmazonClientException, I
throw new AmazonClientException("No matching keypair found on EC2. Is the EC2 private key a valid one?");
}
- RunInstancesRequest request = new RunInstancesRequest(ami, 1, 1);
+ RunInstancesRequest riRequest = new RunInstancesRequest(ami, 1, 1);
+ List<Filter> diFilters = new ArrayList<Filter>();
+ diFilters.add(new Filter("image-id").withValues(ami));
+
if (StringUtils.isNotBlank(getZone())) {
Placement placement = new Placement(getZone());
- request.setPlacement(placement);
+ riRequest.setPlacement(placement);
+ diFilters.add(new Filter("availability-zone").withValues(getZone()));
}
if (StringUtils.isNotBlank(getSubnetId())) {
- request.setSubnetId(getSubnetId());
+ riRequest.setSubnetId(getSubnetId());
+ diFilters.add(new Filter("subnet-id").withValues(getSubnetId()));
/* If we have a subnet ID then we can only use VPC security groups */
if (!securityGroupSet.isEmpty()) {
@@ -243,36 +248,79 @@ public EC2Slave provision(TaskListener listener) throws AmazonClientException, I
}
if (!group_ids.isEmpty()) {
- request.setSecurityGroupIds(group_ids);
+ riRequest.setSecurityGroupIds(group_ids);
+ diFilters.add(new Filter("instance.group-id").withValues(group_ids));
}
}
} else {
/* No subnet: we can use standard security groups by name */
- request.setSecurityGroups(securityGroupSet);
+ riRequest.setSecurityGroups(securityGroupSet);
+ if (securityGroupSet.size() > 0)
+ diFilters.add(new Filter("group-id").withValues(securityGroupSet));
}
- request.setUserData(Base64.encodeBase64String(userData.getBytes()));
- request.setKeyName(keyPair.getKeyName());
- request.setInstanceType(type.toString());
- Instance inst = ec2.runInstances(request).getReservation().getInstances().get(0);
-
- /* Now that we have our instance, we can set tags on it */
+ String userDataString = Base64.encodeBase64String(userData.getBytes());
+ riRequest.setUserData(userDataString);
+ riRequest.setKeyName(keyPair.getKeyName());
+ diFilters.add(new Filter("key-name").withValues(keyPair.getKeyName()));
+ riRequest.setInstanceType(type.toString());
+ diFilters.add(new Filter("instance-type").withValues(type.toString()));
+
+ HashSet<Tag> inst_tags = null;
if (tags != null && !tags.isEmpty()) {
- HashSet<Tag> inst_tags = new HashSet<Tag>();
-
+ inst_tags = new HashSet<Tag>();
for(EC2Tag t : tags) {
- inst_tags.add(new Tag(t.getName(), t.getValue()));
@glasser
glasser added a note

Removing this line broken the ability to set tags on new slaves... inst_tags as sent to ec2.createTags is always empty now. I'll send a PR.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
+ diFilters.add(new Filter("tag:"+t.getName()).withValues(t.getValue()));
}
-
- CreateTagsRequest tag_request = new CreateTagsRequest();
- tag_request.withResources(inst.getInstanceId()).setTags(inst_tags);
- ec2.createTags(tag_request);
-
- // That was a remote request - we should also update our local instance data.
- inst.setTags(inst_tags);
}
-
+
+ DescribeInstancesRequest diRequest = new DescribeInstancesRequest();
+ diFilters.add(new Filter("instance-state-name").withValues(InstanceStateName.Stopped.toString(),
+ InstanceStateName.Stopping.toString()));
+ diRequest.setFilters(diFilters);
+ logger.println("Looking for existing instances: "+diRequest);
+
+ DescribeInstancesResult diResult = ec2.describeInstances(diRequest);
+ if (diResult.getReservations().size() == 0) {
+ // Have to create a new instance
+ Instance inst = ec2.runInstances(riRequest).getReservation().getInstances().get(0);
+
+ /* Now that we have our instance, we can set tags on it */
+ if (inst_tags != null) {
+ CreateTagsRequest tag_request = new CreateTagsRequest();
+ tag_request.withResources(inst.getInstanceId()).setTags(inst_tags);
+ ec2.createTags(tag_request);
+
+ // That was a remote request - we should also update our local instance data.
+ inst.setTags(inst_tags);
+ }
+ logger.println("No existing instance found - created: "+inst);
+ return newSlave(inst);
+ }
+
+ Instance inst = diResult.getReservations().get(0).getInstances().get(0);
+ logger.println("Found existing stopped instance: "+inst);
+ List<String> instances = new ArrayList<String>();
+ instances.add(inst.getInstanceId());
+ StartInstancesRequest siRequest = new StartInstancesRequest(instances);
+ StartInstancesResult siResult = ec2.startInstances(siRequest);
+ logger.println("Starting existing instance: "+inst+ " result:"+siResult);
+
+ List<Node> nodes = Hudson.getInstance().getNodes();
+ for (int i = 0, len = nodes.size(); i < len; i++) {
+ if (!(nodes.get(i) instanceof EC2Slave))
+ continue;
+ EC2Slave ec2Node = (EC2Slave) nodes.get(i);
+ if (ec2Node.getInstanceId().equals(inst.getInstanceId())) {
+ logger.println("Found existing corresponding: "+ec2Node);
+ return ec2Node;
+ }
+ }
+
+ // Existing slave not found
+ logger.println("Creating new slave for existing instance: "+inst);
return newSlave(inst);
+
} catch (FormException e) {
throw new AssertionError(); // we should have discovered all configuration issues upfront
}
View
6 src/main/resources/hudson/plugins/ec2/EC2Slave/help-stopOnTerminate.html
@@ -0,0 +1,6 @@
+<div>
+ Check this if you want the instance to be stopped instead of terminated when it times out
+ due to being idle. This allows existing instances to be used when provisioning a slave.
+ You would use this option to preserve the state of the instance, including that of the
+ EBS volume connections.
+</div>
View
2  src/main/resources/hudson/plugins/ec2/SlaveTemplate/config.jelly
@@ -91,7 +91,7 @@ THE SOFTWARE.
<f:textbox />
</f:entry>
- <f:entry title="${%Stop on Terminate}" field="stopOnTerminate">
+ <f:entry title="${%Stop/Disconnect on Idle Timeout}" field="stopOnTerminate">
<f:checkbox />
</f:entry>
Please sign in to comment.
Something went wrong with that request. Please try again.