JIRA-14884 Resume Stopped Ec2 Instances

src/main/java/hudson/plugins/ec2/EC2Cloud.java

@@ -218,7 +218,8 @@ public void doProvision(StaplerRequest req, StaplerResponse rsp, @QueryParameter
 	public Collection<PlannedNode> provision(Label label, int excessWorkload) {
         try {
 
-            final SlaveTemplate t = getTemplate(label);
+
+        	final SlaveTemplate t = getTemplate(label);
 
             List<PlannedNode> r = new ArrayList<PlannedNode>();
             for( ; excessWorkload>0; excessWorkload-- ) {

src/main/java/hudson/plugins/ec2/EC2Computer.java

@@ -103,7 +103,8 @@ private Instance _describeInstance() throws AmazonClientException {
     @Override
     public HttpResponse doDoDelete() throws IOException {
         checkPermission(DELETE);
-        getNode().terminate();
+        if (getNode() != null)
+        	getNode().terminate();
         return new HttpRedirect("..");
     }
 

src/main/java/hudson/plugins/ec2/EC2ComputerLauncher.java

@@ -6,9 +6,14 @@
 
 import java.io.IOException;
 import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.List;
 
 import com.amazonaws.AmazonClientException;
+import com.amazonaws.services.ec2.AmazonEC2;
 import com.amazonaws.services.ec2.model.Instance;
+import com.amazonaws.services.ec2.model.StartInstancesRequest;
+import com.amazonaws.services.ec2.model.StartInstancesResult;
 
 /**
  * {@link ComputerLauncher} for EC2 that waits for the instance to really come up before proceeding to
@@ -27,10 +32,20 @@ public void launch(SlaveComputer _computer, TaskListener listener) {
             while(true) {
                 switch (computer.getState()) {
                     case PENDING:
+                    case STOPPING:
                         Thread.sleep(5000); // check every 5 secs
                         continue OUTER;
                     case RUNNING:
                         break OUTER;
+                    case STOPPED:
+                    	AmazonEC2 ec2 = EC2Cloud.get().connect();
+                        List<String> instances = new ArrayList<String>();
+                        instances.add(computer.getInstanceId());
+
+                        StartInstancesRequest siRequest = new StartInstancesRequest(instances);
+                        StartInstancesResult siResult = ec2.startInstances(siRequest);
+                        logger.println("Starting existing instance: "+computer.getInstanceId()+ " result:"+siResult);
+                        continue OUTER;
                     case SHUTTING_DOWN:
                     case TERMINATED:
                         // abort

src/main/java/hudson/plugins/ec2/EC2RetentionStrategy.java

@@ -40,13 +40,15 @@ public synchronized long check(EC2Computer c) {
 
         /* If we've been told never to terminate, then we're done. */
         if  (idleTerminationMinutes == 0) return 1;
-
-        if (c.isIdle() && !disabled) {
+        final long idleMilliseconds1 = System.currentTimeMillis() - c.getIdleStartMilliseconds();
+        System.out.println(c.getName() + " idle: " + idleMilliseconds1);
+
+        if (c.isIdle() && c.isOnline() && !disabled) {
             // TODO: really think about the right strategy here
             final long idleMilliseconds = System.currentTimeMillis() - c.getIdleStartMilliseconds();
             if (idleMilliseconds > TimeUnit2.MINUTES.toMillis(idleTerminationMinutes)) {
-                LOGGER.info("Disconnecting "+c.getName());
-                c.getNode().terminate();
+                LOGGER.info("Idle timeout: "+c.getName());
+                c.getNode().idleTimeout();
             }
         }
         return 1;

src/main/java/hudson/plugins/ec2/EC2Slave.java

@@ -29,6 +29,7 @@
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.services.ec2.AmazonEC2;
 import com.amazonaws.services.ec2.model.*;
+
 import net.sf.json.JSONObject;
 
 /**
@@ -137,27 +138,36 @@ public Computer createComputer() {
         return new EC2Computer(this);
     }
 
+    public static Instance getInstance(String instanceId) {
+        DescribeInstancesRequest request = new DescribeInstancesRequest();
+    	request.setInstanceIds(Collections.<String>singletonList(instanceId));
+        EC2Cloud cloudInstance = EC2Cloud.get();
+        if (cloudInstance == null)
+        	return null;
+        AmazonEC2 ec2 = cloudInstance.connect();
+    	List<Reservation> reservations = ec2.describeInstances(request).getReservations();
+        Instance i = null;
+    	if (reservations.size() > 0) {
+    		List<Instance> instances = reservations.get(0).getInstances();
+    		if (instances.size() > 0)
+    			i = instances.get(0);
+    	}
+    	return i;
+    }
+
     /**
      * Terminates the instance in EC2.
      */
     public void terminate() {
         try {
             if (!isAlive(true)) {
                 /* The node has been killed externally, so we've nothing to do here */
-                LOGGER.info("EC2 instance already terminated: " + getInstanceId());
+                LOGGER.info("EC2 instance already terminated: "+getInstanceId());
             } else {
-                /* The node is still alive - do the appropriate thing */
-
                 AmazonEC2 ec2 = EC2Cloud.get().connect();
-                if (stopOnTerminate) {
-                	StopInstancesRequest request = new StopInstancesRequest(Collections.singletonList(getInstanceId()));
-                	ec2.stopInstances(request);
-                    LOGGER.info("Terminated EC2 instance (stopped): "+getInstanceId());
-                } else {
-                    TerminateInstancesRequest request = new TerminateInstancesRequest(Collections.singletonList(getInstanceId()));
-                	ec2.terminateInstances(request);
-                    LOGGER.info("Terminated EC2 instance (terminated): "+getInstanceId());
-                }
+                TerminateInstancesRequest request = new TerminateInstancesRequest(Collections.singletonList(getInstanceId()));
+                ec2.terminateInstances(request);
+                LOGGER.info("Terminated EC2 instance (terminated): "+getInstanceId());
             }
             Hudson.getInstance().removeNode(this);
         } catch (AmazonClientException e) {
@@ -167,6 +177,26 @@ public void terminate() {
         }
     }
 
+	void idleTimeout() {
+		LOGGER.info("EC2 instance idle time expired: "+getInstanceId());
+		if (!stopOnTerminate) {
+			terminate();
+			return;
+		}
+
+		try {
+			AmazonEC2 ec2 = EC2Cloud.get().connect();
+			StopInstancesRequest request = new StopInstancesRequest(
+					Collections.singletonList(getInstanceId()));
+			ec2.stopInstances(request);
+			toComputer().disconnect(null);
+		} catch (AmazonClientException e) {
+	        Instance i = getInstance(getNodeName());
+			LOGGER.log(Level.WARNING, "Failed to terminate EC2 instance: "+getInstanceId() + " info: "+((i != null)?i:"") , e);
+		}
+		LOGGER.info("EC2 instance stopped: " + getInstanceId());
+	}
+
     String getRemoteAdmin() {
         if (remoteAdmin == null || remoteAdmin.length() == 0)
             return "root";
@@ -207,12 +237,12 @@ private void fetchLiveInstanceData( boolean force ) throws AmazonClientException
             return;
         }
 
-        DescribeInstancesRequest request = new DescribeInstancesRequest();
-    	request.setInstanceIds(Collections.<String>singletonList(getNodeName()));
-        Instance i = EC2Cloud.get().connect().describeInstances(request).getReservations().get(0).getInstances().get(0);
+        Instance i = getInstance(getNodeName());
 
         lastFetchTime = now;
         lastFetchInstance = i;
+        if (i == null)
+        	return;
 
         publicDNS = i.getPublicDnsName();
         privateDNS = i.getPrivateIpAddress();
@@ -226,9 +256,7 @@ private void fetchLiveInstanceData( boolean force ) throws AmazonClientException
 
 	/* Clears all existing tag data so that we can force the instance into a known state */
     private void clearLiveInstancedata() throws AmazonClientException {
-        DescribeInstancesRequest request = new DescribeInstancesRequest();
-        request.setInstanceIds( Collections.<String>singletonList(getNodeName()));
-        Instance inst = EC2Cloud.get().connect().describeInstances(request).getReservations().get(0).getInstances().get(0);
+        Instance inst = getInstance(getNodeName());
 
         /* Now that we have our instance, we can clear the tags on it */
         if (!tags.isEmpty()) {
@@ -247,9 +275,7 @@ private void clearLiveInstancedata() throws AmazonClientException {
 
     /* Sets tags on an instance.  This will not clear existing tag data, so call clearLiveInstancedata if needed */
     private void pushLiveInstancedata() throws AmazonClientException {
-        DescribeInstancesRequest request = new DescribeInstancesRequest();
-        request.setInstanceIds(Collections.<String>singletonList(getNodeName()));
-        Instance inst = EC2Cloud.get().connect().describeInstances(request).getReservations().get(0).getInstances().get(0);
+        Instance inst = getInstance(getNodeName());
 
         /* Now that we have our instance, we can set tags on it */
         if (tags != null && !tags.isEmpty()) {

src/main/java/hudson/plugins/ec2/SlaveTemplate.java

@@ -160,7 +160,7 @@ public String getidleTerminationMinutes() {
         return idleTerminationMinutes;
     }
 
-    public Set getLabelSet(){
+    public Set<LabelAtom> getLabelSet(){
         return labelSet;
     }
 
@@ -202,15 +202,20 @@ public EC2Slave provision(TaskListener listener) throws AmazonClientException, I
                 throw new AmazonClientException("No matching keypair found on EC2. Is the EC2 private key a valid one?");
             }
 
-            RunInstancesRequest request = new RunInstancesRequest(ami, 1, 1);
+            RunInstancesRequest riRequest = new RunInstancesRequest(ami, 1, 1);
 
+            List<Filter> diFilters = new ArrayList<Filter>();
+            diFilters.add(new Filter("image-id").withValues(ami));
+
             if (StringUtils.isNotBlank(getZone())) {
             	Placement placement = new Placement(getZone());
-            	request.setPlacement(placement);
+            	riRequest.setPlacement(placement);
+                diFilters.add(new Filter("availability-zone").withValues(getZone()));
             }
 
             if (StringUtils.isNotBlank(getSubnetId())) {
-               request.setSubnetId(getSubnetId());
+               riRequest.setSubnetId(getSubnetId());
+               diFilters.add(new Filter("subnet-id").withValues(getSubnetId()));
 
                /* If we have a subnet ID then we can only use VPC security groups */
                if (!securityGroupSet.isEmpty()) {
@@ -243,36 +248,79 @@ public EC2Slave provision(TaskListener listener) throws AmazonClientException, I
                   }
 
                   if (!group_ids.isEmpty()) {
-                     request.setSecurityGroupIds(group_ids);
+                     riRequest.setSecurityGroupIds(group_ids);
+                     diFilters.add(new Filter("instance.group-id").withValues(group_ids));
                   }
                }
             } else {
                /* No subnet: we can use standard security groups by name */
-               request.setSecurityGroups(securityGroupSet);
+            	riRequest.setSecurityGroups(securityGroupSet);
+            	if (securityGroupSet.size() > 0)
+            		diFilters.add(new Filter("group-id").withValues(securityGroupSet));
             }
 
-            request.setUserData(Base64.encodeBase64String(userData.getBytes()));
-            request.setKeyName(keyPair.getKeyName());
-            request.setInstanceType(type.toString());
-            Instance inst = ec2.runInstances(request).getReservation().getInstances().get(0);
-
-            /* Now that we have our instance, we can set tags on it */
+            String userDataString = Base64.encodeBase64String(userData.getBytes());
+            riRequest.setUserData(userDataString);
+            riRequest.setKeyName(keyPair.getKeyName());
+            diFilters.add(new Filter("key-name").withValues(keyPair.getKeyName()));
+            riRequest.setInstanceType(type.toString());
+            diFilters.add(new Filter("instance-type").withValues(type.toString()));
+
+            HashSet<Tag> inst_tags = null;
             if (tags != null && !tags.isEmpty()) {
-                HashSet<Tag> inst_tags = new HashSet<Tag>();
-
+                inst_tags = new HashSet<Tag>();
                 for(EC2Tag t : tags) {
-                    inst_tags.add(new Tag(t.getName(), t.getValue()));
+                    diFilters.add(new Filter("tag:"+t.getName()).withValues(t.getValue()));
                 }
-
-                CreateTagsRequest tag_request = new CreateTagsRequest();
-                tag_request.withResources(inst.getInstanceId()).setTags(inst_tags);
-                ec2.createTags(tag_request);
-
-                // That was a remote request - we should also update our local instance data.
-                inst.setTags(inst_tags);
             }
-
+
+            DescribeInstancesRequest diRequest = new DescribeInstancesRequest();
+            diFilters.add(new Filter("instance-state-name").withValues(InstanceStateName.Stopped.toString(), 
+            		InstanceStateName.Stopping.toString()));
+            diRequest.setFilters(diFilters);
+            logger.println("Looking for existing instances: "+diRequest);
+
+            DescribeInstancesResult diResult = ec2.describeInstances(diRequest);
+            if (diResult.getReservations().size() == 0) {
+                // Have to create a new instance
+                Instance inst = ec2.runInstances(riRequest).getReservation().getInstances().get(0);
+
+                /* Now that we have our instance, we can set tags on it */
+                if (inst_tags != null) {
+                    CreateTagsRequest tag_request = new CreateTagsRequest();
+                    tag_request.withResources(inst.getInstanceId()).setTags(inst_tags);
+                    ec2.createTags(tag_request);
+
+                    // That was a remote request - we should also update our local instance data.
+                    inst.setTags(inst_tags);
+                }
+                logger.println("No existing instance found - created: "+inst);
+                return newSlave(inst);
+            }
+
+            Instance inst = diResult.getReservations().get(0).getInstances().get(0);
+            logger.println("Found existing stopped instance: "+inst);
+            List<String> instances = new ArrayList<String>();
+            instances.add(inst.getInstanceId());
+            StartInstancesRequest siRequest = new StartInstancesRequest(instances);
+            StartInstancesResult siResult = ec2.startInstances(siRequest);
+            logger.println("Starting existing instance: "+inst+ " result:"+siResult);
+
+            List<Node> nodes = Hudson.getInstance().getNodes();
+            for (int i = 0, len = nodes.size(); i < len; i++) {
+            	if (!(nodes.get(i) instanceof EC2Slave))
+            		continue;
+            	EC2Slave ec2Node = (EC2Slave) nodes.get(i);
+            	if (ec2Node.getInstanceId().equals(inst.getInstanceId())) {
+                    logger.println("Found existing corresponding: "+ec2Node);
+            		return ec2Node;
+            	}
+            }
+
+            // Existing slave not found 
+            logger.println("Creating new slave for existing instance: "+inst);
             return newSlave(inst);
+
         } catch (FormException e) {
             throw new AssertionError(); // we should have discovered all configuration issues upfront
         }

src/main/resources/hudson/plugins/ec2/EC2Slave/help-stopOnTerminate.html

@@ -0,0 +1,6 @@
+<div>
+    Check this if you want the instance to be stopped instead of terminated when it times out
+    due to being idle. This allows existing instances to be used when provisioning a slave.
+    You would use this option to preserve the state of the instance, including that of the 
+    EBS volume connections.
+</div>

src/main/resources/hudson/plugins/ec2/SlaveTemplate/config.jelly

@@ -91,7 +91,7 @@ THE SOFTWARE.
       <f:textbox />
     </f:entry>
 
-    <f:entry title="${%Stop on Terminate}" field="stopOnTerminate">
+    <f:entry title="${%Stop/Disconnect on Idle Timeout}" field="stopOnTerminate">
       <f:checkbox />
     </f:entry>