Skip to content
This repository
  • 8 commits
  • 19 files changed
  • 0 comments
  • 1 contributor

Showing 19 changed files with 1,112 additions and 16 deletions. Show diff stats Hide diff stats

  1. +1 0  service-api/java/source/src/org/nimbustools/api/_repr/vm/_Schedule.java
  2. +10 0 service-api/java/source/src/org/nimbustools/api/defaults/repr/vm/DefaultSchedule.java
  3. +1 0  service-api/java/source/src/org/nimbustools/api/repr/vm/Schedule.java
  4. +112 0 service/service/java/source/etc/workspace-service/backfill.conf
  5. +10 0 service/service/java/source/etc/workspace-service/other/main.conflocator.xml
  6. +31 0 service/service/java/source/etc/workspace-service/other/main.xml
  7. +1 0  service/service/java/source/etc/workspace-service/other/resource-locator-ACTIVE.xml
  8. +1 0  service/service/java/source/etc/workspace-service/other/resource-locator-default.xml
  9. +484 0 service/service/java/source/src/org/globus/workspace/Backfill.java
  10. +65 0 service/service/java/source/src/org/globus/workspace/BackfillTimer.java
  11. +51 7 service/service/java/source/src/org/globus/workspace/creation/defaults/DefaultCreation.java
  12. +11 1 service/service/java/source/src/org/globus/workspace/manager/DelegatingManager.java
  13. +3 1 service/service/java/source/src/org/globus/workspace/scheduler/Scheduler.java
  14. +85 0 service/service/java/source/src/org/globus/workspace/scheduler/defaults/BackfillNode.java
  15. +50 4 service/service/java/source/src/org/globus/workspace/scheduler/defaults/DefaultSchedulerAdapter.java
  16. +157 1 service/service/java/source/src/org/globus/workspace/scheduler/defaults/DefaultSlotManagement.java
  17. +15 2 service/service/java/source/src/org/globus/workspace/scheduler/defaults/NodeRequest.java
  18. +5 0 service/service/java/source/src/org/globus/workspace/scheduler/defaults/SlotManagement.java
  19. +19 0 service/service/java/source/src/org/globus/workspace/scheduler/defaults/pilot/PilotSlotManagement.java
1  service-api/java/source/src/org/nimbustools/api/_repr/vm/_Schedule.java
@@ -25,4 +25,5 @@
25 25 public void setStartTime(Calendar startTime);
26 26 public void setDurationSeconds(int durationSeconds);
27 27 public void setDestructionTime(Calendar destructionTime);
  28 + public void setBackfillReq(boolean backfillReq);
28 29 }
10 service-api/java/source/src/org/nimbustools/api/defaults/repr/vm/DefaultSchedule.java
@@ -37,6 +37,7 @@
37 37 private Calendar startTime;
38 38 private int durationSeconds = -1;
39 39 private Calendar destructionTime;
  40 + private boolean backfillReq = false;
40 41
41 42
42 43 // -------------------------------------------------------------------------
@@ -55,6 +56,10 @@ public Calendar getDestructionTime() {
55 56 return this.destructionTime;
56 57 }
57 58
  59 + public boolean getBackfillReq() {
  60 + return this.backfillReq;
  61 + }
  62 +
58 63
59 64 // -------------------------------------------------------------------------
60 65 // implements org.nimbustools.api._repr.vm._Schedule
@@ -72,6 +77,10 @@ public void setDestructionTime(Calendar destructionTime) {
72 77 this.destructionTime = destructionTime;
73 78 }
74 79
  80 + public void setBackfillReq(boolean backfillReq) {
  81 + this.backfillReq = backfillReq;
  82 + }
  83 +
75 84
76 85 // -------------------------------------------------------------------------
77 86 // DEBUG STRING
@@ -82,6 +91,7 @@ public String toString() {
82 91 "startTime=" + calendarStr(this.startTime) +
83 92 ", durationSeconds='" + this.durationSeconds +
84 93 "', destructionTime='" + calendarStr(this.destructionTime) +
  94 + ", backfillReq=" + this.backfillReq +
85 95 "'}";
86 96 }
87 97
1  service-api/java/source/src/org/nimbustools/api/repr/vm/Schedule.java
@@ -23,4 +23,5 @@
23 23 public Calendar getStartTime();
24 24 public int getDurationSeconds();
25 25 public Calendar getDestructionTime();
  26 + public boolean getBackfillReq();
26 27 }
112 service/service/java/source/etc/workspace-service/backfill.conf
... ... @@ -0,0 +1,112 @@
  1 +################################################################################
  2 +#
  3 +# This file is used for configuring VM backfill deployment on idle VMM nodes.
  4 +#
  5 +# The intent of backfill is to provide a Nimbus cloud with the ability to
  6 +# deploy a generic VM on idle nodes. Such a VM could be configured with a
  7 +# service like Condor, allowing the VM to contribute cycles to some other
  8 +# purpose instead of wasting cycles on idle cloud nodes. When a user request is
  9 +# recieved the VM is terminated immediately and the user VM is deployed.
  10 +#
  11 +# NOTE: Backfill must be configured and run by the Nimbus administrator. It is
  12 +# not currently a feature that users can leverage directly.
  13 +#
  14 +# The VM used by backfill (and the services inside of it) must be able to
  15 +# handle a hard shutdown. A hard shutdown is used to minimize the turn-
  16 +# around time for responding to user requests.
  17 +#
  18 +################################################################################
  19 +
  20 +
  21 +# BACKFILL
  22 +#
  23 +# If you would like to use backfill, you can enable it here.
  24 +#
  25 +# The default is disabled.
  26 +
  27 +backfill.disabled=true
  28 +
  29 +
  30 +# MAX INSTANCES
  31 +#
  32 +# Max instances is the maximum number of VM instances that backfill will deploy
  33 +# if it is enabled. If there is not enough space on the cloud for the maximum
  34 +# number of instances it will deploy as many as it can. For example, if max
  35 +# instances is set to 12 on a 16 node cloud but there are 10 active user VMs
  36 +# then backfill will still launch 6 backfill nodes.
  37 +#
  38 +# If max instances is set to 0 then backfill will use all idle VMMs.
  39 +#
  40 +# The default is 0.
  41 +
  42 +max.instances=0
  43 +
  44 +
  45 +# DISK IMAGE
  46 +#
  47 +# The disk image is currently a pointer to the image file on the VMM nodes.
  48 +# The image must be in the same location on all nodes. Also, the image must
  49 +# be copied there manually.
  50 +
  51 +disk.image=/usr/local/myawesomeimage.img
  52 +
  53 +
  54 +# INSTANCE MEMORY
  55 +#
  56 +# The instance memory is the amount of memory (in MB) that the backfill image
  57 +# will use.
  58 +
  59 +memory.MB=64
  60 +
  61 +
  62 +# INSTANCE DURATION
  63 +#
  64 +# Instance duration is the amount of time (in seconds) that the backfill instance
  65 +# will potentially be active. Obviously, the nature of backfill instances
  66 +# necessitates that when a user request arrives the backfill instance will be
  67 +# preempted, regardless of this setting. When the duration has elapsed the
  68 +# instance will be terminated. It is possible that another backfill instance
  69 +# will be dispatched almost immediately after the previous one was terminated.
  70 +#
  71 +# The default duration is one week.
  72 +
  73 +duration.seconds=604800
  74 +
  75 +
  76 +# INSTANCE TERMINATION POLICY
  77 +#
  78 +# The instance termination policy specifies the mechanism that will be used
  79 +# when selecting a backfill VM to terminate in order to satisfy a user
  80 +# request.
  81 +#
  82 +# Options:
  83 +#
  84 +# - ANY this will pick any VM to terminate, it doesn't consider
  85 +# any factors (such as time running) when deciding, it
  86 +# simply grabs the next VM instance out of the hash table
  87 +#
  88 +# - MOST_RECENT this will pick the VM that has been running for the least
  89 +# amount of time
  90 +#
  91 +# The default is MOST_RECENT.
  92 +
  93 +termination.policy=MOST_RECENT
  94 +
  95 +
  96 +# INSTANCE RETRY PERIOD
  97 +#
  98 +# Instance retry period is the amount of time (in seconds) the backfill timer
  99 +# waits in between attempts to launch backfill VMs on available cloud nodes.
  100 +# This timer is restarted (and hence, it attempts to launch backfill VMs)
  101 +# whenever a user VM is terminated (thus freeing a node for backfill).
  102 +#
  103 +# The default is five minutes.
  104 +
  105 +retry.period=300
  106 +
  107 +
  108 +# NETWORK
  109 +#
  110 +# Set the network to public or private.
  111 +
  112 +network=public
10 service/service/java/source/etc/workspace-service/other/main.conflocator.xml
@@ -73,6 +73,16 @@
73 73 value="$ACCOUNTING{" />
74 74 </bean>
75 75
  76 + <bean id="backfillSettings"
  77 + class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
  78 +
  79 + <property name="location"
  80 + value="$NIMBUS_HOME/services/etc/nimbus/workspace-service/backfill.conf" />
  81 + <property name="placeholderPrefix"
  82 + value="$BACKFILL{" />
  83 +
  84 + </bean>
  85 +
76 86 <bean id="sshSettings"
77 87 class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
78 88
31 service/service/java/source/etc/workspace-service/other/main.xml
@@ -20,6 +20,7 @@
20 20 <constructor-arg ref="nimbus-repr.ReprFactory"/>
21 21 <constructor-arg ref="nimbus-rm.persistence.DataConvert"/>
22 22 <constructor-arg ref="nimbus-rm.loglevels" />
  23 + <constructor-arg ref="nimbus-rm.backfill" />
23 24
24 25 <property name="accounting"
25 26 ref="nimbus-rm.accounting" />
@@ -90,6 +91,7 @@
90 91 <constructor-arg ref="nimbus-rm.home.cosched" />
91 92 <constructor-arg ref="other.timerManager" />
92 93 <constructor-arg ref="nimbus-rm.loglevels" />
  94 + <constructor-arg ref="nimbus-rm.backfill" />
93 95
94 96 <property name="accountingEventAdapter"
95 97 ref="nimbus-rm.accounting" />
@@ -253,6 +255,34 @@
253 255
254 256 </bean>
255 257
  258 + <bean id="nimbus-rm.backfill"
  259 + class="org.globus.workspace.Backfill"
  260 + init-method="validate">
  261 +
  262 + <constructor-arg ref="nimbus-repr.ReprFactory"/>
  263 + <constructor-arg ref="nimbus-rm.PathConfigs" />
  264 + <constructor-arg ref="other.timerManager" />
  265 + <constructor-arg ref="nimbus-rm.scheduler.SlotManagement" />
  266 +
  267 + <property name="backfillDisabled"
  268 + value="$BACKFILL{backfill.disabled}" />
  269 + <property name="maxInstances"
  270 + value="$BACKFILL{max.instances}" />
  271 + <property name="diskImage"
  272 + value="$BACKFILL{disk.image}" />
  273 + <property name="memoryMB"
  274 + value="$BACKFILL{memory.MB}" />
  275 + <property name="durationSeconds"
  276 + value="$BACKFILL{duration.seconds}" />
  277 + <property name="terminationPolicy"
  278 + value="$BACKFILL{termination.policy}" />
  279 + <property name="retryPeriod"
  280 + value="$BACKFILL{retry.period}" />
  281 + <property name="network"
  282 + value="$BACKFILL{network}" />
  283 +
  284 + </bean>
  285 +
256 286
257 287 <!-- ===================================================================
258 288 org.globus.workspace.network.* interfaces
@@ -525,6 +555,7 @@
525 555 <constructor-arg ref="nimbus-rm.service.binding.GlobalPolicies" />
526 556 <constructor-arg ref="nimbus-rm.persistence.DataConvert" />
527 557 <constructor-arg ref="nimbus-rm.loglevels" />
  558 + <constructor-arg ref="nimbus-rm.backfill" />
528 559
529 560
530 561
1  service/service/java/source/etc/workspace-service/other/resource-locator-ACTIVE.xml
@@ -21,6 +21,7 @@
21 21
22 22 <constructor-arg ref="nimbus-rm.persistence.PersistenceAdapter" />
23 23 <constructor-arg ref="nimbus-rm.loglevels" />
  24 + <constructor-arg ref="nimbus-rm.PathConfigs" />
24 25
25 26 <!-- set after object creation time to avoid circular dep with home -->
26 27 <property name="home" ref="nimbus-rm.home.instance" />
1  service/service/java/source/etc/workspace-service/other/resource-locator-default.xml
@@ -21,6 +21,7 @@
21 21
22 22 <constructor-arg ref="nimbus-rm.persistence.PersistenceAdapter" />
23 23 <constructor-arg ref="nimbus-rm.loglevels" />
  24 + <constructor-arg ref="nimbus-rm.PathConfigs" />
24 25
25 26 <!-- set after object creation time to avoid circular dep with home -->
26 27 <property name="home" ref="nimbus-rm.home.instance" />
484 service/service/java/source/src/org/globus/workspace/Backfill.java
... ... @@ -0,0 +1,484 @@
  1 +/*
  2 + * Copyright 1999-2010 University of Chicago
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License"); you may not
  5 + * use this file except in compliance with the License. You may obtain a copy
  6 + * of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12 + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13 + * License for the specific language governing permissions and limitations
  14 + * under the License.
  15 + */
  16 +
  17 +package org.globus.workspace;
  18 +
  19 +import org.globus.workspace.manager.DelegatingManager;
  20 +import org.globus.workspace.scheduler.defaults.SlotManagement;
  21 +
  22 +import org.nimbustools.api._repr._Caller;
  23 +import org.nimbustools.api._repr._CreateRequest;
  24 +import org.nimbustools.api._repr.vm._NIC;
  25 +import org.nimbustools.api._repr.vm._RequiredVMM;
  26 +import org.nimbustools.api._repr.vm._ResourceAllocation;
  27 +import org.nimbustools.api._repr.vm._Schedule;
  28 +import org.nimbustools.api._repr.vm._VMFile;
  29 +import org.nimbustools.api.repr.Caller;
  30 +import org.nimbustools.api.repr.CreateRequest;
  31 +import org.nimbustools.api.repr.CreateResult;
  32 +import org.nimbustools.api.repr.vm.NIC;
  33 +import org.nimbustools.api.repr.vm.ResourceAllocation;
  34 +import org.nimbustools.api.repr.vm.VMFile;
  35 +import org.nimbustools.api.repr.ReprFactory;
  36 +
  37 +import org.apache.commons.logging.Log;
  38 +import org.apache.commons.logging.LogFactory;
  39 +
  40 +import java.util.Date;
  41 +import java.net.URI;
  42 +import java.io.File;
  43 +import java.io.FileReader;
  44 +import java.io.FileWriter;
  45 +import java.io.BufferedReader;
  46 +import java.io.BufferedWriter;
  47 +import java.io.FileNotFoundException;
  48 +
  49 +import commonj.timers.Timer;
  50 +import commonj.timers.TimerManager;
  51 +
  52 +public class Backfill {
  53 +
  54 + private static final Log logger =
  55 + LogFactory.getLog(Backfill.class.getName());
  56 +
  57 + protected final ReprFactory repr;
  58 + protected final PathConfigs pathConfigs;
  59 + protected final TimerManager timerManager;
  60 + protected final SlotManagement slotManager;
  61 + protected DelegatingManager manager;
  62 +
  63 + private boolean backfillDisabled;
  64 + private int maxInstances;
  65 + private int curNumInstances = 0;
  66 + private String diskImage;
  67 + private int memoryMB;
  68 + private int durationSeconds;
  69 + private String terminationPolicy;
  70 + private int retryPeriod;
  71 + private String network;
  72 + private File backfillReqFile;
  73 + private Timer backfillTimer = null;
  74 +
  75 + public Backfill(ReprFactory reprFactory,
  76 + PathConfigs pathConfigs,
  77 + TimerManager timerManager,
  78 + SlotManagement slotManager) {
  79 +
  80 + if (reprFactory == null) {
  81 + throw new IllegalArgumentException("reprFactory may not be null");
  82 + }
  83 + this.repr = reprFactory;
  84 +
  85 + if (pathConfigs == null) {
  86 + throw new IllegalArgumentException("pathConfigs may not be null");
  87 + }
  88 + this.pathConfigs = pathConfigs;
  89 + this.backfillReqFile =
  90 + new File(this.pathConfigs.getLocalTempDirPath() +
  91 + "/backfill.request");
  92 +
  93 + if (timerManager == null) {
  94 + throw new IllegalArgumentException("timerManager may not be null");
  95 + }
  96 + this.timerManager = timerManager;
  97 +
  98 + if (slotManager == null) {
  99 + throw new IllegalArgumentException("slotManager may not be null");
  100 + }
  101 + this.slotManager = slotManager;
  102 + }
  103 +
  104 + public boolean getBackfillDisabled() {
  105 + return this.backfillDisabled;
  106 + }
  107 +
  108 + public int getMaxInstances() {
  109 + return this.maxInstances;
  110 + }
  111 +
  112 + public int getCurNumInstances() {
  113 + return this.curNumInstances;
  114 + }
  115 +
  116 + public String getDiskImage() {
  117 + return this.diskImage;
  118 + }
  119 +
  120 + public int getMemoryMB() {
  121 + return this.memoryMB;
  122 + }
  123 +
  124 + public int getDurationSeconds() {
  125 + return this.durationSeconds;
  126 + }
  127 +
  128 + public String getTerminationPolicy() {
  129 + return this.terminationPolicy;
  130 + }
  131 +
  132 + public int getRetryPeriod() {
  133 + return this.retryPeriod;
  134 + }
  135 +
  136 + public String getNetwork() {
  137 + return this.network;
  138 + }
  139 +
  140 + public void setBackfillDisabled(boolean backfillDisabled) {
  141 + this.backfillDisabled = backfillDisabled;
  142 + }
  143 +
  144 + public void setMaxInstances(int maxInstances) {
  145 + this.maxInstances = maxInstances;
  146 + }
  147 +
  148 + public void addCurInstances(int num) {
  149 + this.curNumInstances += num;
  150 + }
  151 +
  152 + public void subCurInstances(int num) {
  153 + this.curNumInstances -= num;
  154 + }
  155 +
  156 + public void setDiskImage(String diskImage) {
  157 + this.diskImage = diskImage;
  158 + }
  159 +
  160 + public void setMemoryMB(int memoryMB) {
  161 + this.memoryMB = memoryMB;
  162 + }
  163 +
  164 + public void setDurationSeconds(int durationSeconds) {
  165 + this.durationSeconds = durationSeconds;
  166 + }
  167 +
  168 + public void setTerminationPolicy(String terminationPolicy) {
  169 + this.terminationPolicy = terminationPolicy;
  170 + }
  171 +
  172 + public void setNetwork(String network) {
  173 + this.network = network;
  174 + }
  175 +
  176 + public void setRetryPeriod(int retryPeriod) {
  177 + this.retryPeriod = retryPeriod;
  178 + }
  179 +
  180 + public void setDelegatingManager(DelegatingManager manager) {
  181 + this.manager = manager;
  182 + }
  183 +
  184 + /**
  185 + * This method should be called on service startup. It is responsible for
  186 + * starting the backfill timer (if the backfill feature is enabled).
  187 + *
  188 + * It also checks to see if a persistence backfill file exists, and if the
  189 + * backfill configuration has changed. If the backfill configuration has
  190 + * changed then it cancels the previous backfill request and submits a new
  191 + * one.
  192 + */
  193 + public void initiateBackfill() {
  194 +
  195 + if (this.backfillDisabled == false) {
  196 + logger.debug("Backfill is enabled");
  197 +
  198 + if (this.backfillReqFile.exists()) {
  199 + String curBackfillStr = buildCurBackfillReqStrB().toString();
  200 + String prevBackfillStr = readPrevBackfillReqStrB().toString();
  201 +
  202 + if ((curBackfillStr.compareTo(prevBackfillStr)) == 0) {
  203 + logger.debug("Current and previous backfill requests " +
  204 + "match");
  205 + } else {
  206 + logger.debug("The current backfill request doesn't " +
  207 + "match the previous backfill request");
  208 +
  209 + this.cancelBackfillRequest();
  210 + this.writeCurBackfillReq();
  211 +
  212 + logger.debug("New backfill request file written.");
  213 + }
  214 + } else {
  215 + this.writeCurBackfillReq();
  216 + }
  217 +
  218 + logger.debug("Launching backfill timer.");
  219 + this.launchBackfillTimer();
  220 + logger.debug("Backfill timer launched.");
  221 +
  222 + } else {
  223 + logger.info("Backfill is disabled");
  224 + }
  225 + }
  226 +
  227 + public void launchBackfillTimer() {
  228 +
  229 + if (this.backfillDisabled == true) {
  230 + return;
  231 + }
  232 +
  233 + if (this.backfillTimer != null) {
  234 + this.backfillTimer.cancel();
  235 + }
  236 +
  237 + BackfillTimer backfillTimer;
  238 + backfillTimer = new BackfillTimer(this);
  239 + Date backfillStart = new Date();
  240 + this.backfillTimer = this.timerManager.schedule(backfillTimer,
  241 + backfillStart,
  242 + this.retryPeriod * 1000);
  243 + }
  244 +
  245 + public void cancelBackfillRequest() {
  246 + logger.info("Cancelling backfill request");
  247 +
  248 + try {
  249 + this.backfillTimer.cancel();
  250 + } catch (Exception e) {
  251 + logger.debug("Failed to kill backfill timer, error: " +
  252 + e.getMessage());
  253 + }
  254 +
  255 + try {
  256 + this.backfillReqFile.delete();
  257 + } catch (Exception e) {
  258 + logger.debug("Problem deleting backfill request file: " +
  259 + e.getMessage());
  260 + }
  261 +
  262 + logger.debug("Backfill request cancelled.");
  263 + }
  264 +
  265 + // Returns the number of nodes successfully terminated
  266 + public int terminateBackfillNode(int numNodes) {
  267 +
  268 + int vmid;
  269 + int count = 0;
  270 + int successfulTerminations = 0;
  271 +
  272 + while (count < numNodes) {
  273 + if ((this.terminationPolicy.compareTo("ANY")) == 0) {
  274 + logger.debug("Backfill is using the ANY policy");
  275 + vmid = this.slotManager.getBackfillVMID();
  276 + } else {
  277 + logger.debug("Backfill is using the MOST_RECENT policy");
  278 + vmid = this.slotManager.getMostRecentBackfillVMID();
  279 + }
  280 +
  281 + if (vmid != -1) {
  282 + String vmidStr = Integer.toString(vmid);
  283 + logger.debug("Terminating backfill node with ID: " + vmidStr);
  284 +
  285 + Caller caller = this.getBackfillCaller();
  286 + try {
  287 + this.manager.trash(vmidStr, 0, caller);
  288 + this.subCurInstances(1);
  289 + successfulTerminations += 1;
  290 + } catch (Exception e) {
  291 + logger.error("Problem terminating backfill node: " +
  292 + e.getMessage());
  293 + }
  294 + } else {
  295 + logger.debug("No backfill VM to terminate");
  296 + }
  297 +
  298 + count += 1;
  299 + }
  300 +
  301 + return successfulTerminations;
  302 + }
  303 +
  304 + /**
  305 + * This attempts to launch a new backfill node.
  306 + * It should only be called by the backfill timer (ugh,
  307 + * should really be changed so ONLY that can happen).
  308 + * If backfill nodes should be launched (at some place else
  309 + * in the code) then the backfill timer should simply be relaunched
  310 + * (via the launchBackfillTimer method in this class)
  311 + * as it will attempt to launch backfill nodes. The timer is what
  312 + * respects the max backfill instances configuration value, and if
  313 + * backfill is disabled then it won't be relaunched.
  314 + */
  315 + public void createBackfillNode() throws Exception {
  316 +
  317 + CreateRequest req = this.getBackfillRequest("BACKFILL_REQUEST");
  318 + logger.debug("Backfill create request:\n" + req.toString());
  319 +
  320 + Caller caller = this.getBackfillCaller();
  321 + logger.debug("Backfill caller:\n" + caller.toString());
  322 +
  323 + CreateResult create = this.manager.create(req, caller);
  324 + }
  325 +
  326 + /**
  327 + * writeCurBackfillReq, readPrevBackfillReqStrB, and
  328 + * buildCurBackfillReqStrB are simply a hack to support persistence
  329 + * between service restarts or crashes. Ideally, they should become
  330 + * obsolete once the backfill feature is integrated with support for Spot
  331 + * Instances.
  332 + */
  333 + private void writeCurBackfillReq() {
  334 +
  335 + if (this.backfillReqFile.exists()) {
  336 + logger.debug("Backfill request file already exists");
  337 + logger.debug("Skipping write to backfill request file");
  338 + } else {
  339 + logger.debug("Attempting to write backfill file:\n" +
  340 + this.backfillReqFile.toString());
  341 +
  342 + try {
  343 + FileWriter backfillFW = new FileWriter(this.backfillReqFile);
  344 + BufferedWriter backfillBW = new BufferedWriter(backfillFW);
  345 +
  346 + this.backfillReqFile.createNewFile();
  347 +
  348 + backfillBW.write(this.buildCurBackfillReqStrB().toString());
  349 +
  350 + backfillBW.close();
  351 + } catch (Exception e) {
  352 + logger.debug("Error creating and writing a new backfill " +
  353 + "request file: " + e.getMessage());
  354 + }
  355 + }
  356 + }
  357 +
  358 + private StringBuilder readPrevBackfillReqStrB() {
  359 +
  360 + StringBuilder prevBackfillStrB = new StringBuilder();
  361 +
  362 + try {
  363 + logger.debug("Attempting to read backfill request file:\n" +
  364 + this.backfillReqFile.toString());
  365 +
  366 + FileReader backfillFR = new FileReader(this.backfillReqFile);
  367 + BufferedReader backfillBR = new BufferedReader(backfillFR);
  368 + String line;
  369 +
  370 + while (( line = backfillBR.readLine()) != null) {
  371 + prevBackfillStrB.append(line);
  372 + prevBackfillStrB.append("\n");
  373 + }
  374 +
  375 + logger.debug("For the previous backfill request file," +
  376 + " we read:\n" + prevBackfillStrB.toString());
  377 +
  378 + backfillBR.close();
  379 + } catch (FileNotFoundException e) {
  380 + logger.debug("Can't read a file that doesn't exist:\n" +
  381 + this.backfillReqFile.toString());
  382 + } catch (Exception e) {
  383 + logger.info("Unknown problem reading backfill request file: " +
  384 + e.getMessage());
  385 + }
  386 +
  387 + return prevBackfillStrB;
  388 + }
  389 +
  390 + private StringBuilder buildCurBackfillReqStrB() {
  391 +
  392 + StringBuilder curBackfillReqStrB = new StringBuilder();
  393 +
  394 + curBackfillReqStrB.append(Boolean.toString(this.backfillDisabled));
  395 + curBackfillReqStrB.append("\n");
  396 + curBackfillReqStrB.append(Integer.toString(this.maxInstances));
  397 + curBackfillReqStrB.append("\n");
  398 + curBackfillReqStrB.append(this.diskImage);
  399 + curBackfillReqStrB.append("\n");
  400 + curBackfillReqStrB.append(Integer.toString(this.memoryMB));
  401 + curBackfillReqStrB.append("\n");
  402 + curBackfillReqStrB.append(Integer.toString(this.durationSeconds));
  403 + curBackfillReqStrB.append("\n");
  404 + curBackfillReqStrB.append(this.network);
  405 + curBackfillReqStrB.append("\n");
  406 + curBackfillReqStrB.append(this.pathConfigs.getLocalTempDirPath());
  407 + curBackfillReqStrB.append("\n");
  408 +
  409 + return curBackfillReqStrB;
  410 + }
  411 +
  412 + private Caller getBackfillCaller() {
  413 + final _Caller caller = this.repr._newCaller();
  414 + caller.setIdentity("BACKFILL_CALLER");
  415 + return caller;
  416 + }
  417 +
  418 + private CreateRequest getBackfillRequest(String name) throws Exception {
  419 +
  420 + final _CreateRequest req = this.repr._newCreateRequest();
  421 + req.setName(name);
  422 +
  423 + final _NIC nic = this.repr._newNIC();
  424 + nic.setNetworkName(this.network);
  425 + nic.setAcquisitionMethod(NIC.ACQUISITION_AllocateAndConfigure);
  426 + req.setRequestedNics(new _NIC[]{nic});
  427 +
  428 + final _ResourceAllocation ra = this.repr._newResourceAllocation();
  429 + req.setRequestedRA(ra);
  430 + final _Schedule schedule = this.repr._newSchedule();
  431 + schedule.setDurationSeconds(this.durationSeconds);
  432 + schedule.setBackfillReq(true);
  433 + req.setRequestedSchedule(schedule);
  434 + ra.setNodeNumber(1);
  435 + ra.setMemory(this.memoryMB);
  436 + req.setShutdownType(CreateRequest.SHUTDOWN_TYPE_TRASH);
  437 + req.setInitialStateRequest(CreateRequest.INITIAL_STATE_RUNNING);
  438 +
  439 + ra.setArchitecture(ResourceAllocation.ARCH_x86);
  440 + final _RequiredVMM reqVMM = this.repr._newRequiredVMM();
  441 + reqVMM.setType("Xen");
  442 + reqVMM.setVersions(new String[]{"3"});
  443 + req.setRequiredVMM(reqVMM);
  444 +
  445 + final _VMFile file = this.repr._newVMFile();
  446 + file.setRootFile(true);
  447 + file.setBlankSpaceName(null);
  448 + file.setBlankSpaceSize(-1);
  449 + file.setURI(new URI("file://" + this.diskImage));
  450 + file.setMountAs("sda1");
  451 + file.setDiskPerms(VMFile.DISKPERMS_ReadWrite);
  452 + req.setVMFiles(new _VMFile[]{file});
  453 +
  454 + return req;
  455 + }
  456 +
  457 + public void validate() throws Exception {
  458 +
  459 + logger.debug("validating");
  460 +
  461 + if (this.maxInstances < 0) {
  462 + throw new Exception("maxInstances may not be less than 0");
  463 + }
  464 + if (this.diskImage == null) {
  465 + throw new Exception("diskImage may not be null");
  466 + }
  467 + if (this.memoryMB < 8) {
  468 + throw new Exception("memoryMB must be a reasonable value. " +
  469 + "Have you considered at least 64 MB?");
  470 + }
  471 + if (this.durationSeconds < 60) {
  472 + throw new Exception("durationSeconds must be 60 seconds " +
  473 + "or longer.");
  474 + }
  475 + if (this.retryPeriod < 1) {
  476 + throw new Exception("retryPeriod must be 1 or greater");
  477 + }
  478 + if (this.network == null) {
  479 + throw new Exception("network may not be null");
  480 + }
  481 +
  482 + logger.debug("validated");
  483 + }
  484 +}
65 service/service/java/source/src/org/globus/workspace/BackfillTimer.java
... ... @@ -0,0 +1,65 @@
  1 +/*
  2 + * Copyright 1999-2010 University of Chicago
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License"); you may not
  5 + * use this file except in compliance with the License. You may obtain a copy
  6 + * of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12 + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13 + * License for the specific language governing permissions and limitations
  14 + * under the License.
  15 + */
  16 +
  17 +package org.globus.workspace;
  18 +
  19 +import org.nimbustools.api.services.rm.ResourceRequestDeniedException;
  20 +
  21 +import commonj.timers.TimerListener;
  22 +import commonj.timers.Timer;
  23 +import org.apache.commons.logging.Log;
  24 +import org.apache.commons.logging.LogFactory;
  25 +
  26 +public class BackfillTimer implements TimerListener {
  27 +
  28 + private static final Log logger =
  29 + LogFactory.getLog(BackfillTimer.class.getName());
  30 +
  31 + private final Backfill backfill;
  32 +
  33 + public BackfillTimer(Backfill backfill) {
  34 + this.backfill = backfill;
  35 + }
  36 +
  37 + public void timerExpired(Timer timer) {
  38 +
  39 + final int maxInstances = this.backfill.getMaxInstances();
  40 + int curNumInstances = this.backfill.getCurNumInstances();
  41 +
  42 + if (maxInstances == 0) {
  43 + curNumInstances = -1;
  44 + }
  45 + boolean wantMoreBackfill = true;
  46 + while((wantMoreBackfill == true) &&
  47 + (curNumInstances < maxInstances)) {
  48 + try {
  49 + this.backfill.createBackfillNode();
  50 + this.backfill.addCurInstances(1);
  51 + } catch (ResourceRequestDeniedException rDE) {
  52 + logger.info("Backfill launch failed. " +
  53 + "Looks like we're full.");
  54 + wantMoreBackfill = false;
  55 + } catch (Exception e) {
  56 + logger.error("BackfillTimer caught " +
  57 + "an exception: " + e.getMessage());
  58 + wantMoreBackfill = false;
  59 + }
  60 + if (maxInstances > 0) {
  61 + curNumInstances = this.backfill.getCurNumInstances();
  62 + }
  63 + }
  64 + }
  65 +}
58 service/service/java/source/src/org/globus/workspace/creation/defaults/DefaultCreation.java
@@ -45,6 +45,7 @@
45 45 import org.globus.workspace.service.binding.vm.VirtualMachine;
46 46 import org.globus.workspace.service.binding.vm.VirtualMachineDeployment;
47 47 import org.globus.workspace.service.binding.vm.CustomizationNeed;
  48 +import org.globus.workspace.Backfill;
48 49
49 50 import org.nimbustools.api._repr._CreateResult;
50 51 import org.nimbustools.api._repr._Advertised;
@@ -113,6 +114,8 @@
113 114 protected final Lager lager;
114 115 protected final DateFormat localFormat = DateFormat.getDateTimeInstance();
115 116
  117 + private final Backfill backfill;
  118 +
116 119 protected AccountingEventAdapter accounting;
117 120
118 121
@@ -134,7 +137,8 @@ public DefaultCreation(LockManager lockManagerImpl,
134 137 PersistenceAdapter persistenceAdapter,
135 138 DataConvert dataConvertImpl,
136 139 TimerManager timerManagerImpl,
137   - Lager lagerImpl) {
  140 + Lager lagerImpl,
  141 + Backfill backfill) {
138 142
139 143 if (lockManagerImpl == null) {
140 144 throw new IllegalArgumentException("lockManager may not be null");
@@ -210,6 +214,11 @@ public DefaultCreation(LockManager lockManagerImpl,
210 214 throw new IllegalArgumentException("lagerImpl may not be null");
211 215 }
212 216 this.lager = lagerImpl;
  217 +
  218 + if (backfill == null) {
  219 + throw new IllegalArgumentException("backfill may not be null");
  220 + }
  221 + this.backfill = backfill;
213 222 }
214 223
215 224
@@ -316,7 +325,36 @@ protected CreateResult create1(CreateRequest req, Caller caller)
316 325 ResourceRequestDeniedException,
317 326 SchedulingException {
318 327
319   - final VirtualMachine[] bound = this.binding.processRequest(req);
  328 + // We trigger backfill termination here just in case the network
  329 + // binding causes a ResourceRequestDeniedException (because there
  330 + // are no more available IP addresses, for instance)
  331 + VirtualMachine[] bound = null;
  332 + try {
  333 + bound = this.binding.processRequest(req);
  334 + } catch (ResourceRequestDeniedException e) {
  335 + logger.debug("Failed to reserve the resource: " + e.getMessage());
  336 + if (req.getRequestedSchedule().getBackfillReq() == false) {
  337 + logger.debug("The request isn't a backfill request");
  338 + logger.debug("Attempting to terminate backfill nodes");
  339 + boolean continueTerminateBackfill = true;
  340 + while (continueTerminateBackfill == true) {
  341 + try {
  342 + bound = this.binding.processRequest(req);
  343 + continueTerminateBackfill = false;
  344 + } catch (ResourceRequestDeniedException rDE) {
  345 + int numNodes = req.getRequestedRA().getNodeNumber();
  346 + if (this.backfill.terminateBackfillNode(numNodes) == 0) {
  347 + throw rDE;
  348 + } else {
  349 + continueTerminateBackfill = true;
  350 + }
  351 + }
  352 + }
  353 + } else {
  354 + throw e;
  355 + }
  356 + }
  357 +
320 358 if (bound == null || bound.length == 0) {
321 359 throw new CreationException("no binding result but no binding " +
322 360 "error: illegal binding implementation");
@@ -330,10 +368,13 @@ protected CreateResult create1(CreateRequest req, Caller caller)
330 368 final Context context = req.getContext();
331 369 final String groupID = this.getGroupID(creatorID, bound.length);
332 370 final String coschedID = this.getCoschedID(req, creatorID);
  371 + final boolean backfillReq =
  372 + req.getRequestedSchedule().getBackfillReq();
333 373
334 374 // From this point forward an error requires backOutAllocations
335 375 try {
336   - return this.create2(bound, caller, context, groupID, coschedID);
  376 + return this.create2(bound, caller, context, groupID, coschedID,
  377 + backfillReq);
337 378 } catch (CoSchedulingException e) {
338 379 this.backoutBound(bound);
339 380 throw e;
@@ -414,7 +455,8 @@ protected CreateResult create2(VirtualMachine[] bindings,
414 455 Caller caller,
415 456 Context context,
416 457 String groupID,
417   - String coschedID)
  458 + String coschedID,
  459 + boolean backfillReq)
418 460
419 461 throws AuthorizationException,
420 462 CoSchedulingException,
@@ -444,7 +486,8 @@ protected CreateResult create2(VirtualMachine[] bindings,
444 486 bindings.length,
445 487 groupID,
446 488 coschedID,
447   - caller.getIdentity());
  489 + caller.getIdentity(),
  490 + backfillReq);
448 491
449 492 if (res == null) {
450 493 throw new SchedulingException("reservation is missing, illegal " +
@@ -506,7 +549,8 @@ protected Reservation scheduleImpl(VirtualMachine vm,
506 549 int numNodes,
507 550 String groupid,
508 551 String coschedid,
509   - String callerID)
  552 + String callerID,
  553 + boolean backfillReq)
510 554
511 555 throws SchedulingException,
512 556 ResourceRequestDeniedException {
@@ -528,7 +572,7 @@ protected Reservation scheduleImpl(VirtualMachine vm,
528 572 }
529 573
530 574 return this.scheduler.schedule(memory, duration, assocs, numNodes,
531   - groupid, coschedid, callerID);
  575 + groupid, coschedid, callerID, backfillReq);
532 576 }
533 577
534 578
12 service/service/java/source/src/org/globus/workspace/manager/DelegatingManager.java
@@ -21,6 +21,7 @@
21 21
22 22 import org.globus.workspace.PathConfigs;
23 23 import org.globus.workspace.Lager;
  24 +import org.globus.workspace.Backfill;
24 25 import org.globus.workspace.accounting.AccountingReaderAdapter;
25 26 import org.globus.workspace.accounting.ElapsedAndReservedMinutes;
26 27 import org.globus.workspace.creation.Creation;
@@ -88,6 +89,7 @@
88 89 protected final ReprFactory repr;
89 90 protected final DataConvert dataConvert;
90 91 protected final Lager lager;
  92 + protected final Backfill backfill;
91 93
92 94 protected AccountingReaderAdapter accounting;
93 95
@@ -102,7 +104,8 @@ public DelegatingManager(Creation creationImpl,
102 104 WorkspaceCoschedHome coschedHome,
103 105 ReprFactory reprFactory,
104 106 DataConvert dataConvertImpl,
105   - Lager lagerImpl) {
  107 + Lager lagerImpl,
  108 + Backfill backfill) {
106 109
107 110 if (creationImpl == null) {
108 111 throw new IllegalArgumentException("creationImpl may not be null");
@@ -143,6 +146,11 @@ public DelegatingManager(Creation creationImpl,
143 146 throw new IllegalArgumentException("lagerImpl may not be null");
144 147 }
145 148 this.lager = lagerImpl;
  149 +
  150 + if (backfill == null) {
  151 + throw new IllegalArgumentException("backfill may not be null");
  152 + }
  153 + this.backfill = backfill;
146 154 }
147 155
148 156
@@ -188,6 +196,8 @@ public String report() {
188 196 */
189 197 public void recover_initialize() throws Exception {
190 198 this.home.recover_initialize();
  199 + this.backfill.setDelegatingManager(this);
  200 + this.backfill.initiateBackfill();
191 201 }
192 202
193 203 public void shutdownImmediately() {
4 service/service/java/source/src/org/globus/workspace/scheduler/Scheduler.java
@@ -43,6 +43,7 @@
43 43 * @param groupid group ID, can be null
44 44 * @param coschedid co-scheduling ID, can be null
45 45 * @param creatorDN creator's identifying DN, can be null
  46 + * @param backfillReq whether this is a backfill request
46 47 * @return reservation never null
47 48 * @throws ResourceRequestDeniedException will not grant
48 49 * @throws SchedulingException internal problem
@@ -53,7 +54,8 @@ public Reservation schedule(int memory,
53 54 int numNodes,
54 55 String groupid,
55 56 String coschedid,
56   - String creatorDN)
  57 + String creatorDN,
  58 + boolean backfillReq)
57 59
58 60 throws SchedulingException,
59 61 ResourceRequestDeniedException;
85 service/service/java/source/src/org/globus/workspace/scheduler/defaults/BackfillNode.java
... ... @@ -0,0 +1,85 @@
  1 +/*
  2 + * Copyright 1999-2010 University of Chicago
  3 + *
  4 + * Licensed under the Apache License, Version 2.0 (the "License"); you may not
  5 + * use this file except in compliance with the License. You may obtain a copy
  6 + * of the License at
  7 + *
  8 + * http://www.apache.org/licenses/LICENSE-2.0
  9 + *
  10 + * Unless required by applicable law or agreed to in writing, software
  11 + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12 + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13 + * License for the specific language governing permissions and limitations
  14 + * under the License.
  15 + */
  16 +
  17 +package org.globus.workspace.scheduler.defaults;
  18 +
  19 +import java.io.IOException;
  20 +import java.io.ObjectOutputStream;
  21 +import java.io.ObjectInputStream;
  22 +import java.io.Serializable;
  23 +import java.util.Date;
  24 +
  25 +public class BackfillNode implements Serializable {
  26 + private int vmid;
  27 + private Date startDate;
  28 + private String hostname;
  29 +
  30 + public BackfillNode(int vmid,
  31 + Date startDate,
  32 + String hostname) {
  33 +
  34 + this.vmid = vmid;
  35 +
  36 + if (startDate == null) {
  37 + throw new IllegalArgumentException("startDate may not be null");
  38 + }
  39 + this.startDate = startDate;
  40 +
  41 + if (hostname == null) {
  42 + throw new IllegalArgumentException("hostname may not be null");
  43 + }
  44 + this.hostname = hostname;
  45 + }
  46 +
  47 + public int getVMID() {
  48 + return this.vmid;
  49 + }
  50 +
  51 + public Date getStartDate() {
  52 + return this.startDate;
  53 + }
  54 +
  55 + public String getHostname() {
  56 + return this.hostname;
  57 + }
  58 +
  59 + public void setVMID(int vmid) {
  60 + this.vmid = vmid;
  61 + }
  62 +
  63 + public void setStartDate(Date startDate) {
  64 + this.startDate = startDate;
  65 + }
  66 +
  67 + public void setHostname(String hostname) {
  68 + this.hostname = hostname;
  69 + }
  70 +
  71 + public String toString() {
  72 + return "BackfillNode{vmid=" + Integer.toString(this.vmid) +
  73 + ", startDate=" + this.startDate.toString() +
  74 + ", hostname=" + this.hostname + "}";
  75 + }
  76 +
  77 + private void writeObject(ObjectOutputStream out) throws IOException {
  78 + out.defaultWriteObject();
  79 + }
  80 +
  81 + private void readObject(ObjectInputStream in)
  82 + throws IOException, ClassNotFoundException {
  83 + in.defaultReadObject();
  84 + }
  85 +}
54 service/service/java/source/src/org/globus/workspace/scheduler/defaults/DefaultSchedulerAdapter.java
@@ -30,6 +30,7 @@
30 30 import org.globus.workspace.service.WorkspaceHome;
31 31 import org.globus.workspace.service.binding.GlobalPolicies;
32 32 import org.globus.workspace.LockAcquisitionFailure;
  33 +import org.globus.workspace.Backfill;
33 34 import org.nimbustools.api.services.rm.ResourceRequestDeniedException;
34 35 import org.nimbustools.api.services.rm.SchedulingException;
35 36 import org.nimbustools.api.services.rm.DoesNotExistException;
@@ -80,6 +81,7 @@
80 81 // see CreationPending class comment
81 82 protected final CreationPending creationPending = new CreationPending();
82 83
  84 + private final Backfill backfill;
83 85
84 86 // -------------------------------------------------------------------------
85 87 // CONSTRUCTOR
@@ -91,7 +93,8 @@ public DefaultSchedulerAdapter(LockManager lockManager,
91 93 TimerManager timerManager,
92 94 GlobalPolicies globalPolicies,
93 95 DataConvert dataConvert,
94   - Lager lagerImpl) {
  96 + Lager lagerImpl,
  97 + Backfill backfill) {
95 98
96 99 if (lockManager == null) {
97 100 throw new IllegalArgumentException("lockManager may not be null");
@@ -127,6 +130,11 @@ public DefaultSchedulerAdapter(LockManager lockManager,
127 130 throw new IllegalArgumentException("lagerImpl may not be null");
128 131 }
129 132 this.lager = lagerImpl;
  133 +
  134 + if (backfill == null) {
  135 + throw new IllegalArgumentException("backfill may not be null");
  136 + }
  137 + this.backfill = backfill;
130 138 }
131 139
132 140
@@ -219,7 +227,8 @@ public Reservation schedule(int memory,
219 227 int numNodes,
220 228 String groupid,
221 229 String coschedid,
222   - String creatorDN)
  230 + String creatorDN,
  231 + boolean backfillReq)
223 232
224 233 throws SchedulingException,
225 234 ResourceRequestDeniedException {
@@ -252,7 +261,7 @@ public Reservation schedule(int memory,
252 261 this.creationPending.pending(ids);
253 262
254 263 final NodeRequest req =
255   - new NodeRequest(ids, memory, duration, assocs, groupid, creatorDN);
  264 + new NodeRequest(ids, memory, duration, assocs, groupid, creatorDN, backfillReq);
256 265
257 266 try {
258 267
@@ -282,7 +291,35 @@ private Reservation scheduleImpl(NodeRequest req)
282 291 final String invalidResponse = "Implementation problem: slot " +
283 292 "manager returned invalid response";
284 293
285   - final Reservation res = this.slotManager.reserveSpace(req);
  294 + // If a request is denied (and it's not a backfill request) then we
  295 + // should attempt to kill backfill nodes until there are no more
  296 + // backfill nodes to kill or until the request can be satisfied.
  297 + Reservation res = null;
  298 + try {
  299 + res = this.slotManager.reserveSpace(req);
  300 + } catch (ResourceRequestDeniedException e) {
  301 + logger.debug("Failed to reserve the resource: " + e.getMessage());
  302 + if (req.getBackfillReq() == false) {
  303 + logger.debug("The request isn't a backfill request");
  304 + logger.debug("Attempting to terminate backfill nodes");
  305 + boolean continueTerminateBackfill = true;
  306 + while (continueTerminateBackfill == true) {
  307 + try {
  308 + res = this.slotManager.reserveSpace(req);
  309 + continueTerminateBackfill = false;
  310 + } catch (ResourceRequestDeniedException rDE) {
  311 + int numNodes = req.getNumNodes();
  312 + if (this.backfill.terminateBackfillNode(numNodes) == 0) {
  313 + throw rDE;
  314 + } else {
  315 + continueTerminateBackfill = true;
  316 + }
  317 + }
  318 + }
  319 + } else {
  320 + throw e;
  321 + }
  322 + }
286 323
287 324 if (res == null) {
288 325 throw new ResourceRequestDeniedException(
@@ -861,6 +898,15 @@ private void remove(int vmid) throws ManageException {
861 898 this.db.backOutTasks(vmid);
862 899 this.slotManager.releaseSpace(vmid);
863 900 this.db.deleteNodeRequest(vmid);
  901 +
  902 + // Check to see if this was a backfill request being killed off.
  903 + // If not, relaunch the backfill timer to deploy backfill nodes
  904 + // on *potentially* empty slots.
  905 + if ((this.slotManager.isOldBackfillID(vmid) == false) &&
  906 + (this.slotManager.isCurrentBackfillID(vmid) == false)) {
  907 + logger.debug("Relaunching backfill timer");
  908 + this.backfill.launchBackfillTimer();
  909 + }
864 910 }
865 911
866 912 protected void markShutdown(int id) throws WorkspaceDatabaseException {
158 service/service/java/source/src/org/globus/workspace/scheduler/defaults/DefaultSlotManagement.java
@@ -19,6 +19,7 @@
19 19 import org.apache.commons.logging.Log;
20 20 import org.apache.commons.logging.LogFactory;
21 21 import org.globus.workspace.Lager;
  22 +import org.globus.workspace.PathConfigs;
22 23 import org.globus.workspace.ProgrammingError;
23 24 import org.globus.workspace.persistence.WorkspaceDatabaseException;
24 25 import org.globus.workspace.scheduler.*;
@@ -31,6 +32,13 @@
31 32 import org.nimbustools.api.services.rm.DoesNotExistException;
32 33 import org.nimbustools.api.services.rm.ManageException;
33 34
  35 +import java.io.IOException;
  36 +import java.io.File;
  37 +import java.io.ObjectOutputStream;
  38 +import java.io.FileOutputStream;
  39 +import java.io.ObjectInputStream;
  40 +import java.io.FileInputStream;
  41 +import java.io.FileNotFoundException;
34 42 import java.util.*;
35 43
36 44 /**
@@ -56,13 +64,20 @@
56 64
57 65 private boolean greedy;
58 66
  67 + private Hashtable backfillVMs = new Hashtable();
  68 + private Set oldBackfillIDs = new HashSet();
  69 + private File backfillHashFile;
  70 +
  71 + protected final PathConfigs pathConfigs;
  72 +
59 73 </