Skip to content
Browse files

Merge branch 'master' into 0.9.0

  • Loading branch information...
2 parents f771774 + a0e52c7 commit 70242cd178a5619a97a72a2613b3df7bf5836a7b @nathanmarz committed Dec 19, 2012
View
14 src/clj/backtype/storm/scheduler/IsolationScheduler.clj
@@ -96,6 +96,12 @@
(LinkedList. <>)
))
+(defn- host->used-slots [^Cluster cluster]
+ (->> cluster
+ .getUsedSlots
+ (group-by #(.getHost cluster (.getNodeId ^WorkerSlot %)))
+ ))
+
(defn- distribution->sorted-amts [distribution]
(->> distribution
(mapcat (fn [[val amt]] (repeat amt val)))
@@ -156,7 +162,8 @@
^Set worker-specs (get topology-worker-specs top-id)
num-workers (count host-assignments)
]
- (if (and (every? #(= (second %) top-id) assignments)
+ (if (and (contains? iso-ids-set top-id)
+ (every? #(= (second %) top-id) assignments)
(contains? distribution num-workers)
(every? #(contains? worker-specs (nth % 2)) assignments))
(do (decrement-distribution! distribution num-workers)
@@ -168,15 +175,16 @@
))
)))
- (let [^LinkedList sorted-assignable-hosts (host-assignable-slots cluster)]
+ (let [host->used-slots (host->used-slots cluster)
+ ^LinkedList sorted-assignable-hosts (host-assignable-slots cluster)]
;; TODO: can improve things further by ordering topologies in terms of who needs the least workers
(doseq [[top-id worker-specs] topology-worker-specs
:let [amts (distribution->sorted-amts (get topology-machine-distribution top-id))]]
(doseq [amt amts
:let [[host host-slots] (.peek sorted-assignable-hosts)]]
(when (and host-slots (>= (count host-slots) amt))
(.poll sorted-assignable-hosts)
- (.freeSlots cluster host-slots)
+ (.freeSlots cluster (get host->used-slots host))
(doseq [slot (take amt host-slots)
:let [executors-set (remove-elem-from-set! worker-specs)]]
(.assign cluster slot top-id executors-set))
View
6 src/jvm/backtype/storm/scheduler/Cluster.java
@@ -320,8 +320,10 @@ public void freeSlot(WorkerSlot slot) {
* @param slots
*/
public void freeSlots(Collection<WorkerSlot> slots) {
- for (WorkerSlot slot : slots) {
- this.freeSlot(slot);
+ if(slots!=null) {
+ for (WorkerSlot slot : slots) {
+ this.freeSlot(slot);
+ }
}
}

0 comments on commit 70242cd

Please sign in to comment.
Something went wrong with that request. Please try again.