public
Description: Python backup classes to backup files to key/value store, such as Amazon S3
Homepage: http://www.1729.com/software/keevalbak/index.html
Clone URL: git://github.com/pdorrell/keevalbak.git
keevalbak / keevalbak / ThreadedTaskRunner.py
100644 90 lines (79 sloc) 3.62 kb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
# Copyright (c) 2008 Philip Dorrell, http://www.1729.com/
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
 
import Queue
import threading
 
class TaskRunner(object):
    """Simple task runner: runs both parts of tasks synchronously"""
    def __init__(self, checkpointFreq = None):
        self.checkpointFreq = checkpointFreq
        
    def runTasksInit(self):
        pass
    
    def doUnsynchronizedTasks(self, tasks):
        for task in tasks:
            task.doUnsynchronized()
        
    def runTasks(self, tasks, checkpointTask = None):
        self.runTasksInit()
        startIndex = 0
        numTasks = len(tasks)
        while startIndex < numTasks:
            if self.checkpointFreq == None or checkpointTask == None:
                endIndex = numTasks
            else:
                endIndex = min(startIndex+self.checkpointFreq, numTasks)
            self.doUnsynchronizedTasks (tasks[startIndex:endIndex])
            for i in range(startIndex, endIndex):
                tasks[i].doSynchronized()
            if endIndex < numTasks:
                print "CHECKPOINT:"
                if checkpointTask != None:
                    checkpointTask.checkpoint()
            startIndex = endIndex
            
class TaskProcessor(threading.Thread):
    def __init__(self, queue, index):
        threading.Thread.__init__(self)
        self.queue = queue
        self.index = index
        self.threadLocals = None
 
    def run(self):
        while True:
            task = self.queue.get()
            if self.threadLocals == None:
                self.threadLocals = task.getThreadLocals()
            #print "Thread %d performing task ..." % self.index
            for key, value in self.threadLocals.iteritems():
                task.__dict__[key] = value
            task.doUnsynchronized()
            self.queue.task_done()
            #print "Thread %d finished performing task ..." % self.index
            
class ThreadedTaskRunner(TaskRunner):
    def __init__(self, checkpointFreq = 10, numThreads = 10):
        super(ThreadedTaskRunner, self).__init__(checkpointFreq)
        self.queue = Queue.Queue()
        self.processors = [TaskProcessor(self.queue, i) for i in range(numThreads)]
        for processor in self.processors:
            processor.setDaemon(True)
            processor.start()
            
    def runTasksInit(self):
        for processor in self.processors:
            processor.threadLocals = None
                
    def doUnsynchronizedTasks(self, tasks):
        for task in tasks:
            self.queue.put (task)
        self.queue.join()