Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100755 171 lines (154 sloc) 7.249 kb
93f2e77 Welby McRoberts Changed line ending for cloud-sync.py and added shebang
authored
1 #!/usr/bin/env python
2 import ConfigParser
3 import optparse
4 from cloud_providers.swift import *
5 from file_lists.local import *
6 from file_lists.swift import *
efc4103 Welby McRoberts UrlEncoding added
authored
7 from urllib import quote
346212d Welby McRoberts Inital threading support
authored
8 import os
9 import sys
10 import threading
11 from Queue import Queue
12
13 class Worker(threading.Thread):
14 def __init__(self):
15 self._log = Logging().log
16 threading.Thread.__init__(self)
17 self.kill_received = False
18 self.runno = 0
19
20 def run(self):
21 while not self.kill_received:
22 self.work()
23
24 def work(self):
25 task = q.get()
26 if task['direction'] == 'get':
27 self._log.debug('Run %d' % self.runno)
28 task['clouds']['swift'].get(task['source']['container'],quote(task['file'],'/'),
29 task['dest']['container']+task['file'])
30 self.runno += 1
31 elif task['direction'] == 'put':
32 self._log.debug('Run %d' % self.runno)
33 task['clouds']['swift'].put(task['dest']['container'],task['source']['container']+task['file'],
34 quote(task['file'],'/'))
35 self.runno += 1
36 elif task['direction'] == 'kill':
37 self.kill_received = True
38
39
40
93f2e77 Welby McRoberts Changed line ending for cloud-sync.py and added shebang
authored
41
42 def setup_logging(console_level="WARNING",file_level="WARNING",file_name="cloud-sync.log"):
43 _logger = Logging()
44 _logger.setup(console_level=console_level,file_level=file_level,file_name=file_name)
45 return Logging().log
46
47 def config_get(cp,section,name,default):
48 try:
49 c = cp.get(section,name)
50 return c
51 except ConfigParser.NoSectionError:
52 return default
53 except ConfigParser.NoOptionError:
54 return default
55
56 def setup_config():
805eb21 Welby McRoberts Added in some help to optparser and also bombs out when 2 args are not g...
authored
57 usage = "usage: %prog [options] source destination"
58 op = optparse.OptionParser(usage=usage)
93f2e77 Welby McRoberts Changed line ending for cloud-sync.py and added shebang
authored
59 cp = ConfigParser.ConfigParser()
60 cp.read(['/etc/cloud-sync/cloudsync.ini', os.path.expanduser('~/.cloudsync.ini') ])
61 general = op.add_option_group('General')
62 general.add_option('-d','--console-level', dest='console_level',
63 help='Log Level for Console (DEBUG,INFO,WARNING,ERROR,CRITICAL)',
64 default=config_get(cp,'general','console_level','CRITICAL'))
65 general.add_option('-l','--file-level', dest='file_level',
66 help='Log Level for File (DEBUG,INFO,WARNING,ERROR,CRITICAL)',
67 default=config_get(cp,'general','file_level','WARNING'))
68 general.add_option('-f','--logfile', dest='log_file', help='Log File Name',
69 default=config_get(cp,'general','log_file','/var/log/cloud-sync.log'))
70 general.add_option('-m','--md5', dest="md5", help="Enable MD5 Comparison",
71 default=config_get(cp,'general','md5',True))
346212d Welby McRoberts Inital threading support
authored
72 general.add_option('-T','--threads', dest="numthreads", help="Number of Threads",
73 default=config_get(cp,'general','numthreads',1))
93f2e77 Welby McRoberts Changed line ending for cloud-sync.py and added shebang
authored
74 api = op.add_option_group('API')
75 api.add_option('-u','--username', dest="username", help="API Username (ex: 'welby.mcroberts')",
76 default=config_get(cp,'api','username','I_HAVE_NOT_SET_MY_USER_NAME'))
77 api.add_option('-k','--key', dest="key", help="API Key (ex 'abcdefghijklmnopqrstuvwxyz12345",
78 default=config_get(cp,'api','key','I_HAVE_NOT_SET_MY_KEY'))
79 api.add_option('-a','--authurl', dest="authurl",
80 help='Auth URL (ex: https://lon.auth.api.rackspacecloud.com/v1.0 )',
81 default=config_get(cp,'api','authurl','https://lon.auth.api.rackspacecloud.com/v1.0'))
82 api.add_option('-s','--connections',dest="connections",
83 help='Number of Connections to API',
84 default=config_get(cp,'api','connections',1))
85 api.add_option('-t','--timeout',dest="timeout",help='API Timeout',
86 default=config_get(cp,'api','timeout',10))
87 api.add_option('-n','--servicenet',dest="servicenet",help='Use Servicenet',
88 default=config_get(cp,'api','servicenet',False))
89 api.add_option('-z','--useragent',dest="useragent",help='Override Useragent',
90 default=config_get(cp,'api','useragent','com.whmcr.cloudsync'))
91 (op_results,op_args) = op.parse_args()
805eb21 Welby McRoberts Added in some help to optparser and also bombs out when 2 args are not g...
authored
92 if len(op_args) != 2:
93 op.error("Incorrect number of arguments")
93f2e77 Welby McRoberts Changed line ending for cloud-sync.py and added shebang
authored
94 return (op_results,op_args)
95
96 def setup_clouds(op_results,op_args):
97 clouds = {}
98 for arg in op_args:
99 if 'cf://' in arg or 'swift://' in arg:
100 clouds['swift'] = Swift(username=op_results.username,api_key=op_results.key,
101 timeout=int(op_results.timeout),
102 servicenet=op_results.servicenet,
103 useragent=op_results.useragent,
104 auth_url=op_results.authurl)
105 for cloud in clouds:
106 clouds[cloud].connect(pool_count=int(op_results.connections))
107 return clouds
108 def setup_source(clouds,op_results,op_args):
109 if 'swift://' in op_args[0]:
110 list = SwiftList(clouds['swift'],op_args[0][8:])
111 container = op_args[0][8:]
112 return {'list': list, 'container': container, 'type': 'swift'}
113 else:
114 list = DirectoryList(op_args[0])
115 return {'list': list, 'container': op_args[0], 'type': 'local'}
116 def setup_dest(clouds,op_results,op_args):
117 if 'swift://' in op_args[1]:
118 list = SwiftList(clouds['swift'],op_args[1][8:])
119 container = op_args[1][8:]
120 return {'list': list, 'container': container, 'type': 'swift'}
121 else:
122 list = DirectoryList(op_args[1])
123 return {'list': list, 'container': op_args[1], 'type': 'local'}
124 def main():
125 (op_results,op_args) = setup_config()
126 _log = setup_logging(op_results.console_level, op_results.file_level, op_results.log_file)
127 for opt, value in op_results.__dict__.items():
128 _log.debug('Setting %s is %s' % (opt,str(value)))
129 clouds = setup_clouds(op_results,op_args)
130
131 source = setup_source(clouds,op_results,op_args)
132 dest = setup_dest(clouds,op_results,op_args)
133 source['list'].compare(dest['list'].file_list)
134
346212d Welby McRoberts Inital threading support
authored
135 threads =[]
136 for i in op_results.numthreads:
137 t = Worker()
138 threads.append(t)
139 t.start()
140 threads = [t.join(1) for t in threads if t is not None and t.isAlive()]
8850c53 Welby McRoberts Allowing downloading of files from Swift
authored
141 if dest['type'] == 'swift':
142 for file in source['list'].sync_list:
346212d Welby McRoberts Inital threading support
authored
143 q.put({'direction': 'put', 'source': source, 'dest': dest, 'file': file, 'clouds': clouds })
8850c53 Welby McRoberts Allowing downloading of files from Swift
authored
144 else:
145 for file in source['list'].sync_list:
346212d Welby McRoberts Inital threading support
authored
146 q.put({'direction': 'get', 'source': source, 'dest': dest, 'file': file, 'clouds': clouds })
147 q.put({'direction': 'kill'})
148 while len(threads) > 0:
149 try:
150 threads = [t.join(1) for t in threads if t is not None and t.isAlive()]
151 except KeyboardInterrupt:
152 for t in threads:
153 t.kill_received = True
154
155
93f2e77 Welby McRoberts Changed line ending for cloud-sync.py and added shebang
authored
156 if __name__ == '__main__':
346212d Welby McRoberts Inital threading support
authored
157 q = Queue()
93f2e77 Welby McRoberts Changed line ending for cloud-sync.py and added shebang
authored
158 main()
346212d Welby McRoberts Inital threading support
authored
159
93f2e77 Welby McRoberts Changed line ending for cloud-sync.py and added shebang
authored
160 #q = Queue()
161 #q.put({'container': container,'direction': 'put', 'remote': remote, 'local': local})
162 #q.put({'container': container,'direction': 'get', 'remote': remote, 'local': local})
163 #q.put({'direction': 'kill'})
164 #q.put({'direction': 'kill'})
165 #threads = []
166 #for i in range(5):
167 # threads.append(CSThread())
168 #for i in range(5):
169 # threads[i].start()
170
Something went wrong with that request. Please try again.