Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100755 169 lines (154 sloc) 7.249 kb
93f2e77d »
2011-07-27 Changed line ending for cloud-sync.py and added shebang
1 #!/usr/bin/env python
2 import ConfigParser
3 import optparse
4 from cloud_providers.swift import *
5 from file_lists.local import *
6 from file_lists.swift import *
efc41030 »
2011-07-27 UrlEncoding added
7 from urllib import quote
346212d0 »
2011-07-27 Inital threading support
8 import os
9 import sys
10 import threading
11 from Queue import Queue
12
13 class Worker(threading.Thread):
14 def __init__(self):
15 self._log = Logging().log
16 threading.Thread.__init__(self)
17 self.kill_received = False
18 self.runno = 0
19
20 def run(self):
21 while not self.kill_received:
22 self.work()
23
24 def work(self):
25 task = q.get()
26 if task['direction'] == 'get':
27 self._log.debug('Run %d' % self.runno)
28 task['clouds']['swift'].get(task['source']['container'],quote(task['file'],'/'),
29 task['dest']['container']+task['file'])
30 self.runno += 1
31 elif task['direction'] == 'put':
32 self._log.debug('Run %d' % self.runno)
33 task['clouds']['swift'].put(task['dest']['container'],task['source']['container']+task['file'],
34 quote(task['file'],'/'))
35 self.runno += 1
36 elif task['direction'] == 'kill':
37 self.kill_received = True
38
39
40
93f2e77d »
2011-07-27 Changed line ending for cloud-sync.py and added shebang
41
42 def setup_logging(console_level="WARNING",file_level="WARNING",file_name="cloud-sync.log"):
43 _logger = Logging()
44 _logger.setup(console_level=console_level,file_level=file_level,file_name=file_name)
45 return Logging().log
46
47 def config_get(cp,section,name,default):
48 try:
49 c = cp.get(section,name)
50 return c
51 except ConfigParser.NoSectionError:
52 return default
53 except ConfigParser.NoOptionError:
54 return default
55
56 def setup_config():
805eb218 »
2011-07-27 Added in some help to optparser and also bombs out when 2 args are no…
57 usage = "usage: %prog [options] source destination"
58 op = optparse.OptionParser(usage=usage)
93f2e77d »
2011-07-27 Changed line ending for cloud-sync.py and added shebang
59 cp = ConfigParser.ConfigParser()
60 cp.read(['/etc/cloud-sync/cloudsync.ini', os.path.expanduser('~/.cloudsync.ini') ])
61 general = op.add_option_group('General')
62 general.add_option('-d','--console-level', dest='console_level',
63 help='Log Level for Console (DEBUG,INFO,WARNING,ERROR,CRITICAL)',
64 default=config_get(cp,'general','console_level','CRITICAL'))
65 general.add_option('-l','--file-level', dest='file_level',
66 help='Log Level for File (DEBUG,INFO,WARNING,ERROR,CRITICAL)',
67 default=config_get(cp,'general','file_level','WARNING'))
68 general.add_option('-f','--logfile', dest='log_file', help='Log File Name',
69 default=config_get(cp,'general','log_file','/var/log/cloud-sync.log'))
70 general.add_option('-m','--md5', dest="md5", help="Enable MD5 Comparison",
71 default=config_get(cp,'general','md5',True))
346212d0 »
2011-07-27 Inital threading support
72 general.add_option('-T','--threads', dest="numthreads", help="Number of Threads",
73 default=config_get(cp,'general','numthreads',1))
93f2e77d »
2011-07-27 Changed line ending for cloud-sync.py and added shebang
74 api = op.add_option_group('API')
75 api.add_option('-u','--username', dest="username", help="API Username (ex: 'welby.mcroberts')",
76 default=config_get(cp,'api','username','I_HAVE_NOT_SET_MY_USER_NAME'))
77 api.add_option('-k','--key', dest="key", help="API Key (ex 'abcdefghijklmnopqrstuvwxyz12345",
78 default=config_get(cp,'api','key','I_HAVE_NOT_SET_MY_KEY'))
79 api.add_option('-a','--authurl', dest="authurl",
80 help='Auth URL (ex: https://lon.auth.api.rackspacecloud.com/v1.0 )',
81 default=config_get(cp,'api','authurl','https://lon.auth.api.rackspacecloud.com/v1.0'))
82 api.add_option('-s','--connections',dest="connections",
83 help='Number of Connections to API',
84 default=config_get(cp,'api','connections',1))
85 api.add_option('-t','--timeout',dest="timeout",help='API Timeout',
86 default=config_get(cp,'api','timeout',10))
87 api.add_option('-n','--servicenet',dest="servicenet",help='Use Servicenet',
88 default=config_get(cp,'api','servicenet',False))
89 api.add_option('-z','--useragent',dest="useragent",help='Override Useragent',
90 default=config_get(cp,'api','useragent','com.whmcr.cloudsync'))
91 (op_results,op_args) = op.parse_args()
805eb218 »
2011-07-27 Added in some help to optparser and also bombs out when 2 args are no…
92 if len(op_args) != 2:
93 op.error("Incorrect number of arguments")
93f2e77d »
2011-07-27 Changed line ending for cloud-sync.py and added shebang
94 return (op_results,op_args)
95
96 def setup_clouds(op_results,op_args):
97 clouds = {}
98 for arg in op_args:
99 if 'cf://' in arg or 'swift://' in arg:
100 clouds['swift'] = Swift(username=op_results.username,api_key=op_results.key,
101 timeout=int(op_results.timeout),
102 servicenet=op_results.servicenet,
103 useragent=op_results.useragent,
104 auth_url=op_results.authurl)
105 for cloud in clouds:
106 clouds[cloud].connect(pool_count=int(op_results.connections))
107 return clouds
108 def setup_source(clouds,op_results,op_args):
109 if 'swift://' in op_args[0]:
110 list = SwiftList(clouds['swift'],op_args[0][8:])
111 container = op_args[0][8:]
112 return {'list': list, 'container': container, 'type': 'swift'}
113 else:
114 list = DirectoryList(op_args[0])
115 return {'list': list, 'container': op_args[0], 'type': 'local'}
116 def setup_dest(clouds,op_results,op_args):
117 if 'swift://' in op_args[1]:
118 list = SwiftList(clouds['swift'],op_args[1][8:])
119 container = op_args[1][8:]
120 return {'list': list, 'container': container, 'type': 'swift'}
121 else:
122 list = DirectoryList(op_args[1])
123 return {'list': list, 'container': op_args[1], 'type': 'local'}
124 def main():
125 (op_results,op_args) = setup_config()
126 _log = setup_logging(op_results.console_level, op_results.file_level, op_results.log_file)
127 for opt, value in op_results.__dict__.items():
128 _log.debug('Setting %s is %s' % (opt,str(value)))
129 clouds = setup_clouds(op_results,op_args)
130
131 source = setup_source(clouds,op_results,op_args)
132 dest = setup_dest(clouds,op_results,op_args)
133 source['list'].compare(dest['list'].file_list)
134
346212d0 »
2011-07-27 Inital threading support
135 threads =[]
136 for i in op_results.numthreads:
137 t = Worker()
138 threads.append(t)
139 t.start()
140 threads = [t.join(1) for t in threads if t is not None and t.isAlive()]
8850c539 »
2011-07-27 Allowing downloading of files from Swift
141 if dest['type'] == 'swift':
142 for file in source['list'].sync_list:
346212d0 »
2011-07-27 Inital threading support
143 q.put({'direction': 'put', 'source': source, 'dest': dest, 'file': file, 'clouds': clouds })
8850c539 »
2011-07-27 Allowing downloading of files from Swift
144 else:
145 for file in source['list'].sync_list:
346212d0 »
2011-07-27 Inital threading support
146 q.put({'direction': 'get', 'source': source, 'dest': dest, 'file': file, 'clouds': clouds })
147 q.put({'direction': 'kill'})
148 while len(threads) > 0:
149 try:
150 threads = [t.join(1) for t in threads if t is not None and t.isAlive()]
151 except KeyboardInterrupt:
152 for t in threads:
153 t.kill_received = True
154
155
93f2e77d »
2011-07-27 Changed line ending for cloud-sync.py and added shebang
156 if __name__ == '__main__':
346212d0 »
2011-07-27 Inital threading support
157 q = Queue()
93f2e77d »
2011-07-27 Changed line ending for cloud-sync.py and added shebang
158 main()
346212d0 »
2011-07-27 Inital threading support
159
93f2e77d »
2011-07-27 Changed line ending for cloud-sync.py and added shebang
160 #q = Queue()
161 #q.put({'container': container,'direction': 'put', 'remote': remote, 'local': local})
162 #q.put({'container': container,'direction': 'get', 'remote': remote, 'local': local})
163 #q.put({'direction': 'kill'})
164 #q.put({'direction': 'kill'})
165 #threads = []
166 #for i in range(5):
167 # threads.append(CSThread())
168 #for i in range(5):
169 # threads[i].start()
170
Something went wrong with that request. Please try again.