/
utils.py
executable file
·456 lines (388 loc) · 13.6 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
import os
import fnmatch
import re
import subprocess
import sys
import readline
import shutil
import random
import urllib
try:
import simplejson as json
except ImportError:
import json
settings_file = '%s/.modeshape_dev_settings' % os.getenv('HOME')
### Known config keys
svn_base_key = "svn_base"
local_tags_dir_key = "local_tags_dir"
local_mvn_repo_dir_key = "local_mvn_repo_dir"
maven_pom_xml_namespace = "http://maven.apache.org/POM/4.0.0"
default_settings = {'dry_run': False, 'multi_threaded': False, 'verbose': False, 'use_colors': True, 'skip_tests' : False}
boolean_keys = ['dry_run', 'multi_threaded', 'verbose']
settings = dict(default_settings)
class Colors(object):
MAGENTA = '\033[95m'
GREEN = '\033[92m'
YELLOW = '\033[93m'
RED = '\033[91m'
CYAN = '\033[96m'
END = '\033[0m'
UNDERLINE = '\033[4m'
@staticmethod
def magenta():
if use_colors():
return Colors.MAGENTA
else:
return ""
@staticmethod
def green():
if use_colors():
return Colors.GREEN
else:
return ""
@staticmethod
def yellow():
if use_colors():
return Colors.YELLOW
else:
return ""
@staticmethod
def red():
if use_colors():
return Colors.RED
else:
return ""
@staticmethod
def cyan():
if use_colors():
return Colors.CYAN
else:
return ""
@staticmethod
def end_color():
if use_colors():
return Colors.END
else:
return ""
class Levels(Colors):
C_DEBUG = Colors.CYAN
C_INFO = Colors.GREEN
C_WARNING = Colors.YELLOW
C_FATAL = Colors.RED
C_ENDC = Colors.END
DEBUG = "DEBUG"
INFO = "INFO"
WARNING = "WARNING"
FATAL = "FATAL"
@staticmethod
def get_color(level):
if use_colors():
return getattr(Levels, "C_" + level)
else:
return ""
def use_colors():
return ('use_colors' in settings and settings['use_colors']) or ('use_colors' not in settings)
def prettyprint(message, level):
start_color = Levels.get_color(level)
end_color = Levels.end_color()
print "[%s%s%s] %s" % (start_color, level, end_color, message)
def apply_defaults(s):
for e in default_settings.items():
if e[0] not in s:
s[e[0]] = e[1]
return s
def to_bool(x):
if type(x) == bool:
return x
if type(x) == str:
return {'true': True, 'false': False}.get(x.strip().lower())
def get_settings():
"""Retrieves user-specific settings for all ModeShape tools. Returns a dict of key/value pairs, or an empty dict if the settings file doesn't exist."""
f = None
try:
settings = {}
f = open(settings_file)
for l in f:
if not l.strip().startswith("#"):
kvp = l.split("=")
if kvp and len(kvp) > 0 and kvp[0] and len(kvp) > 1:
settings[kvp[0].strip()] = kvp[1].strip()
settings = apply_defaults(settings)
for k in boolean_keys:
settings[k] = to_bool(settings[k])
return settings
except IOError as ioe:
return {}
finally:
if f:
f.close()
def input_with_default(msg, default):
i = raw_input("%s %s[%s]%s: " % (msg, Colors.magenta(), default, Colors.end_color()))
if i.strip() == "":
i = default
return i
def handle_release_virgin():
"""This sounds dirty!"""
prettyprint("""
It appears that this is the first time you are using this script. Please answer a few questions before
we can proceed. Press ENTER to accept the default value, which appears in brackets.
Lets get started!
""", Levels.WARNING)
s = {}
s["verbose"] = input_with_default("Be verbose?", False)
s["multi_threaded"] = input_with_default("Run multi-threaded? (Disable to debug)", True)
s["use_colors"] = input_with_default("Use colors?", True)
s = apply_defaults(s)
f = open(settings_file, "w")
try:
for e in s.keys():
f.write(" %s = %s \n" % (e, s[e]))
finally:
f.close()
def require_settings_file(recursive = False):
"""Tests whether the settings file exists, and if not prompts the user to create one."""
f = None
try:
f = open(settings_file)
except IOError as ioe:
if not recursive:
handle_release_virgin()
require_settings_file(True)
prettyprint("User-specific environment settings file %s created! Please start this script again!" % settings_file, Levels.INFO)
sys.exit(4)
else:
prettyprint("User-specific environment settings file %s is missing! Cannot proceed!" % settings_file, Levels.FATAL)
prettyprint("Please create a file called %s with the following lines:" % settings_file, Levels.FATAL)
prettyprint( '''
verbose = False
use_colors = True
multi_threaded = True
''', Levels.INFO)
sys.exit(3)
finally:
if f:
f.close()
def get_search_path(executable):
"""Retrieves a search path based on where the current executable is located. Returns a string to be prepended to add"""
in_bin_dir = re.compile('^.*/?bin/.*.py')
if in_bin_dir.search(executable):
return "./"
else:
return "../"
def strip_leading_dots(filename):
return filename.strip('/. ')
def to_set(list):
"""Crappy implementation of creating a Set from a List. To cope with older Python versions"""
temp_dict = {}
for entry in list:
temp_dict[entry] = "dummy"
return temp_dict.keys()
class GlobDirectoryWalker:
"""A forward iterator that traverses a directory tree"""
def __init__(self, directory, pattern="*"):
self.stack = [directory]
self.pattern = pattern
self.files = []
self.index = 0
def __getitem__(self, index):
while True:
try:
file = self.files[self.index]
self.index = self.index + 1
except IndexError:
# pop next directory from stack
self.directory = self.stack.pop()
self.files = os.listdir(self.directory)
self.index = 0
else:
# got a filename
fullname = os.path.join(self.directory, file)
if os.path.isdir(fullname) and not os.path.islink(fullname):
self.stack.append(fullname)
if fnmatch.fnmatch(file, self.pattern):
return fullname
def replace_all_in(self, afile, search_expr, replacement ):
f = open(afile)
content = f.read()
f.close()
s_exp = re.compile(search_expr,re.M)
modified = False
if s_exp.search(content):
content = s_exp.sub(replacement,content)
#print content
f = open(afile,'w')
f.write(content)
f.close()
modified = True
return modified
class Git(object):
'''Encapsulates git functionality necessary for releasing ModeShape'''
cmd = 'git'
# Helper functions to clean up branch lists
@staticmethod
def clean(e): return e.strip().replace(' ', '').replace('*', '')
@staticmethod
def non_empty(e): return e != None and e.strip() != ''
@staticmethod
def current(e): return e != None and e.strip().replace(' ', '').startswith('*')
def __init__(self, branch, tag_name):
if not self.is_git_directory():
raise Exception('Attempting to run git outside of a repository. Current directory is %s' % os.path.abspath(os.path.curdir))
self.branch = branch
self.tag = tag_name
self.verbose = False
if settings['verbose']:
self.verbose = True
rand = '%x'.upper() % (random.random() * 100000)
self.working_branch = '__temp_%s' % rand
self.original_branch = self.current_branch()
def run_git(self, opts):
call = [self.cmd]
if type(opts) == list:
for o in opts:
call.append(o)
elif type(opts) == str:
for o in opts.split(' '):
if o != '':
call.append(o)
else:
raise Error("Cannot handle argument of type %s" % type(opts))
if settings['verbose']:
prettyprint( 'Executing %s' % call, Levels.DEBUG )
return subprocess.Popen(call, stdout=subprocess.PIPE).communicate()[0].split('\n')
def is_git_directory(self):
return self.run_git('branch')[0] != ''
def is_upstream_clone(self):
r = self.run_git('remote show -n origin')
cleaned = map(self.clean, r)
def push(e): return e.startswith('PushURL:')
def remove_noise(e): return e.replace('PushURL:', '')
push_urls = map(remove_noise, filter(push, cleaned))
return len(push_urls) == 1 and push_urls[0] == 'git@github.com:ModeShape/modeshape.git'
def clean_branches(self, raw_branch_list):
return map(self.clean, filter(self.non_empty, raw_branch_list))
def remote_branch_exists(self):
'''Tests whether the branch exists on the remote origin'''
branches = self.clean_branches(self.run_git("branch -r"))
def replace_origin(b): return b.replace('origin/', '')
return self.branch in map(replace_origin, branches)
def switch_to_branch(self):
'''Switches the local repository to the specified branch. Creates it if it doesn't already exist.'''
local_branches = self.clean_branches(self.run_git("branch"))
if self.branch not in local_branches:
self.run_git("branch %s origin/%s" % (self.branch, self.branch))
self.run_git("checkout %s" % self.branch)
def create_tag_branch(self):
'''Creates and switches to a temp tagging branch, based off the release branch.'''
self.run_git("checkout -b %s %s" % (self.working_branch, self.branch))
def commit(self, files):
'''Commits the set of files to the current branch with a generated commit message.'''
for f in files:
self.run_git("add %s" % f)
self.run_git(["commit", "-m", "'Release: update versions for %s'" % self.tag])
def tag_for_release(self):
'''Tags the current branch for release using the tag name.'''
self.run_git(["tag", "-a", "-m", "'Release: tag %s'" % self.tag, self.tag])
def push_to_origin(self):
'''Pushes the updated tags to origin'''
self.run_git("push origin --tags")
def current_branch(self):
'''Returns the current branch you are on'''
return map(self.clean, filter(self.current, self.run_git('branch')))[0]
def cleanup(self):
'''Cleans up any temporary branches created'''
self.run_git("checkout %s" % self.original_branch)
self.run_git("branch -D %s" % self.working_branch)
class DryRun(object):
location_root = "%s/%s" % (os.getenv("HOME"), "modeshape_release_dry_run")
flags = "-r"
def __init__(self):
if settings['verbose']:
self.flags = "-rv"
def find_version(self, url):
return os.path.split(url)[1]
def copy(self, src, dst):
prettyprint( " DryRun: Executing %s" % ['rsync', self.flags, src, dst], Levels.DEBUG)
try:
os.makedirs(dst)
except:
pass
subprocess.check_call(['rsync', self.flags, src, dst])
class Uploader(object):
def __init__(self):
if settings['verbose']:
self.scp_cmd = ['scp', '-rv']
self.rsync_cmd = ['rsync', '-rv']
else:
self.scp_cmd = ['scp', '-r']
self.rsync_cmd = ['rsync', '-r']
def upload_scp(self, fr, to, flags = []):
self.upload(fr, to, flags, list(self.scp_cmd))
def upload_rsync(self, fr, to, flags = []):
self.upload(fr, to, flags, list(self.rsync_cmd))
def upload(self, fr, to, flags, cmd):
for e in flags:
cmd.append(e)
cmd.append(fr)
cmd.append(to)
subprocess.check_call(cmd)
class DryRunUploader(DryRun):
def upload_scp(self, fr, to, flags = []):
self.upload(fr, to, "scp")
def upload_rsync(self, fr, to, flags = []):
self.upload(fr, to.replace(':', '____').replace('@', "__"), "rsync")
def upload(self, fr, to, type):
self.copy(fr, "%s/%s/%s" % (self.location_root, type, to))
def maven_build_distribution(version):
"""Builds the distribution in the current working dir"""
mvn_commands = [["clean", "install", "-Passembly"],["deploy","-Passembly","-DskipTests"]]
for c in mvn_commands:
if settings['dry_run']:
c.append("-Dmaven.deploy.skip=true")
if settings['skip_tests']:
c.append("-Dmaven.deploy.skip=true")
if not settings['verbose']:
c.insert(0, '-q')
c.insert(0, 'mvn')
subprocess.check_call(c)
print "Verifying build"
# Check an assembly files ...
fn = "modeshape-distribution/target/modeshape-%s-javadoc.zip" % (version)
if os.path.isfile(fn):
f = open(fn)
xsd = f.read()
f.close()
def get_version_pattern():
return re.compile("^([2-9]\.[0-9])\.[0-9]\.(Final|(Alpha|Beta|CR)[1-9][0-9]?)$", re.IGNORECASE)
def get_version_major_minor(full_version):
pattern = get_version_pattern()
matcher = pattern.match(full_version)
return matcher.group(1)
def assert_python_minimum_version(major, minor):
e = re.compile('([0-9])\.([0-9])\.([0-9]).*')
m = e.match(sys.version)
major_ok = int(m.group(1)) == major
minor_ok = int(m.group(2)) >= minor
if not (minor_ok and major_ok):
prettyprint( "This script requires Python >= %s.%s.0. You have %s" % (major, minor, sys.version), Levels.FATAL)
sys.exit(3)
def unmarkdown(markdown_text):
# Define the dictionary of replacement regular expressions, keyed by the search expression
expressions = {
"\[(.*?)\]\[.*?]" : "\\1", # replace links
"^#{3,4}\\s*" : "", # remove h3 and h4 headers
"^##\\s*(.*)$" : "\\1\\n-------------------------------------------------------------------",
"^#\\s*" : "", # remove h1 headers
"^#\\s*" : "", # remove h2 headers
"\*\*(.*?)\*\*" : "\\1", # remove emphasis
"\*(.*?)\*" : "\\1", # remove emphasis
"__(.*?)__" : "\\1", # remove emphasis
"_(.*?)_" : "\\1", # remove emphasis
"^\\s*\[(.*?)\]\\s*:\\s*(http|ftp|ssh).*$\\n" : "", # replace links
}
# Now replace each of the regex pairs
text = markdown_text
for s_exp,r_exp in expressions.items():
search_exp = re.compile(s_exp,re.M)
text = search_exp.sub(r_exp,text)
return text;