-
Notifications
You must be signed in to change notification settings - Fork 1
/
common.py
395 lines (342 loc) · 13.9 KB
/
common.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
#
# A collection of shared functions for managing help flag mapping files.
#
import os
import sys
import pkgutil
import glob
from collections import defaultdict
from xml.sax.saxutils import escape
from oslo.config import cfg
# gettext internationalisation function requisite:
import __builtin__
__builtin__.__dict__['_'] = lambda x: x
def git_check(repo_path):
from git import Repo
"""
Check a passed directory to verify it is a valid git repository.
"""
try:
repo = Repo(repo_path)
assert repo.bare is False
package_name = os.path.basename(repo.remotes.origin.url).rstrip('.git')
except:
print "\nThere is a problem verifying that the directory passed in"
print "is a valid git repoistory. Please try again.\n"
sys.exit(1)
return package_name
def populate_groups(filepath):
"""
Takes a file formatted with lines of config option and group
separated by a space and constructs a dictionary indexed by
group, which is returned..
"""
groups = defaultdict(list)
groups_file = open(os.path.expanduser(filepath), 'r')
for line in groups_file:
try:
option, group = line.split(None, 1)
except ValueError:
print "Couldn't read groups file line:%s" % line
print "Check for formatting errors - did you add the group?"
sys.exit(1)
groups[group.strip()].append(option)
return groups
def extract_flags(repo_location, module_name, verbose=0, names_only=True):
"""
Loops through the repository, importing module by module to
populate the configuration object (cfg.CONF) created from Oslo.
"""
usable_dirs = []
module_location = os.path.dirname(repo_location + '/' + module_name)
for root, dirs, files in os.walk(module_location + '/' + module_name):
for name in dirs:
abs_path = os.path.join(root.split(module_location)[1][1:], name)
if ('/tests' not in abs_path and '/locale' not in abs_path and
'/cmd' not in abs_path and '/db/migration' not in abs_path):
usable_dirs.append(os.path.join(root.split(module_location)[1][1:], name))
for directory in usable_dirs:
for python_file in glob.glob(module_location + '/' + directory + "/*.py"):
if '__init__' not in python_file:
usable_dirs.append(os.path.splitext(python_file)[0][len(module_location) + 1:])
package_name = directory.replace('/', '.')
try:
__import__(package_name)
if verbose >= 1:
print "imported %s" % package_name
except ImportError as e:
"""
work around modules that don't like being imported in this way
FIXME This could probably be better, but does not affect the
configuration options found at this stage
"""
if verbose >= 2:
print str(e)
print "Failed to import: %s (%s)" % (package_name, e)
continue
flags = cfg.CONF._opts.items()
#extract group information
for group in cfg.CONF._groups.keys():
flags = flags + cfg.CONF._groups[group]._opts.items()
flags.sort()
return flags
def extract_flags_test(repo_loc, module, verbose=0):
"""
TEST TEST TEST TEST TEST TEST
TEST TEST TEST TEST TEST TEST
Loops through the repository, importing module by module to
populate the configuration object (cfg.CONF) created from Oslo.
TEST TEST TEST TEST TEST TEST
TEST TEST TEST TEST TEST TEST
"""
flag_data = {}
flag_files = []
usable_dirs = []
module_location = os.path.dirname(repo_loc + '/' + module)
for root, dirs, files in os.walk(module_location + '/' + module):
for name in dirs:
abs_path = os.path.join(root.split(module_location)[1][1:], name)
if ('/tests' not in abs_path and '/locale' not in abs_path and
'/cmd' not in abs_path and '/db/migration' not in abs_path):
usable_dirs.append(os.path.join(root.split(module_location)[1][1:], name))
for directory in usable_dirs:
for python_file in glob.glob(module_location + '/' + directory + "/*.py"):
if '__init__' not in python_file:
usable_dirs.append(os.path.splitext(python_file)[0][len(module_location) + 1:])
package_name = directory.replace('/', '.')
try:
__import__(package_name)
if verbose >= 1:
print "imported %s" % package_name
flag_data[str(package_name)] = sorted(cfg.CONF._opts.items())
except ImportError as e:
"""
work around modules that don't like being imported in this way
FIXME This could probably be better, but does not affect the
configuration options found at this stage
"""
if verbose >= 2:
print str(e)
print "Failed to import: %s (%s)" % (package_name, e)
continue
return flag_data
def write_test(file, repo_dir, pkg_name):
"""
"""
file1 = file + ".test"
flags = extract_flags_test(repo_dir, pkg_name)
with open(file1, 'a+') as f:
f.write("\n")
for filename, flag_info in flags.iteritems():
f.write("\n -- start file name area --\n")
f.write(filename)
f.write("\n -- end file name area --\n")
print "\n -- start file name area --\n"
print filename
print "\n -- end file name area --\n"
print len(flag_info)
for name, value in flag_info:
opt = value['opt']
#print type(opt)
#print opt
#print name
#print value
f.write(name)
f.write("\n")
def write_header(filepath, verbose=0):
"""
Write header to output flag file.
"""
pass
def write_buffer(file, flags, verbose=0):
"""
Write flag data to file. (The header is written with the write_header function.)
"""
pass
#with open(os.path.expanduser(filepath), 'wb') as f:
def write_flags(filepath, flags, name_only=True, verbose=0):
"""
write out the list of flags in the cfg.CONF object to filepath
if name_only is True - write only a list of names, one per line,
otherwise use MediaWiki syntax to write out the full table with
help text and default values.
"""
with open(os.path.expanduser(filepath), 'wb') as f:
if not name_only:
f.write("{|\n") # start table
# print headers
f.write("!")
f.write("!!".join(["name", "default", "description"]))
f.write("\n|-\n")
for name, value in flags:
opt = value['opt']
if not opt.help:
opt.help = "No help text available for this option"
if not name_only:
f.write("|")
f.write("||".join([name,
str(opt.default),
opt.help.replace("\n", " ")]))
f.write("\n|-\n")
else:
f.write(name + "\n")
if not name_only:
f.write("|}\n") # end table
def write_docbook(directory, flags, groups, package_name, verbose=0):
"""
Prints a docbook-formatted table for every group of options.
"""
count = 0
for group in groups.items():
groups_file = open(package_name + '-' + group[0] + '.xml', 'w')
groups_file.write('<?xml version="1.0" encoding="UTF-8"?>\n\
<para xmlns="http://docbook.org/ns/docbook" version="5.0">\n\
<table rules="all">\n\
<caption>Description of configuration options for ' + group[0] +
'</caption>\n\
<col width="50%"/>\n\
<col width="50%"/>\n\
<thead>\n\
<tr>\n\
<td>Configuration option=Default value</td>\n\
<td>(Type) Description</td>\n\
</tr>\n\
</thead>\n\
<tbody>')
for flag_name in group[1]:
for flag in flags:
if flag[0] == flag_name:
count = count + 1
opt = flag[1]["opt"]
if not opt.help:
opt.help = "No help text available for this option"
if type(opt).__name__ == "ListOpt" and opt.default is not None:
opt.default = ",".join(opt.default)
groups_file.write('\n <tr>\n\
<td>' + flag_name + '=' + str(opt.default) + '</td>\n\
<td>(' + type(opt).__name__ + ') '
+ escape(opt.help) + '</td>\n\
</tr>')
groups_file.write('\n </tbody>\n\
</table>\n\
</para>')
groups_file.close()
def create(flag_file, repo_path):
"""
Create new flag mappings file, containing help information for
the project whose repo location has been passed in at the command line.
"""
# flag_file testing.
#try:
# Test for successful creation of flag_file.
#except:
# If the test(s) fail, exit noting the problem(s).
# repo_path git repo validity testing.
#try:
# Test to be sure the repo_path passed in is a valid directory
# and that directory is a valid existing git repo.
#except:
# If the test(s) fail, exit noting the problem(s).
# get as much help as possible, searching recursively through the
# entire repo source directory tree.
#help_data = get_help(repo_path)
# Write this information to the file.
#write_file(flag_file, help_data)
def update(filepath, flags, name_only=True, verbose=0):
"""
Update flag mappings file, adding or removing entries as needed.
This will update the file content, essentially overriding the data.
The primary difference between create and update is that create will
make a new file, and update will just work with the data that is
data that is already there.
"""
original_flags = []
updated_flags = []
write_flags(filepath + '.new', flags, name_only=True, verbose=0)
original_flag_file = open(filepath)
updated_flag_file = open(filepath + '.new', 'r')
for line in original_flag_file:
original_flags.append(line.split()[0])
for line in updated_flag_file:
updated_flags.append(line.rstrip())
updated_flag_file.close()
removed_flags = set(original_flags) - set(updated_flags)
added_flags = set(updated_flags) - set(original_flags)
print "\nRemoved Flags\n"
for line in sorted(removed_flags):
print line
print "\nAdded Flags\n"
for line in sorted(added_flags):
print line
updated_flag_file = open(filepath + '.new', 'wb')
original_flag_file.seek(0)
for line in original_flag_file:
flag_name = line.split()[0]
if flag_name not in removed_flags:
for added_flag in added_flags:
if flag_name > added_flag:
updated_flag_file.write(added_flag + ' Unknown\n')
added_flags.remove(added_flag)
break
updated_flag_file.write(line)
def verify(flag_file):
"""
Verify flag file contents. No actions are taken.
"""
pass
def usage():
print "\nUsage: %s docbook <groups file> <source loc>" % sys.argv[0]
print "\nGenerate a list of all flags for package in source loc and"\
"\nwrites them in a docbook table format, grouped by the groups"\
"\nin the groups file, one file per group.\n"
print "\n %s names <names file> <source loc>" % sys.argv[0]
print "\nGenerate a list of all flags names for the package in"\
"\nsource loc and writes them to names file, one per line \n"
def parse_me_args():
import argparse
parser = argparse.ArgumentParser(
description='Manage flag files, to aid in updatingdocumentation.',
epilog='Example: %(prog)s -a create -in ./nova.flagfile -fmt docbook\
-p /nova',
usage='%(prog)s [options]')
parser.add_argument('-a', '--action',
choices=['create', 'update', 'verify'],
dest='action',
help='action (create, update, verify) [REQUIRED]',
required=True,
type=str,)
# trying str data type... instead of file.
parser.add_argument('-i', '-in', '--input',
dest='file',
help='flag file being worked with [REQUIRED]',
required=True,
type=str,)
parser.add_argument('-f', '-fmt', '--format', '-o', '-out',
dest='format',
help='file output format (options: docbook, names)',
required=False,
type=str,)
# ..tried having 'dir' here for the type, but the git.Repo function
# requires a string is passed to it.. a directory won't work.
parser.add_argument('-p', '--path',
dest='repo',
help='path to valid git repository [REQUIRED]',
required=True,
type=str,)
parser.add_argument('-v', '--verbose',
action='count',
default=0,
dest='verbose',
required=False,)
parser.add_argument('-no', '--name_only',
action='store_true',
dest='name',
help='whether output should contain names only',
required=False,)
parser.add_argument('-test',
action='store_true',
dest='test',
help=argparse.SUPPRESS,
required=False,)
args = vars(parser.parse_args())
return args