-
Notifications
You must be signed in to change notification settings - Fork 86
/
toolsdivers.py
402 lines (335 loc) · 14.7 KB
/
toolsdivers.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
#! /usr/bin/env python
# -*- coding: utf-8 -*-
"""Various tools.
"""
from __future__ import absolute_import, print_function
import os, time, warnings
from collections import OrderedDict as _OrderedDict
import numpy as np
from matplotlib import pyplot as plt
from subprocess import CalledProcessError, STDOUT
import pkg_resources
from . import genericsettings, testbedsettings
class Infolder(object):
"""Contextmanager to do some work in a folder of choice and change dir
back in the end.
Usage:
>>> import os
>>> import cocopp.toolsdivers
>>> dir_ = os.getcwd() # for the record
>>> with cocopp.toolsdivers.Infolder('..'):
... # do some work in a folder here, e.g. open a file
... len(dir_) > len(os.getcwd()) and os.getcwd() in dir_
True
>>> # magically we are back in the original folder
>>> assert dir_ == os.getcwd()
"""
def __init__(self, foldername):
self.target_dir = foldername
def __enter__(self):
self.root_dir = os.getcwd()
os.chdir(self.target_dir)
def __exit__(self, *args):
os.chdir(self.root_dir)
class StringList(list):
"""A microtool to join a list of strings using property `as_string`.
`StringList` can also be initialized with a string.
>>> from cocopp.toolsdivers import StringList
>>> StringList('ab bc') == ['ab', 'bc']
True
>>> word_list = StringList(['this', 'has', 'a', 'leading', 'and',
... 'trailing', 'space'])
>>> word_list.as_string
' this has a leading and trailing space '
`as_string` is less typing than
>>> ' ' + ' '.join(word_list) + ' ' == word_list.as_string
True
and provides tab completion.
"""
def __init__(self, list_or_str):
try:
inlist = list_or_str.split()
except AttributeError:
inlist = list_or_str
list.__init__(self, inlist)
@property
def as_string(self):
"""return concatenation with spaces between"""
return ' ' + ' '.join(self) + ' '
class AlgorithmList(list):
"""Not in use. Not necessary when the algorithm dict is an `OrderedDict` anyway.
A `list` representing the algorithm name arguments in original order.
The method `ordered_dict` allows to transform an algorithm `dict` into an
`OrderedDict` using the order in self.
>>> from cocopp.toolsdivers import AlgorithmList
>>> l = ['b', 'a', 'c']
>>> al = AlgorithmList(l)
>>> d = dict(zip(l[-1::-1], [1, 2, 3]))
>>> for i, name in enumerate(al.ordered_dict(d)):
... assert name == l[i]
"""
def ordered_dict(self, algorithms_dict):
"""return algorithms_dict as `OrderedDict` in order of self.
Keys that are not in self are sorted using `sorted`.
"""
if set(algorithms_dict) != set(self):
warnings.warn("keys in algorithm dict: \n%s\n"
"do not agree with original algorithm list: \n%s\n"
% (str(algorithms_dict.keys()), str(self)))
res = _OrderedDict()
for name in self:
if name in algorithms_dict:
res[name] = algorithms_dict[name]
for name in sorted(algorithms_dict):
if name not in res:
res[name] = algorithms_dict[name]
assert res == algorithms_dict # compares keys and values
return res
def print_done(message=' done'):
"""prints a message with time stamp"""
print(message, '(' + time.asctime() + ').')
def equals_approximately(a, b, eps=1e-12):
if a < 0:
a, b = -1 * a, -1 * b
return a - eps < b < a + eps or (1 - eps) * a < b < (1 + eps) * a
def less(a, b):
"""return a < b, while comparing nan results in False without warning"""
current_err_setting = np.geterr()
np.seterr(invalid='ignore')
res = a < b
np.seterr(**current_err_setting)
return res
def diff_attr(m1, m2, exclude=('_', )):
"""return `list` of ``[name, val1, val2]`` triplets for
attributes with different values.
Attributes whose names start with any string from the
`exclude` list are skipped. Furthermore, only attributes
present in both `m1` and `m2` are compared.
This function was introduced to compare the `genericsettings`
module with its state directly after import. It should be
applicable any other two class instances as well.
Details: to "find" the attributes, `m1.__dict__` is iterated over.
"""
return [[key, getattr(m1, key), getattr(m2, key)]
for key in m1.__dict__
if hasattr(m2, key) and
not any(key.startswith(s) for s in exclude)
and np.all(getattr(m1, key) != getattr(m2, key))]
def prepend_to_file(filename, lines, maxlines=1000, warn_message=None):
""""prepend lines the tex-command filename """
try:
with open(filename, 'r') as f:
lines_to_append = list(f)
except IOError:
lines_to_append = []
with open(filename, 'w') as f:
for line in lines:
f.write(line + '\n')
for i, line in enumerate(lines_to_append):
f.write(line)
if i > maxlines:
print(warn_message)
break
def replace_in_file(filename, old_text, new_text):
""""replace a string in the file with another string"""
lines = []
try:
with open(filename, 'r') as f:
lines = list(f)
except IOError:
print('File %s does not exist.' % filename)
if lines:
with open(filename, 'w') as f:
for line in lines:
f.write(line.replace(old_text, new_text))
def truncate_latex_command_file(filename, keeplines=200):
"""truncate file but keep in good latex shape"""
open(filename, 'a').close()
with open(filename, 'r') as f:
lines = list(f)
with open(filename, 'w') as f:
for i, line in enumerate(lines):
if i > keeplines and line.startswith('\providecommand'):
break
f.write(line)
def strip_pathname(name):
"""remove ../ and ./ and leading/trailing blanks and path separators
from input string ``name`` and replace any remaining path separator
with '/'"""
return name.replace('..' + os.sep, '').replace('.' + os.sep, '').strip().strip(os.sep).replace(os.sep, '/')
def strip_pathname1(name):
"""remove ../ and ./ and leading/trailing blanks and path separators
from input string ``name``, replace any remaining path separator
with '/', and keep only the last part of the path"""
return (name.replace('..' + os.sep, '').replace('.' + os.sep, '').strip().strip(os.sep).split(os.sep)[-1]).replace('data', '').replace('Data', '').replace('DATA', '').replace('.tar.gz', '').replace('.tgz', '').replace('.tar', '').replace(genericsettings.extraction_folder_prefix, '').strip(os.sep).replace(os.sep, '/')
def strip_pathname2(name):
"""remove ../ and ./ and leading/trailing blanks and path separators
from input string ``name``, replace any remaining path separator
with '/', and keep only the last two parts of the path, or only the
last"""
return os.sep.join(name.replace('..' + os.sep, '').replace('.' + os.sep, '').strip().strip(os.sep).split(os.sep)[-2:]).replace('data', '').replace('Data', '').replace('DATA', '').replace('.tar.gz', '').replace('.tgz', '').replace('.tar', '').strip(os.sep).replace(os.sep, '/')
def str_to_latex(string):
"""do replacements in ``string`` such that it most likely compiles with latex """
#return string.replace('\\', r'\textbackslash{}').replace('_', '\\_').replace(r'^', r'\^\,').replace(r'%', r'\%').replace(r'~', r'\ensuremath{\sim}').replace(r'#', r'\#')
return string.replace('\\', r'\textbackslash{}').replace('_', ' ').replace(r'^', r'\^\,').replace(r'%', r'\%').replace(r'~', r'\ensuremath{\sim}').replace(r'#', r'\#')
def number_of_digits(val, precision=1e-13):
"""returns the number of non-zero digits of a number, e.g. two for 1200 or three for 2.03.
"""
raise NotImplementedError()
def num2str(val, significant_digits=2, force_rounding=False,
max_predecimal_digits=5, max_postdecimal_leading_zeros=1,
remove_trailing_zeros=True):
"""returns the shortest string representation with either ``significant_digits``
digits shown or its true value, whichever is shorter.
``force_rounding`` shows no more than the desired number of significant digits,
which means, e.g., ``12345`` becomes ``12000``.
``remove_trailing_zeros`` removes zeros, if and only if the value is exactly.
>>> from cocopp import toolsdivers as td
>>> print([td.num2str(val) for val in [12345, 1234.5, 123.45, 12.345, 1.2345, .12345, .012345, .0012345]])
['12345', '1234', '123', '12', '1.2', '0.12', '0.012', '1.2e-3']
"""
if val == 0:
return '0'
assert significant_digits > 0
is_negative = val < 0
original_value = val
val = float(np.abs(val))
order_of_magnitude = int(np.floor(np.log10(val)))
# number of digits before decimal point == order_of_magnitude + 1
fac = 10**(significant_digits - 1 - order_of_magnitude)
val_rounded = np.round(fac * val) / fac
# the strategy is now to produce two string representations
# cut each down to the necessary length and return the better
# the first is %f format
if order_of_magnitude + 1 >= significant_digits:
s = str(int(val_rounded if force_rounding else np.round(val)))
else:
s = str(val_rounded)
idx1 = 0 # first non-zero index
while idx1 < len(s) and s[idx1] in ('-', '0', '.'):
idx1 += 1 # find index of first significant number
idx2 = idx1 + significant_digits + (s.find('.') > idx1)
# print(val, val_rounded, s, len(s), idx1, idx2)
# pad some zeros in the end, in case
if val != val_rounded:
if len(s) < idx2:
s += '0' * (idx2 - len(s))
# remove zeros from the end, in case
if val == val_rounded and remove_trailing_zeros:
while s[-1] == '0':
s = s[0:-1]
if s[-1] == '.':
s = s[0:-1]
s_float = ('-' if is_negative else '') + s
# now the second, %e format
s = ('%.' + str(significant_digits - 1) + 'e') % val
if eval(s) == val and s.find('.') > 0:
while s.find('0e') > 0:
s = s.replace('0e', 'e')
s = s.replace('.e', 'e')
s = s.replace('e+', 'e')
while s.find('e0') > 0:
s = s.replace('e0', 'e')
while s.find('e-0') > 0:
s = s.replace('e-0', 'e-')
if s[-1] == 'e':
s = s[:-1]
s_exp = ('-' if is_negative else '') + s
# print(s_float, s_exp)
# now return the better (most of the time the shorter) representation
if (len(s_exp) < len(s_float) or
s_float.find('0.' + '0' * (max_postdecimal_leading_zeros + 1)) > -1 or
np.abs(val_rounded) >= 10**(max_predecimal_digits + 1)
):
return s_exp
else:
return s_float
def number_to_latex(number_as_string):
"""usage as ``number_to_latex(num2str(1.023e-12)) == "'-1.0\\times10^{-12}'"``"""
s = number_as_string
if s.find('e') > 0:
if s.startswith('1e') or s.startswith('-1e'):
s = s.replace('1e', '10^{')
else:
s = s.replace('e', '\\times10^{')
s += '}'
return s
def number_to_html(number_as_string):
"""usage as ``number_to_html(num2str(1.023e-12)) == "'-1.0 x 10<sup>-12</sup>'"``"""
s = number_as_string
if s.find('e') > 0:
if s.startswith('1e') or s.startswith('-1e'):
s = s.replace('1e', '10<sup>')
else:
s = s.replace('e', ' x 10<sup>')
s += '</sup>'
return s
def legend(*args, **kwargs):
kwargs.setdefault('framealpha', 0.2)
try:
plt.legend(*args, **kwargs)
except:
warnings.warn("framealpha not effective")
kwargs.pop('framealpha')
plt.legend(*args, **kwargs)
try:
from subprocess import check_output
except ImportError:
import subprocess
def check_output(*popenargs, **kwargs):
r"""Run command with arguments and return its output as a byte string.
Backported from Python 2.7 as it's implemented as pure python on stdlib.
WARNING: This method is also defined in ../../code-experiments/tools/cocoutils.py.
If you change something you have to change it in both files.
"""
process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
output, unused_err = process.communicate()
retcode = process.poll()
if retcode:
cmd = kwargs.get("args")
if cmd is None:
cmd = popenargs[0]
error = subprocess.CalledProcessError(retcode, cmd)
error.output = output
raise error
return output
def git(args):
"""Run a git command and return its output.
All errors are deemed fatal and the system will quit.
WARNING: This method is also defined in ../../code-experiments/tools/cocoutils.py.
If you change something you have to change it in both files.
"""
full_command = ['git']
full_command.extend(args)
try:
output = check_output(full_command, env=os.environ,
stderr=STDOUT, universal_newlines=True)
output = output.rstrip()
except CalledProcessError as e:
# print('Failed to execute "%s"' % str(full_command))
raise
return output
def get_version_label(algorithmID=None):
""" Returns a string with the COCO version of the installed postprocessing,
potentially adding the hash of the hypervolume reference values from
the actual experiments (in the `bbob-biobj` setting).
If algorithmID==None, the set of different hypervolume reference values
from all algorithms, read in by the postprocessing, are returned in
the string. If more than one reference value is present in the data,
the string displays also a warning.
"""
coco_version = pkg_resources.require('cocopp')[0].version
reference_values = testbedsettings.get_reference_values(algorithmID)
if reference_values and type(reference_values) is set:
label = "v%s, hv-hashes inconsistent:" % (coco_version)
for r in reference_values:
label = label + " %s and" % (r)
label = label[:-3] + "found!"
else:
label = "v%s" % (coco_version) if reference_values is None else "v%s, hv-hash=%s" % (coco_version, reference_values)
return label
def path_in_package(sub_path=""):
"""return the absolute path prepended to `subpath` in this module.
"""
egg_info = pkg_resources.require('cocopp')[0]
return os.path.join(egg_info.location, egg_info.project_name, sub_path)