/
utils.py
634 lines (515 loc) · 19.2 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
#!/usr/bin/env python
#
# utils.py
"""
General utilities.
"""
#
# Copyright © 2020 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
#
# stdlib
import datetime
import os
import pathlib
import re
import sys
import textwrap
from datetime import date, timedelta
from io import StringIO
from types import ModuleType
from typing import Any, Callable, Iterable, Iterator, List, Mapping, Optional, TypeVar, Union, no_type_check
# 3rd party
import dulwich.repo
import isort
import isort.settings
import jinja2
import yapf_isort
from apeye.requests_url import RequestsURL
from domdf_python_tools.compat import importlib_resources
from domdf_python_tools.dates import calc_easter
from domdf_python_tools.import_tools import discover_entry_points
from domdf_python_tools.paths import PathPlus, sort_paths
from domdf_python_tools.pretty_print import FancyPrinter
from domdf_python_tools.stringlist import StringList
from domdf_python_tools.typing import PathLike
from jinja2 import Environment
from ruamel.yaml import YAML
from shippinglabel import normalize
from southwark import open_repo_closing, status
from typing_extensions import ContextManager
# this package
from repo_helper.configupdater2 import ConfigUpdater, Section
__all__ = [
"resource",
"IniConfigurator",
"discover_entry_points",
"easter_egg",
"indent_join",
"indent_with_tab",
"license_lookup",
"no_dev_versions",
"normalize",
"pformat_tabs",
"reformat_file",
"today",
"sort_paths",
"commit_changes",
"stage_changes",
"get_license_text",
"set_gh_actions_versions",
]
KT = TypeVar("KT")
VT = TypeVar("VT")
#: Under normal circumstances returns :meth:`datetime.date.today`.
today: date = date.today()
def indent_with_tab(
text: str,
depth: int = 1,
predicate: Optional[Callable[[str], bool]] = None,
) -> str:
r"""
Adds ``'\t'`` to the beginning of selected lines in 'text'.
:param text: The text to indent.
:param depth: The depth of the indentation.
:param predicate: If given, ``'\t'`` will only be added to the lines where ``predicate(line)``
is :py:obj`True`. If ``predicate`` is not provided, it will default to adding ``'\t'``
to all non-empty lines that do not consist solely of whitespace characters.
"""
return textwrap.indent(text, '\t' * depth, predicate=predicate)
def pformat_tabs(
obj: object,
width: int = 80,
depth: Optional[int] = None,
*,
compact: bool = False,
) -> str:
"""
Format a Python object into a pretty-printed representation.
Indentation is set at one tab.
:param obj: The object to format.
:param width: The maximum width of the output.
:param depth:
:param compact:
"""
prettyprinter = FancyPrinter(indent=4, width=width, depth=depth, compact=compact)
buf = StringList()
for line in prettyprinter.pformat(obj).splitlines():
buf.append(re.sub("^ {4}", r"\t", line))
return str(buf)
#: Mapping of license short codes to license names used in trove classifiers.
license_lookup = {
"AFL-1.1": "Academic Free License (AFL)",
"AFL-1.2": "Academic Free License (AFL)",
"AFL-2.0": "Academic Free License (AFL)",
"AFL-2.1": "Academic Free License (AFL)",
"AFL-3.0": "Academic Free License (AFL)",
"Apache": "Apache Software License",
"Apache-1.0": "Apache Software License",
"Apache-1.1": "Apache Software License",
"Apache-2.0": "Apache Software License",
"APSL-1.0": "Apple Public Source License",
"APSL-1.1": "Apple Public Source License",
"APSL-1.2": "Apple Public Source License",
"APSL-2.0": "Apple Public Source License",
"Artistic-1.0": "Artistic License",
"AAL": "Attribution Assurance License",
"BSD": "BSD License",
"BSD-2-Clause": "BSD License",
"BSD-3-Clause": "BSD License",
"BSL-1.0": "Boost Software License 1.0 (BSL-1.0)",
"CDDL-1.0": "Common Development and Distribution License 1.0 (CDDL-1.0)",
"CPL-1.0": "Common Public License",
"EPL-1.0": "Eclipse Public License 1.0 (EPL-1.0)",
"EPL-2.0": "Eclipse Public License 2.0 (EPL-2.0)",
"EFL-1.0": "Eiffel Forum License",
"EFL-2.0": "Eiffel Forum License",
"EUPL 1.0": "European Union Public Licence 1.0 (EUPL 1.0)",
"EUPL 1.1": "European Union Public Licence 1.1 (EUPL 1.1)",
"EUPL 1.2": "European Union Public Licence 1.2 (EUPL 1.2)",
"AGPL-3.0-only": "GNU Affero General Public License v3",
"AGPL-3.0": "GNU Affero General Public License v3",
"AGPL-3.0-or-later": "GNU Affero General Public License v3 or later (AGPLv3+)",
"AGPL-3.0+": "GNU Affero General Public License v3 or later (AGPLv3+)",
"FDL": "GNU Free Documentation License (FDL)",
"GFDL-1.1-only": "GNU Free Documentation License (FDL)",
"GFDL-1.1-or-later": "GNU Free Documentation License (FDL)",
"GFDL-1.2-only": "GNU Free Documentation License (FDL)",
"GFDL-1.2-or-later": "GNU Free Documentation License (FDL)",
"GFDL-1.3-only": "GNU Free Documentation License (FDL)",
"GFDL-1.3-or-later": "GNU Free Documentation License (FDL)",
"GFDL-1.2": "GNU Free Documentation License (FDL)",
"GFDL-1.1": "GNU Free Documentation License (FDL)",
"GFDL-1.3": "GNU Free Documentation License (FDL)",
"GPL": "GNU General Public License (GPL)",
"GPL-1.0-only": "GNU General Public License (GPL)",
"GPL-1.0-or-later": "GNU General Public License (GPL)",
"GPLv2": "GNU General Public License v2 (GPLv2)",
"GPL-2.0-only": "GNU General Public License v2 (GPLv2)",
"GPLv2+": "GNU General Public License v2 or later (GPLv2+)",
"GPL-2.0-or-later": "GNU General Public License v2 or later (GPLv2+)",
"GPLv3": "GNU General Public License v3 (GPLv3)",
"GPL-3.0-only": "GNU General Public License v3 (GPLv3)",
"GPLv3+": "GNU General Public License v3 or later (GPLv3+)",
"GPL-3.0-or-later": "GNU General Public License v3 or later (GPLv3+)",
"LGPLv2": "GNU Lesser General Public License v2 (LGPLv2)",
"LGPLv2+": "GNU Lesser General Public License v2 or later (LGPLv2+)",
"LGPLv3": "GNU Lesser General Public License v3 (LGPLv3)",
"LGPL-3.0-only": "GNU Lesser General Public License v3 (LGPLv3)",
"LGPLv3+": "GNU Lesser General Public License v3 or later (LGPLv3+)",
"LGPL-3.0-or-later": "GNU Lesser General Public License v3 or later (LGPLv3+)",
"LGPL": "GNU Library or Lesser General Public License (LGPL)",
"HPND": "Historical Permission Notice and Disclaimer (HPND)",
"IPL-1.0": "IBM Public License",
"ISCL": "ISC License (ISCL)",
"Intel": "Intel Open Source License",
"MIT": "MIT License",
"MirOS": "MirOS License (MirOS)",
"Motosoto": "Motosoto License",
"MPL": "Mozilla Public License 1.0 (MPL)",
"MPL-1.0": "Mozilla Public License 1.0 (MPL)",
"MPL 1.1": "Mozilla Public License 1.1 (MPL 1.1)",
"MPL 2.0": "Mozilla Public License 2.0 (MPL 2.0)",
"NGPL": "Nethack General Public License",
"Nokia": "Nokia Open Source License",
"OGTSL": "Open Group Test Suite License",
"OSL-3.0": "Open Software License 3.0 (OSL-3.0)",
"PostgreSQL": "PostgreSQL License",
"CNRI-Python": "Python License (CNRI Python License)",
"PSF-2.0": "Python Software Foundation License",
"QPL-1.0": "Qt Public License (QPL)",
"RSCPL": "Ricoh Source Code Public License",
"OFL-1.1": "SIL Open Font License 1.1 (OFL-1.1)",
"Sleepycat": "Sleepycat License",
"SISSL": "Sun Industry Standards Source License (SISSL)",
"SISSL-1.2": "Sun Industry Standards Source License (SISSL)",
"SPL-1.0": "Sun Public License",
"UPL": "Universal Permissive License (UPL)",
"UPL-1.0": "Universal Permissive License (UPL)",
"NCSA": "University of Illinois/NCSA Open Source License",
"VSL-1.0": "Vovida Software License 1.0",
"W3C": "W3C License",
"Xnet": "X.Net License",
"ZPL-1.1": "Zope Public License",
"ZPL-2.0": "Zope Public License",
"ZPL-2.1": "Zope Public License",
"Zlib": "zlib/libpng License",
"Proprietary": "Other/Proprietary License",
"Other": "Other/Proprietary License",
"PD": "Public Domain",
"Public Domain": "Public Domain",
}
def reformat_file(filename: PathLike, yapf_style: str, isort_config_file: str) -> int:
"""
Reformat the given file.
:param filename:
:param yapf_style: The name of the yapf style, or the path to the yapf style file.
:param isort_config_file: The filename of the isort configuration file.
"""
old_isort_settings = isort.settings.CONFIG_SECTIONS.copy()
try:
isort.settings.CONFIG_SECTIONS["isort.cfg"] = ("settings", "isort")
isort_config = isort.Config(settings_file=str(isort_config_file))
r = yapf_isort.Reformatter(filename, yapf_style, isort_config)
ret = r.run()
r.to_file()
return ret
finally:
isort.settings.CONFIG_SECTIONS = old_isort_settings
def indent_join(iterable: Iterable[str]) -> str:
"""
Join an iterable of strings with newlines,
and indent each line with a tab if there is more then one element.
:param iterable:
""" # noqa: D400
iterable = list(iterable)
if len(iterable) > 1:
if not iterable[0] == '':
iterable.insert(0, '')
return indent_with_tab(textwrap.dedent('\n'.join(iterable)))
class IniConfigurator:
"""
Base class to generate ``.ini`` configuration files.
:param base_path:
"""
managed_sections: List[str]
_ini: ConfigUpdater
_output: StringList
managed_message: str = "This file is managed by 'repo_helper'."
filename: str
def __init__(self, base_path: pathlib.Path):
self.base_path = base_path
self._ini = ConfigUpdater()
self._output = StringList([
f"# {self.managed_message}",
"# You may add new sections, but any changes made to the following sections will be lost:",
])
self.managed_sections = self.managed_sections[:]
for sec in self.managed_sections:
self._ini.add_section(sec)
self._output.append(f"# * {sec}")
self._output.blankline(ensure_single=True)
def merge_existing(self, ini_file: pathlib.Path) -> None:
"""
Merge existing sections in the configuration file into the new configuration.
:param ini_file: The existing ``.ini`` file.
"""
if ini_file.is_file():
existing_config = ConfigUpdater()
existing_config.read(str(ini_file))
for section in existing_config.sections_blocks():
if section.name not in self.managed_sections:
self._ini.add_section(section)
def write_out(self) -> None:
"""
Write out to the ``.ini`` file.
"""
ini_file = PathPlus(self.base_path / self.filename)
for section_name in self.managed_sections:
getattr(self, re.sub("[:.-]", '_', section_name))()
self.merge_existing(ini_file)
self._output.append(str(self._ini))
ini_file.write_lines(self._output)
def copy_existing_value(self, section: Section, key: str) -> None:
"""
Copy the existing value for ``key``, if present, to the new configuration.
:param section:
:param key:
"""
if key in section:
self._ini[section.name][key] = section[key].value
def easter_egg() -> None: # noqa: D103 # pragma: no cover
easter = calc_easter(today.year)
easter_margin = timedelta(days=7)
if today - easter_margin <= easter <= today + easter_margin:
print("🐇 🐣 🥚")
elif date(today.year, 10, 24) <= today <= date(today.year, 11, 2):
print("🎃 👻 🦇")
elif today == date(today.year, 11, 5):
print("🎆 🔥 🚀")
elif today == date(today.year, 11, 11):
print("We will remember them.")
elif today.month == 12:
print("🎅 ☃️ 🎁")
def no_dev_versions(versions: Iterable[str]) -> List[str]:
"""
Returns the subset of ``versions`` which does not end with ``-dev``.
:param versions:
"""
return [v for v in versions if not v.endswith("-dev")]
def stage_changes(
repo: Union[PathLike, dulwich.repo.Repo],
files: Iterable[PathLike],
) -> List[PathPlus]:
"""
Stage any files that have been updated, added or removed.
:param repo: The repository.
:param files: List of files to stage.
:returns: A list of staged files.
Not all files in ``files`` will have been changed, and only changes are staged.
.. versionadded:: 2020.11.23
"""
with open_repo_closing(repo) as repo:
stat = status(repo)
unstaged_changes = stat.unstaged
untracked_files = stat.untracked
staged_files = []
for filename in files:
filename = PathPlus(filename)
if filename.is_absolute():
filename = filename.relative_to(repo.path)
if filename in unstaged_changes or filename in untracked_files:
repo.stage(os.path.normpath(filename))
staged_files.append(filename)
elif (
filename in stat.staged["add"] or filename in stat.staged["modify"]
or filename in stat.staged["delete"]
):
staged_files.append(filename)
return staged_files
def commit_changes(
repo: Union[PathLike, dulwich.repo.Repo],
message: str = "Updated files with 'repo_helper'.",
) -> str:
"""
Commit staged changes.
:param repo: The repository to commit in.
:param message: The commit message to use.
:returns: The SHA of the commit.
.. versionadded:: 2020.11.23
"""
with open_repo_closing(repo) as repo:
current_time = datetime.datetime.now(datetime.timezone.utc).astimezone()
tzinfo = current_time.tzinfo
assert tzinfo is not None
time_offset = tzinfo.utcoffset(None)
assert time_offset is not None
current_timezone = time_offset.total_seconds()
commit_sha = repo.do_commit(
message=message.encode("UTF-8"),
commit_timestamp=current_time.timestamp(),
commit_timezone=current_timezone,
)
return commit_sha.decode("UTF-8")
def brace(string: str) -> str:
return f"{{{{ {string} }}}}"
def set_gh_actions_versions(py_versions: Iterable[str]) -> List[str]:
"""
Convert development Python versions into the appropriate versions for GitHub Actions.
:param py_versions:
"""
py_versions = list(py_versions)
# Keep in sync with https://github.com/actions/python-versions/releases
if "3.9-dev" in py_versions:
py_versions[py_versions.index("3.9-dev")] = "3.9"
if "3.10-dev" in py_versions:
py_versions[py_versions.index("3.10-dev")] = "3.10"
if "3.11-dev" in py_versions:
py_versions[py_versions.index("3.11-dev")] = "3.11"
if "3.12-dev" in py_versions:
py_versions[py_versions.index("3.12-dev")] = "3.12"
if "3.13-dev" in py_versions:
py_versions[py_versions.index("3.13-dev")] = "3.13.0-alpha.5"
if "3.13" in py_versions:
py_versions[py_versions.index("3.13")] = "3.13.0-alpha.5"
if "pypy3" in py_versions:
py_versions[py_versions.index("pypy3")] = "pypy-3.6"
if "pypy36" in py_versions:
py_versions[py_versions.index("pypy36")] = "pypy-3.6"
if "pypy3.6" in py_versions:
py_versions[py_versions.index("pypy3.6")] = "pypy-3.6"
if "pypy37" in py_versions:
py_versions[py_versions.index("pypy37")] = "pypy-3.7"
if "pypy3.7" in py_versions:
py_versions[py_versions.index("pypy3.7")] = "pypy-3.7"
if "pypy38" in py_versions:
py_versions[py_versions.index("pypy38")] = "pypy-3.8"
if "pypy3.8" in py_versions:
py_versions[py_versions.index("pypy3.8")] = "pypy-3.8"
if "pypy39" in py_versions:
py_versions[py_versions.index("pypy39")] = "pypy-3.9"
if "pypy3.9" in py_versions:
py_versions[py_versions.index("pypy3.9")] = "pypy-3.9"
if "pypy310" in py_versions:
py_versions[py_versions.index("pypy310")] = "pypy-3.10"
if "pypy3.10" in py_versions:
py_versions[py_versions.index("pypy3.10")] = "pypy-3.10"
if "rustpython" in py_versions:
py_versions.remove("rustpython")
return py_versions
_yaml_round_trip_dumper = YAML(typ="rt")
_yaml_round_trip_dumper.default_flow_style = False
def _round_trip_dump(obj: Any) -> str:
stream = StringIO()
_yaml_round_trip_dumper.dump(obj, stream=stream)
return stream.getvalue()
@no_type_check
def resource(
package: Union[str, ModuleType],
resource: PathLike,
) -> ContextManager[pathlib.Path]:
"""
Retrieve the path to a resource inside a package.
.. versionadded:: 2022.4.4
:param package: The name of the package, or a module object representing it.
:param resource: The name of the resource.
"""
if sys.version_info < (3, 7) or sys.version_info >= (3, 11):
return importlib_resources.as_file(importlib_resources.files(package) / os.fspath(resource))
else:
return importlib_resources.path(package, resource)
base_license_url = RequestsURL("https://raw.githubusercontent.com/licenses/license-templates/master/templates/")
license_file_lookup = dict([
(
"GNU Lesser General Public License v3 (LGPLv3)",
(base_license_url / "lgpl.txt", "lgpl3.py"),
),
(
"GNU Lesser General Public License v3 or later (LGPLv3+)",
(base_license_url / "lgpl.txt", "lgpl3_plus.py")
),
("GNU General Public License v3 (GPLv3)", (base_license_url / "gpl3.txt", "gpl3.py")),
("GNU General Public License v3 or later (GPLv3+)", (base_license_url / "gpl3.txt", "gpl3_plus.py")),
(
"GNU General Public License v2 (GPLv2)",
(RequestsURL("https://www.gnu.org/licenses/old-licenses/gpl-2.0.txt"), "gpl2.py"),
),
(
"GNU General Public License v2 or later (GPLv2+)",
(RequestsURL("https://www.gnu.org/licenses/old-licenses/gpl-2.0.txt"), "gpl2_plus.py")
),
("MIT License", (base_license_url / "mit.txt", "mit.py")),
])
def get_license_text(
license_name: str,
copyright_years: Union[str, int],
author: str,
project_name: str,
) -> str:
"""
Obtain the license text for the given license.
.. versionadded:: 2022.4.4
:param license_name: The name of the license.
:param copyright_years: The copyright years (e.g. ``'2019-2021'``) to display in the license.
Not supported by all licenses.
:param author: The name of the author/copyright holder to display in the license.
Not supported by all licenses.
:param project_name: The name of the project to display in the license.
Not supported by all licenses.
Licenses are obtained from https://github.com/licenses/license-templates.
"""
if license_name in license_lookup:
license_name = license_lookup[license_name]
# Licenses from https://github.com/licenses/license-templates/tree/master/templates
license_url: Optional[RequestsURL] = None
license_text: str = ''
# TODO: 2 vs 3 clause BSD
if license_name in license_file_lookup:
license_url = license_file_lookup[license_name][0]
elif license_name == "BSD License":
license_url = base_license_url / "bsd2.txt"
elif license_name == "Apache Software License":
license_url = base_license_url / "apache.txt"
if license_url is not None:
for attempt in [1, 2]:
try:
response = license_url.get()
except Exception:
# except requests.exceptions.RequestException:
if attempt == 1:
continue
else:
raise
if response.status_code == 200:
license_text = response.text
license_template = Environment(
loader=jinja2.BaseLoader(),
undefined=jinja2.StrictUndefined,
).from_string(license_text)
return license_template.render(
year=copyright_years,
organization=author,
project=project_name,
)
def get_keys(mapping: Mapping[KT, VT], *keys: KT) -> Iterator[VT]:
r"""
Returns an iterator over ``\*keys`` from ``mapping``.
:param mapping:
:param \*keys:
"""
for key in keys:
yield mapping[key]