-
Notifications
You must be signed in to change notification settings - Fork 3
/
build.py
328 lines (283 loc) · 12.2 KB
/
build.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
import hashlib
import json
import re
import shutil
import tempfile
from datetime import datetime
from pathlib import Path
from typing import Callable
import click
from .common import GrablibError, main_logger, progress_logger
STARTS_DOWNLOAD = re.compile('^(?:DOWNLOAD|DL)/')
STARTS_NODE_M = re.compile('^(?:NODE_MODULES|NM)/')
STARTS_SRC = re.compile('^SRC/')
class Builder:
"""
main class for "building" assets eg. concatenating and minifying js and compiling sass
"""
def __init__(self, *, build_root, build, download_root: str=None, debug=False, **data):
self.build_root = Path(build_root).absolute()
self.build = build
self.download_root = download_root and Path(download_root).resolve()
self.files_built = 0
self.debug = debug
self._jsmin = None
def __call__(self):
wipe_data = self.build.get('wipe', None)
wipe_data and self.wipe(wipe_data)
cat_data = self.build.get('cat', None)
cat_data and self.cat(cat_data)
sass_data = self.build.get('sass', None)
sass_data and self.sass(sass_data)
def cat(self, data):
start = datetime.now()
total_files_combined = 0
for dest, srcs in data.items():
if not isinstance(srcs, list):
raise GrablibError('source files for concatenation should be a list')
final_content, files_combined = '', 0
for src in srcs:
if isinstance(src, str):
src = {'src': src}
path = self._file_path(src['src'])
content = self._read_file(path)
files_combined += 1
for pattern, rep in src.get('replace', {}).items():
content = re.sub(pattern, rep, content)
final_content += '/* === {} === */\n{}\n'.format(path.name, content.strip('\n'))
progress_logger.debug(' appending %s', path.name)
if files_combined == 0:
main_logger.warning('no files found to form "%s"', dest)
continue
dest_path = self._dest_path(dest)
dest_path.relative_to(self.build_root)
self._write(dest_path, final_content)
total_files_combined += files_combined
progress_logger.info('%d files combined to form "%s"', files_combined, dest)
time_taken = (datetime.now() - start).total_seconds() * 1000
main_logger.info('%d files concatenated in %0.0fms', total_files_combined, time_taken)
def sass(self, data):
for dest, d in data.items():
if isinstance(d, str):
d = {'src': d}
src_path = self._file_path(d['src'])
dest_path = self._dest_path(dest)
sass_gen = SassGenerator(
input_dir=src_path,
output_dir=dest_path,
download_root=self.download_root,
include=d.get('include'),
exclude=d.get('exclude'),
replace=d.get('replace'),
debug=self.debug)
sass_gen()
def wipe(self, regexes):
if isinstance(regexes, str):
regexes = [regexes]
count = 0
regexes = [re.compile(r) for r in regexes]
for path in self.build_root.glob('**/*'):
relative_path = str(path.relative_to(self.build_root))
for regex in regexes:
if regex.fullmatch(relative_path):
if path.is_dir():
progress_logger.debug('deleting directory "%s" based on "%s"', relative_path, regex.pattern)
shutil.rmtree(str(path))
else:
assert path.is_file()
progress_logger.debug('deleting file "%s" on "%s"', relative_path, regex.pattern)
path.unlink()
count += 1
break
main_logger.info('%d paths deleted', count)
def _dest_path(self, p):
new_path = self.build_root.joinpath(p)
new_path.relative_to(self.build_root)
return new_path
def _file_path(self, src_path: str):
if STARTS_DOWNLOAD.match(src_path):
assert self.download_root
_src_path = STARTS_DOWNLOAD.sub('', src_path)
return self.download_root.joinpath(_src_path).resolve()
else:
return Path(src_path).resolve()
@property
def jsmin(self) -> Callable[[str, str], str]:
if self._jsmin is None:
try:
from jsmin import jsmin
except ImportError as e:
main_logger.error('ImportError importing jsmin: %s', e)
raise GrablibError(
'Error importing jsmin. Build requirements probably not installed, run `pip install grablib[build]`'
) from e
else:
self._jsmin = jsmin
return self._jsmin
def _read_file(self, file_path: Path):
content = file_path.read_text()
if not self.debug and file_path.name.endswith('.js') and not file_path.name.endswith('.min.js'):
return self.jsmin(content, quote_chars='\'"`')
return content
def _write(self, new_path: Path, data):
new_path.parent.mkdir(parents=True, exist_ok=True)
new_path.write_text(data)
class SassGenerator:
_errors = _files_generated = None
def __init__(self, *,
input_dir: Path,
output_dir: Path,
include: str=None,
exclude: str=None,
replace: dict=None,
download_root: Path,
debug: bool=False):
self._in_dir = input_dir
dir_hash = hashlib.md5(str(self._in_dir).encode()).hexdigest()
self._size_cache_file = Path(tempfile.gettempdir()) / 'grablib_cache.{}.json'.format(dir_hash)
assert self._in_dir.is_dir()
self._out_dir = output_dir
self._debug = debug
if self._debug:
self._out_dir_src = self._out_dir / '.src'
self._src_dir = self._out_dir_src
else:
self._src_dir = self._in_dir
self._include = re.compile(include or '/[^_][^/]+\.(?:css|sass|scss)$')
self._exclude = exclude and re.compile(exclude)
self._replace = replace or {}
self.download_root = download_root
self._nm = self._find_node_modules()
self._old_size_cache = {}
self._new_size_cache = {}
def __call__(self):
start = datetime.now()
self._errors, self._files_generated = 0, 0
if self._debug:
self._out_dir.mkdir(parents=True, exist_ok=True)
if self._out_dir_src.exists():
raise GrablibError('With debug switched on the directory "{}" must not exist before building, '
'you should delete it with the "wipe" option.'.format(self._out_dir_src))
shutil.copytree(str(self._in_dir.resolve()), str(self._out_dir_src))
if self._size_cache_file.exists():
with self._size_cache_file.open() as f:
self._old_size_cache = json.load(f)
self.process_directory(self._src_dir)
with self._size_cache_file.open('w') as f:
json.dump(self._new_size_cache, f, indent=2)
time_taken = (datetime.now() - start).total_seconds() * 1000
if not self._errors:
main_logger.info('%d css files generated in %0.0fms, 0 errors', self._files_generated, time_taken)
else:
main_logger.error('%d css files generated in %0.0fms, %d errors',
self._files_generated, time_taken, self._errors)
raise GrablibError('sass errors')
def process_directory(self, d: Path):
assert d.is_dir()
for p in d.iterdir():
if p.is_dir():
self.process_directory(p)
else:
assert p.is_file()
self.process_file(p)
def process_file(self, f: Path):
if not self._include.search(str(f)):
return
if self._exclude and self._exclude.search(str(f)):
return
rel_path = f.relative_to(self._src_dir)
css_path = (self._out_dir / rel_path).with_suffix('.css')
map_path = None
if self._debug:
map_path = css_path.with_suffix('.map')
css = self.generate_css(f, map_path)
if css is None:
return
log_msg = None
try:
css_path.parent.mkdir(parents=True, exist_ok=True)
if self._debug:
css, css_map = css
# correct the link to map file in css
css = re.sub(r'/\*# sourceMappingURL=\S+ \*/', '/*# sourceMappingURL={} */'.format(map_path.name), css)
map_path.write_text(css_map)
css, log_msg = self._regex_modify(rel_path, css)
finally:
self._log_file_creation(rel_path, css_path, css)
if log_msg:
progress_logger.debug(log_msg)
css_path.write_text(css)
self._files_generated += 1
def generate_css(self, f: Path, map_path):
output_style = 'nested' if self._debug else 'compressed'
sass = self.get_sass()
try:
return sass.compile(
filename=str(f),
source_map_filename=map_path and str(map_path),
output_style=output_style,
precision=10,
importers=[(0, self._clever_imports)]
)
except sass.CompileError as e:
self._errors += 1
main_logger.error('"%s", compile error: %s', f, e)
def _regex_modify(self, rel_path, css):
log_msg = None
for path_regex, regex_map in self._replace.items():
if re.search(path_regex, str(rel_path)):
progress_logger.debug('%s has regex replace matches for "%s"', rel_path, path_regex)
for pattern, repl in regex_map.items():
hash1 = hash(css)
css = re.sub(pattern, repl, css)
if hash(css) == hash1:
log_msg = ' "{}" ➤ "{}" didn\'t modify the source'.format(pattern, repl)
else:
log_msg = ' "{}" ➤ "{}" modified the source'.format(pattern, repl)
return css, log_msg
def _log_file_creation(self, rel_path, css_path, css):
src, dst = str(rel_path), str(css_path.relative_to(self._out_dir))
size = len(css)
p = str(css_path)
self._new_size_cache[p] = size
old_size = self._old_size_cache.get(p)
c = None
if old_size:
change_p = (size - old_size) / old_size * 100
if abs(change_p) > 0.5:
c = 'green' if change_p <= 0 else 'red'
change_p = click.style('{:+0.0f}%'.format(change_p), fg=c)
progress_logger.info('%30s ➤ %-30s %7s %s', src, dst, fmt_size(size), change_p)
if c is None:
progress_logger.info('%30s ➤ %-30s %7s', src, dst, fmt_size(size))
def _clever_imports(self, src_path):
_new_path = None
if STARTS_SRC.match(src_path):
_new_path = self._in_dir.joinpath(STARTS_SRC.sub('', src_path))
elif self._nm and STARTS_NODE_M.match(src_path):
_new_path = self._nm.joinpath(STARTS_NODE_M.sub('', src_path))
elif self.download_root and STARTS_DOWNLOAD.match(src_path):
_new_path = self.download_root.joinpath(STARTS_DOWNLOAD.sub('', src_path))
return _new_path and [(str(_new_path),)]
def _find_node_modules(self):
for d in self._in_dir.parents:
nm = d / 'node_modules'
if nm.is_dir():
return nm
def get_sass(self):
try:
import sass
except ImportError as e:
main_logger.error('ImportError importing sass: %s', e)
raise GrablibError(
'Error importing sass. Build requirements probably not installed, run `pip install grablib[build]`'
) from e
return sass
KB, MB = 1024, 1024 ** 2
def fmt_size(num):
if num <= KB:
return '{:0.0f}B'.format(num)
elif num <= MB:
return '{:0.1f}KB'.format(num / KB)
else:
return '{:0.1f}MB'.format(num / MB)