-
Notifications
You must be signed in to change notification settings - Fork 1
/
build.py
321 lines (259 loc) · 10.2 KB
/
build.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
from __future__ import print_function
import os
import shutil
import subprocess
import pkg_resources
import tarfile
import functools
import contextlib
import socket
from six.moves import urllib
import yaml
import path
from more_itertools import always_iterable
from vr.common.utils import tmpdir, mkdir, file_md5, chowntree
from vr.builder.models import (BuildPack, update_buildpack, update_app,
lock_or_wait, CACHE_HOME)
from vr.common.models import ProcData
from vr.common.paths import get_container_path
from vr.builder.slugignore import clean_slug_dir
from .py31compat import _defrag
from .hashes import hash_text
pkg_filename = functools.partial(pkg_resources.resource_filename, 'vr.builder')
class NullSaver(object):
def save_compile_log(self, app_folder):
pass
def save_lxcdebug_log(self, app_folder):
pass
def make_tarball(self, app_folder, build_data):
pass
class OutputSaver(object):
def __init__(self):
self.outfolder = os.getcwd()
def _save_logfile(self, app_folder, srcname, dstname):
srclog = os.path.join(app_folder, srcname)
if os.path.isfile(srclog):
dstlog = os.path.join(self.outfolder, dstname)
shutil.copyfile(srclog, dstlog)
print('copied %r to %r' % (srclog, dstlog))
else:
print("No file at %s" % srclog)
def save_compile_log(self, app_folder):
"Copy compilation log into outfolder"
self._save_logfile(app_folder, '.compile.log', 'compile.log')
def save_lxcdebug_log(self, app_folder):
"Copy lxc debug log into outfolder"
self._save_logfile(app_folder, '.lxcdebug.log', 'lxcdebug.log')
def make_tarball(self, app_folder, build_data):
"""
Following a successful build, create a tarball and build result.
"""
# slugignore
clean_slug_dir(app_folder)
# tar up the result
with tarfile.open('build.tar.gz', 'w:gz') as tar:
tar.add(app_folder, arcname='')
build_data.build_md5 = file_md5('build.tar.gz')
tardest = os.path.join(self.outfolder, 'build.tar.gz')
shutil.move('build.tar.gz', tardest)
build_data_path = os.path.join(self.outfolder, 'build_result.yaml')
print("Writing", build_data_path)
with open(build_data_path, 'w') as f:
f.write(build_data.as_yaml())
def cmd_build(build_data, runner_cmd='run', make_tarball=True):
# runner_cmd may be 'run' or 'shell'.
saver = OutputSaver() if make_tarball else NullSaver()
with tmpdir():
app_folder = _cmd_build(build_data, runner_cmd, saver)
saver.make_tarball(app_folder, build_data)
def _cmd_build(build_data, runner_cmd, saver):
print("Building on", socket.getfqdn())
here = path.Path.getcwd()
user = getattr(build_data, 'user', 'nobody')
# clone/pull repo to latest
build_folder = here / 'build'
mkdir(build_folder)
app_folder = pull_app(build_folder,
build_data.app_name,
build_data.app_repo_url,
build_data.version,
vcs_type=build_data.app_repo_type)
app_basename = os.path.basename(app_folder)
chowntree(build_folder, username=user)
app_folder_inside = os.path.join('/build', app_basename)
def _volume(name):
"Return a volume mount mapping of a named folder into the root"
return [str(here / name), '/' + name]
volumes = [
_volume('build')
]
buildpack_url = getattr(build_data, 'buildpack_url', None)
buildpack_urls = always_iterable(
buildpack_url or build_data.buildpack_urls)
buildpack_folders = pull_buildpacks(buildpack_urls)
buildpacks_env = ':'.join('/' + bp for bp in buildpack_folders)
env_key = 'BUILDPACK_DIR' if buildpack_url else 'BUILDPACK_DIRS'
env = {env_key: buildpacks_env}
volumes.extend(
_volume(folder)
for folder in buildpack_folders
)
# Some buildpacks (Node) like to rm -rf the whole cache folder they're
# given. They can't do that to a mountpoint, so we have to provide a
# buildpack_cache folder nested inside the /cache mountpoint.
cachefolder = os.path.join(CACHE_HOME, app_basename)
if os.path.isdir(cachefolder):
with lock_or_wait(cachefolder):
mkdir('cache')
shutil.copytree(
cachefolder, 'cache/buildpack_cache', symlinks=True)
else:
mkdir('cache/buildpack_cache')
# Maybe we're on a brand new host that's never had CACHE_HOME
# created. Ensure that now.
mkdir(CACHE_HOME)
chowntree('cache', username=user)
volumes.append(_volume('cache'))
cmd = '/builder.sh %s /cache/buildpack_cache' % app_folder_inside
container_path = _write_buildproc_yaml(
build_data, env, user, cmd, volumes, app_folder)
runner = 'vrun' if build_data.image_url else 'vrun_precise'
def run(run_cmd):
cmd = runner, run_cmd, 'buildproc.yaml'
return subprocess.check_call(cmd, stderr=subprocess.STDOUT)
try:
with _setup_container(run):
with _prepare_build(container_path, user, build_data, app_folder):
run(runner_cmd)
assert_compile_finished(app_folder)
except BaseException:
saver.save_lxcdebug_log(app_folder)
raise
finally:
saver.save_compile_log(app_folder)
with lock_or_wait(cachefolder):
shutil.rmtree(cachefolder, ignore_errors=True)
shutil.move('cache/buildpack_cache', cachefolder)
return app_folder
@contextlib.contextmanager
def _setup_container(run):
try:
run('setup')
yield
finally:
run('teardown')
@contextlib.contextmanager
def _prepare_build(container_path, user, build_data, app_folder):
# copy the builder.sh script into place.
script_src = pkg_filename('scripts/builder.sh')
script_dst = path.Path(container_path) / 'builder.sh'
shutil.copy(script_src, script_dst)
# Make sure builder.sh is chmod a+x
script_dst.chmod('a+x')
# make /app/vendor
slash_app = os.path.join(container_path, 'app')
mkdir(os.path.join(slash_app, 'vendor'))
chowntree(slash_app, username=user)
yield
build_data.release_data = recover_release_data(app_folder)
bp = recover_buildpack(app_folder)
build_data.buildpack_url = bp.url + '#' + bp.version
build_data.buildpack_version = bp.version
def _write_buildproc_yaml(build_data, env, user, cmd, volumes, app_folder):
"""
Write a proc.yaml for the container and return the container path
"""
buildproc = ProcData({
'app_folder': str(app_folder),
'app_name': build_data.app_name,
'app_repo_url': '',
'app_repo_type': '',
'buildpack_url': '',
'buildpack_version': '',
'config_name': 'build',
'env': env,
'host': '',
'port': 0,
'version': build_data.version,
'release_hash': '',
'settings': {},
'user': user,
'cmd': cmd,
'volumes': volumes,
'proc_name': 'build',
'image_name': build_data.image_name,
'image_url': build_data.image_url,
'image_md5': build_data.image_md5,
})
# write a proc.yaml for the container.
with open('buildproc.yaml', 'w') as f:
f.write(buildproc.as_yaml())
return get_container_path(buildproc)
def assert_compile_finished(app_folder):
"""
Once builder.sh has invoked the compile script, it should return and we
should set a flag to the script returned. If that flag is missing, then
it is an indication that the container crashed, and we generate an error.
This function will clean up the flag after the check is performed, so only
call this function once. See issue #141.
"""
fpath = os.path.join(app_folder, '.postbuild.flag')
if not os.path.isfile(fpath):
msg = ('No postbuild flag set, LXC container may have crashed while '
'building. Check compile logs for build.')
raise AssertionError(msg)
try:
os.remove(fpath)
except OSError:
# It doesn't matter if it fails.
pass
def recover_release_data(app_folder):
"""
Given the path to an app folder where an app was just built, return a
dictionary containing the data emitted from running the buildpack's release
script.
Relies on the builder.sh script storing the release data in ./.release.yaml
inside the app folder.
"""
with open(os.path.join(app_folder, '.release.yaml'), 'rb') as f:
return yaml.safe_load(f)
def recover_buildpack(app_folder):
"""
Given the path to an app folder where an app was just built, return a
BuildPack object pointing to the dir for the buildpack used during the
build.
Relies on the builder.sh script storing the buildpack location in
/.buildpack inside the container.
"""
filepath = os.path.join(app_folder, '.buildpack')
with open(filepath) as f:
buildpack_picked = f.read()
buildpack_picked = buildpack_picked.lstrip('/')
buildpack_picked = buildpack_picked.rstrip('\n')
buildpack_picked = os.path.join(os.getcwd(), buildpack_picked)
return BuildPack(buildpack_picked)
def pull_app(parent_folder, name, url, version, vcs_type):
defrag = _defrag(urllib.parse.urldefrag(url))
with lock_or_wait(defrag.url):
app = update_app(name, url, version, vcs_type=vcs_type)
dest_name = name + '-' + hash_text(defrag.url)
dest = os.path.join(parent_folder, dest_name)
# copy symlinks instead of their contents
shutil.copytree(app.folder, dest, symlinks=True)
return dest
def pull_buildpack(url):
"""
Update a buildpack in its shared location, then make a copy into the
current directory, using an md5 of the url.
"""
defrag = _defrag(urllib.parse.urldefrag(url))
with lock_or_wait(defrag.url):
bp = update_buildpack(url)
dest = bp.basename + '-' + hash_text(defrag.url)
shutil.copytree(bp.folder, dest)
# Make the buildpack dir writable, per
# https://bitbucket.org/yougov/velociraptor/issues/178
path.Path(dest).chmod('a+wx')
return dest
def pull_buildpacks(urls):
return [pull_buildpack(u) for u in urls]