From 22434ed08dd956c3bbd91d1222954fecc02a58a8 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 14:45:18 -0500 Subject: [PATCH 001/101] Create a python 2 virtualenv and install bbfreeze. --- .circleci/config.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2b67684..ea66a98 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,7 +22,11 @@ jobs: python3 -m venv .env . .env/bin/activate pip install -r development-requirements.txt - + deactivate + pip install virtualenv + virtualenv -p /usr/bin/python2 .env-2 + . .env-2/bin/activate + pip install bbfreeze - save_cache: paths: - ./.env @@ -40,6 +44,10 @@ jobs: command: | python setup.py sdist bdist_wheel + - run: + name: Build Self-Hosted Bundle + . .env-2/bin/activate + python setup.py bdist_bbfreeze - store_artifacts: path: htmlcov destination: coverage-report From 90dae078e9ea9420942f0e0be861ef5c582815d5 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 14:48:47 -0500 Subject: [PATCH 002/101] Deactivate the python 3 virtualenv after creating the python 2 one. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ea66a98..ffee259 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,9 +22,9 @@ jobs: python3 -m venv .env . .env/bin/activate pip install -r development-requirements.txt - deactivate pip install virtualenv virtualenv -p /usr/bin/python2 .env-2 + deactivate . .env-2/bin/activate pip install bbfreeze - save_cache: From bee67c56d0f0ee36782e2b2821c1e702da8ec2c2 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 15:10:08 -0500 Subject: [PATCH 003/101] Add a first pass of building with bbfreeze. --- .circleci/config.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ffee259..00f46e6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,6 +26,7 @@ jobs: virtualenv -p /usr/bin/python2 .env-2 deactivate . .env-2/bin/activate + sudo apt-get install patchelf python-dev pip install bbfreeze - save_cache: paths: @@ -47,7 +48,11 @@ jobs: - run: name: Build Self-Hosted Bundle . .env-2/bin/activate - python setup.py bdist_bbfreeze + pip install . + mv .env-2/lib/python2.7/site-packages/exodus_bundler/cli.py .env-2/lib/python2.7/site-packages/exodus_bundler/exodus.py + echo "main()" >> .env-2/lib/python2.7/site-packages/exodus_bundler/exodus.py + bbfreeze .env-2/lib/python2.7/site-packages/exodus_bundler/exodus.py + cp -r src/exodus_bundler/templates/ dist/ - store_artifacts: path: htmlcov destination: coverage-report From ff5fdfe49fa04151bbafdf98c80e9cb9534c9518 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 15:10:22 -0500 Subject: [PATCH 004/101] Add a hack to find the template directory when using bbfreeze. --- src/exodus_bundler/templating.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/exodus_bundler/templating.py b/src/exodus_bundler/templating.py index 161533d..8847cbb 100644 --- a/src/exodus_bundler/templating.py +++ b/src/exodus_bundler/templating.py @@ -8,6 +8,10 @@ parent_directory = os.path.dirname(os.path.realpath(__file__)) template_directory = os.path.join(parent_directory, 'templates') +# This is kind of a hack to make the templates accessible using bbfreeze. +if os.path.dirname(parent_directory) == 'library.zip' and not os.path.exists(template_directory): + template_directory = os.normpath(os.path.join(parent_directory, '..', '..', 'templates')) + def render_template(string, **context): for key, value in context.items(): From b970c59705b76ab3885a96bc792d1e867455ef27 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 15:12:02 -0500 Subject: [PATCH 005/101] Add missing command key. --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 00f46e6..0133c8f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,6 +47,7 @@ jobs: - run: name: Build Self-Hosted Bundle + command: | . .env-2/bin/activate pip install . mv .env-2/lib/python2.7/site-packages/exodus_bundler/cli.py .env-2/lib/python2.7/site-packages/exodus_bundler/exodus.py From cf6a7a55bab63e90de0737a5156312c124144465 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 15:17:26 -0500 Subject: [PATCH 006/101] Fix the bbfreeze template hack. --- src/exodus_bundler/templating.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/exodus_bundler/templating.py b/src/exodus_bundler/templating.py index 8847cbb..76d1b84 100644 --- a/src/exodus_bundler/templating.py +++ b/src/exodus_bundler/templating.py @@ -8,9 +8,11 @@ parent_directory = os.path.dirname(os.path.realpath(__file__)) template_directory = os.path.join(parent_directory, 'templates') + # This is kind of a hack to make the templates accessible using bbfreeze. -if os.path.dirname(parent_directory) == 'library.zip' and not os.path.exists(template_directory): - template_directory = os.normpath(os.path.join(parent_directory, '..', '..', 'templates')) +grandparent_basename = os.path.basename(os.path.dirname(parent_directory)) +if grandparent_basename == 'library.zip' and not os.path.exists(template_directory): + template_directory = os.path.normpath(os.path.join(parent_directory, '..', '..', 'templates')) def render_template(string, **context): From 8cfe20a87ce0d31b5fdda39cc9172cfc7e4e19b1 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 15:23:50 -0500 Subject: [PATCH 007/101] Remove the for loop initial declaration in the c launcher. --- src/exodus_bundler/templates/launcher.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/exodus_bundler/templates/launcher.c b/src/exodus_bundler/templates/launcher.c index 10b849d..c5387f5 100644 --- a/src/exodus_bundler/templates/launcher.c +++ b/src/exodus_bundler/templates/launcher.c @@ -19,14 +19,15 @@ int main(int argc, char *argv[]) { // Prefix each segment with the current working directory so it's an absolute path. int library_segments = 1; - for (int i = 0; original_library_path[i]; i++) { + int i; + for (i = 0; original_library_path[i]; i++) { library_segments += (original_library_path[i] == ':'); } char *library_path = malloc( (strlen(original_library_path) + library_segments * strlen(current_directory) + 1) * sizeof(char)); strcpy(library_path, current_directory); int character_offset = current_directory_length; - for (int i = 0; original_library_path[i]; i++) { + for (i = 0; original_library_path[i]; i++) { library_path[character_offset] = original_library_path[i]; character_offset++; if (original_library_path[i] == ':') { From e62e243903ab62448ad885a95128326e765b9c56 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 15:33:21 -0500 Subject: [PATCH 008/101] Exclude temporary exodus bundle directories when parsing strace output. --- src/exodus_bundler/input_parsing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/exodus_bundler/input_parsing.py b/src/exodus_bundler/input_parsing.py index 18c73d9..7ffb63a 100644 --- a/src/exodus_bundler/input_parsing.py +++ b/src/exodus_bundler/input_parsing.py @@ -4,6 +4,7 @@ '/proc/', '/run/', '/sys/', + '/tmp/exodus-bundle-' ] exec_methods = [ From 96ff234d2313338705a7802f274744de9d50edb9 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 15:35:21 -0500 Subject: [PATCH 009/101] Add musl and gcc as dependencies. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0133c8f..5941a8d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,7 +26,7 @@ jobs: virtualenv -p /usr/bin/python2 .env-2 deactivate . .env-2/bin/activate - sudo apt-get install patchelf python-dev + sudo apt-get install gcc musl musl-tools patchelf python-dev pip install bbfreeze - save_cache: paths: From d4c000ba0a4ffc339a21861603adab3a7a459f2a Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 15:35:38 -0500 Subject: [PATCH 010/101] Create the final bundles using exodus. --- .circleci/config.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5941a8d..533a192 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,9 +54,17 @@ jobs: echo "main()" >> .env-2/lib/python2.7/site-packages/exodus_bundler/exodus.py bbfreeze .env-2/lib/python2.7/site-packages/exodus_bundler/exodus.py cp -r src/exodus_bundler/templates/ dist/ + strace ./dist/exodus -q ls 2>&1 | exodus ./dist/exodus --output exodus-x64.sh + strace ./dist/exodus -q ls 2>&1 | exodus ./dist/exodus --tar --output exodus-x64.tgz - store_artifacts: path: htmlcov destination: coverage-report + - store_artifacts: + path: exodus-x64.sh + destination: exodus-x64.sh + - store_artifacts: + path: exodus-x64.tgz + destination: exodus-x64.tgz - persist_to_workspace: root: ~/exodus From 0566551175ec6e18d00c6bc8e3e48ca5a6a043d7 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 15:36:02 -0500 Subject: [PATCH 011/101] Add strace as a dependency. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 533a192..1561b21 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,7 +26,7 @@ jobs: virtualenv -p /usr/bin/python2 .env-2 deactivate . .env-2/bin/activate - sudo apt-get install gcc musl musl-tools patchelf python-dev + sudo apt-get install gcc musl musl-tools patchelf python-dev strace pip install bbfreeze - save_cache: paths: From 9dfb33736534f5229efcf276e95654a42ffacf6b Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 15:39:20 -0500 Subject: [PATCH 012/101] Remove the for loop initial declaration in fizz-buzz. --- tests/data/binaries/fizz-buzz.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/data/binaries/fizz-buzz.c b/tests/data/binaries/fizz-buzz.c index b8224db..37521cb 100644 --- a/tests/data/binaries/fizz-buzz.c +++ b/tests/data/binaries/fizz-buzz.c @@ -2,7 +2,8 @@ int main() { - for(int i = 0; i <= 100; i++) { + int i; + for(i = 0; i <= 100; i++) { if (i % 3 == 0) { printf("FIZZ"); } From 0363903b5f0ae673c586dd3eed6942f2a2fd886f Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 15:43:16 -0500 Subject: [PATCH 013/101] Add a trailing comma. --- src/exodus_bundler/input_parsing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/exodus_bundler/input_parsing.py b/src/exodus_bundler/input_parsing.py index 7ffb63a..7792909 100644 --- a/src/exodus_bundler/input_parsing.py +++ b/src/exodus_bundler/input_parsing.py @@ -4,7 +4,7 @@ '/proc/', '/run/', '/sys/', - '/tmp/exodus-bundle-' + '/tmp/exodus-bundle-', ] exec_methods = [ From dd5f6d820bc78ffcbfc3a568bff24f4379844336 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 15:43:43 -0500 Subject: [PATCH 014/101] Use `gcc` instead of `ls` to produce the `strace` output for bundling. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1561b21..89d5bc6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,8 +54,8 @@ jobs: echo "main()" >> .env-2/lib/python2.7/site-packages/exodus_bundler/exodus.py bbfreeze .env-2/lib/python2.7/site-packages/exodus_bundler/exodus.py cp -r src/exodus_bundler/templates/ dist/ - strace ./dist/exodus -q ls 2>&1 | exodus ./dist/exodus --output exodus-x64.sh - strace ./dist/exodus -q ls 2>&1 | exodus ./dist/exodus --tar --output exodus-x64.tgz + strace ./dist/exodus -q gcc 2>&1 | exodus ./dist/exodus --output exodus-x64.sh + strace ./dist/exodus -q gcc 2>&1 | exodus ./dist/exodus --tar --output exodus-x64.tgz - store_artifacts: path: htmlcov destination: coverage-report From 12ff86a32f491a8ecc7f2ea26578d774f7a2cacc Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 15:49:36 -0500 Subject: [PATCH 015/101] Add an `exodus-bundle-` prefix to the temporary launcher filename. --- src/exodus_bundler/input_parsing.py | 1 + src/exodus_bundler/launchers.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/exodus_bundler/input_parsing.py b/src/exodus_bundler/input_parsing.py index 7792909..29d9dd2 100644 --- a/src/exodus_bundler/input_parsing.py +++ b/src/exodus_bundler/input_parsing.py @@ -4,6 +4,7 @@ '/proc/', '/run/', '/sys/', + # This isn't a directory exactly, but it will filter out active bundling. '/tmp/exodus-bundle-', ] diff --git a/src/exodus_bundler/launchers.py b/src/exodus_bundler/launchers.py index ec12141..13c14ae 100644 --- a/src/exodus_bundler/launchers.py +++ b/src/exodus_bundler/launchers.py @@ -32,7 +32,7 @@ def compile_diet(code): def compile_helper(code, initial_args): - f, input_filename = tempfile.mkstemp(suffix='.c') + f, input_filename = tempfile.mkstemp(prefix='exodus-bundle-', suffix='.c') os.close(f) f, output_filename = tempfile.mkstemp() os.close(f) From 024aa1ceaee5c6248fe2605cbe8eb5881c3c695f Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 15:53:00 -0500 Subject: [PATCH 016/101] Use the `exodus-bundle-` prefix for the build output too. --- src/exodus_bundler/launchers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/exodus_bundler/launchers.py b/src/exodus_bundler/launchers.py index 13c14ae..5bdf376 100644 --- a/src/exodus_bundler/launchers.py +++ b/src/exodus_bundler/launchers.py @@ -34,7 +34,7 @@ def compile_diet(code): def compile_helper(code, initial_args): f, input_filename = tempfile.mkstemp(prefix='exodus-bundle-', suffix='.c') os.close(f) - f, output_filename = tempfile.mkstemp() + f, output_filename = tempfile.mkstemp(prefix='exodus-bundle-') os.close(f) try: with open(input_filename, 'w') as input_file: From 816502c2e9073c5ce3c4d3564ea654e4c14446cf Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 16:21:00 -0500 Subject: [PATCH 017/101] Exclude the bbfreeze library.zip file from having a launcher. --- src/exodus_bundler/bundling.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index e7c2231..d392883 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -606,6 +606,10 @@ def requires_launcher(self): if in_lib_directory and not in_bin_directory: return False + # This is a bbfreeze related hack. + if os.path.basename(self.path) == 'library.zip': + return False + # Most libraries will include `.so` in the filename. return re.match('\.so(?:\.|$)') From 9f08327db0679d1af34e7358ee97f4909bf453f7 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 16:25:34 -0500 Subject: [PATCH 018/101] Fix the bbfreeze library.zip hack. --- src/exodus_bundler/bundling.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index 6ca7500..b3adeb1 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -588,6 +588,11 @@ def requires_launcher(self): # This is unfortunately a heuristic approach because many executables are compiled # as shared libraries, and many mostly-libraries are executable (*e.g.* glibc). + # This is a bbfreeze related hack. + # TODO: There should be a command-line argument to override the heuristics. + if self.path == '/home/circleci/exodus/dist/library.zip': + return False + # The easy ones. if self.library or not self.elf or not self.elf.linker_file or not self.executable: return False @@ -606,10 +611,6 @@ def requires_launcher(self): if in_lib_directory and not in_bin_directory: return False - # This is a bbfreeze related hack. - if os.path.basename(self.path) == 'library.zip': - return False - # Most libraries will include `.so` in the filename. return re.search('\.so(?:\.|$)', self.path) From 07a4b5ce5bacb5eac21825e51bd7dc6defabba3d Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 16:40:14 -0500 Subject: [PATCH 019/101] Remove the bbfreeze hacks. --- src/exodus_bundler/bundling.py | 5 ----- src/exodus_bundler/templating.py | 6 ------ 2 files changed, 11 deletions(-) diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index b3adeb1..db829c1 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -588,11 +588,6 @@ def requires_launcher(self): # This is unfortunately a heuristic approach because many executables are compiled # as shared libraries, and many mostly-libraries are executable (*e.g.* glibc). - # This is a bbfreeze related hack. - # TODO: There should be a command-line argument to override the heuristics. - if self.path == '/home/circleci/exodus/dist/library.zip': - return False - # The easy ones. if self.library or not self.elf or not self.elf.linker_file or not self.executable: return False diff --git a/src/exodus_bundler/templating.py b/src/exodus_bundler/templating.py index 76d1b84..161533d 100644 --- a/src/exodus_bundler/templating.py +++ b/src/exodus_bundler/templating.py @@ -9,12 +9,6 @@ template_directory = os.path.join(parent_directory, 'templates') -# This is kind of a hack to make the templates accessible using bbfreeze. -grandparent_basename = os.path.basename(os.path.dirname(parent_directory)) -if grandparent_basename == 'library.zip' and not os.path.exists(template_directory): - template_directory = os.path.normpath(os.path.join(parent_directory, '..', '..', 'templates')) - - def render_template(string, **context): for key, value in context.items(): string = string.replace('{{%s}}' % key, value) From e619dff0114cbb29f16e614b7b2a4d66d66e947b Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 16:40:52 -0500 Subject: [PATCH 020/101] Remove the bbfreeze stuff from the CircleCI config. --- .circleci/config.yml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 89d5bc6..2df7441 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,12 +22,6 @@ jobs: python3 -m venv .env . .env/bin/activate pip install -r development-requirements.txt - pip install virtualenv - virtualenv -p /usr/bin/python2 .env-2 - deactivate - . .env-2/bin/activate - sudo apt-get install gcc musl musl-tools patchelf python-dev strace - pip install bbfreeze - save_cache: paths: - ./.env @@ -50,12 +44,7 @@ jobs: command: | . .env-2/bin/activate pip install . - mv .env-2/lib/python2.7/site-packages/exodus_bundler/cli.py .env-2/lib/python2.7/site-packages/exodus_bundler/exodus.py - echo "main()" >> .env-2/lib/python2.7/site-packages/exodus_bundler/exodus.py - bbfreeze .env-2/lib/python2.7/site-packages/exodus_bundler/exodus.py - cp -r src/exodus_bundler/templates/ dist/ - strace ./dist/exodus -q gcc 2>&1 | exodus ./dist/exodus --output exodus-x64.sh - strace ./dist/exodus -q gcc 2>&1 | exodus ./dist/exodus --tar --output exodus-x64.tgz + strace exodus -q gcc 2>&1 | exodus exodus --tar --output exodus-x64.tgz - store_artifacts: path: htmlcov destination: coverage-report From 85f13b0a690721daeb8244b1d870ea6193f24f7e Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 16:41:22 -0500 Subject: [PATCH 021/101] Fix: make sure that the directory exists before copying. --- src/exodus_bundler/bundling.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index db829c1..5b86d6f 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -682,6 +682,9 @@ def create_bundle(self): if not file.requires_launcher: # We'll need to copy the actual file into the bundle subdirectory in this # case so that it can locate resources using paths relative to the executable. + parent_directory = os.path.dirname(file_path) + if not os.path.exists(parent_directory): + os.makedirs(parent_directory) shutil.copy(file.path, file_path) continue From 31c41bed1ad8afbc262ad30def63ef9cc6279f8b Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 16:43:20 -0500 Subject: [PATCH 022/101] Add back the installation of the extra dependencies. --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2df7441..f34f0e6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,6 +22,7 @@ jobs: python3 -m venv .env . .env/bin/activate pip install -r development-requirements.txt + sudo apt-get install gcc musl musl-tools patchelf python-dev strace - save_cache: paths: - ./.env From 83831ec9ab71023ade94de58876c95a17fc2f25c Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 16:45:44 -0500 Subject: [PATCH 023/101] Add in all of the extra dependencies to the exodus bundle. --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f34f0e6..986a90b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,7 +45,8 @@ jobs: command: | . .env-2/bin/activate pip install . - strace exodus -q gcc 2>&1 | exodus exodus --tar --output exodus-x64.tgz + strace exodus -q gcc 2>&1 | exodus exodus --add .env/lib/python*/site-packages/exodus_bundler/ --tar --output exodus-x64.tgz + strace exodus -q gcc 2>&1 | exodus exodus --add .env/lib/python*/site-packages/exodus_bundler/ --output exodus-x64.sh - store_artifacts: path: htmlcov destination: coverage-report From f9040f3dfc1c6a5b5e8cf8ffaf2d276e61882fff Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 16:46:19 -0500 Subject: [PATCH 024/101] Activate the python 3 virtualenv instead of the old python 2 one. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 986a90b..20c5251 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,7 +43,7 @@ jobs: - run: name: Build Self-Hosted Bundle command: | - . .env-2/bin/activate + . .env/bin/activate pip install . strace exodus -q gcc 2>&1 | exodus exodus --add .env/lib/python*/site-packages/exodus_bundler/ --tar --output exodus-x64.tgz strace exodus -q gcc 2>&1 | exodus exodus --add .env/lib/python*/site-packages/exodus_bundler/ --output exodus-x64.sh From 517bc7c0ce89d3e4546de86fc53eb3521d5827ca Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 17:11:19 -0500 Subject: [PATCH 025/101] Construct a small C launcher to use as an entrypoint for exodus. --- .circleci/config.yml | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 20c5251..662c0c0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,7 +45,44 @@ jobs: command: | . .env/bin/activate pip install . - strace exodus -q gcc 2>&1 | exodus exodus --add .env/lib/python*/site-packages/exodus_bundler/ --tar --output exodus-x64.tgz + (cat < ./exodus.c + #include + #include + #include + #include + + int main(int argc, char *argv[]) { + char *python = "./.env/bin/python"; + char *exodus = "./.env/bin/exodus"; + + char buffer[4096] = { 0 }; + if (readlink("/proc/self/exe", buffer, sizeof(buffer) - 1)) { + char *current_directory = dirname(buffer); + int current_directory_length = strlen(current_directory); + + char python[4096] = { 0 }; + strcpy(python, current_directory); + strcat(python, "/.env/bin/python"); + + char exodus[4096] = { 0 }; + strcpy(exodus, current_directory); + strcat(exodus, "/.env/bin/exodus"); + + char **combined_args = malloc(sizeof(char*) * (argc + 2)); + combined_args[0] = python; + combined_args[1] = exodus; + memcpy(combined_args + 2, argv + 1, sizeof(char*) * (argc - 1)); + combined_args[argc + 1] = NULL; + + execv(python, combined_args); + } + + return 1; + } + EOF + ) + musl-gcc -static -O3 exodus.c -o exodus + strace ./exodus -q gcc 2>&1 | ./exodus ./exodus --add .env/lib/python*/site-packages/exodus_bundler/ --tar --output exodus-x64.tgz strace exodus -q gcc 2>&1 | exodus exodus --add .env/lib/python*/site-packages/exodus_bundler/ --output exodus-x64.sh - store_artifacts: path: htmlcov From 7f282705923047451f9955d1c31e2c60b12ac2c0 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 17:29:42 -0500 Subject: [PATCH 026/101] Avoid buffer overflows. --- .circleci/config.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 662c0c0..8df1ff8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,11 +52,8 @@ jobs: #include int main(int argc, char *argv[]) { - char *python = "./.env/bin/python"; - char *exodus = "./.env/bin/exodus"; - char buffer[4096] = { 0 }; - if (readlink("/proc/self/exe", buffer, sizeof(buffer) - 1)) { + if (readlink("/proc/self/exe", buffer, sizeof(buffer) - 25)) { char *current_directory = dirname(buffer); int current_directory_length = strlen(current_directory); From cf67340073d752c6634bac74759276d26128e1e8 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 17:30:26 -0500 Subject: [PATCH 027/101] Set LD_LIBRARY_PATH before bundling exodus. --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8df1ff8..c166789 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,6 +79,7 @@ jobs: EOF ) musl-gcc -static -O3 exodus.c -o exodus + export LD_LIBRARY_PATH=/usr/local/lib/:${LD_LIBRARY_PATH} strace ./exodus -q gcc 2>&1 | ./exodus ./exodus --add .env/lib/python*/site-packages/exodus_bundler/ --tar --output exodus-x64.tgz strace exodus -q gcc 2>&1 | exodus exodus --add .env/lib/python*/site-packages/exodus_bundler/ --output exodus-x64.sh - store_artifacts: From 93c41d2c2f4fa5929a4c0944145e2a6ba9cdca81 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 17:34:11 -0500 Subject: [PATCH 028/101] Add `/lib/x86_64-linux-gnu/` to LD_LIBRARY_PATH. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c166789..377c211 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,7 +79,7 @@ jobs: EOF ) musl-gcc -static -O3 exodus.c -o exodus - export LD_LIBRARY_PATH=/usr/local/lib/:${LD_LIBRARY_PATH} + export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} strace ./exodus -q gcc 2>&1 | ./exodus ./exodus --add .env/lib/python*/site-packages/exodus_bundler/ --tar --output exodus-x64.tgz strace exodus -q gcc 2>&1 | exodus exodus --add .env/lib/python*/site-packages/exodus_bundler/ --output exodus-x64.sh - store_artifacts: From d5fcf028ae2a61d4963bab78562b1e58b39b17e2 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 17:44:08 -0500 Subject: [PATCH 029/101] Set the python home environment variable. --- .circleci/config.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 377c211..622fdcd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -71,7 +71,15 @@ jobs: memcpy(combined_args + 2, argv + 1, sizeof(char*) * (argc - 1)); combined_args[argc + 1] = NULL; - execv(python, combined_args); + char *envp[2]; + char pythonhome[4096] = { 0 } + strcpy(pythonhome, "PYTHONHOME="); + strcat(pythonhome, current_directory); + strcat(pythonhome, "/usr/local/lib/python3.6/"); + envp[0] = pythonhome; + envp[1] = NULL; + + execve(python, combined_args, envp); } return 1; From ffa24aceff723a65007586a28a4b271f22a57cfa Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 17:46:22 -0500 Subject: [PATCH 030/101] Add missing semicolon. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 622fdcd..32aad2e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -72,7 +72,7 @@ jobs: combined_args[argc + 1] = NULL; char *envp[2]; - char pythonhome[4096] = { 0 } + char pythonhome[4096] = { 0 }; strcpy(pythonhome, "PYTHONHOME="); strcat(pythonhome, current_directory); strcat(pythonhome, "/usr/local/lib/python3.6/"); From a917d415840f162c76b1e21ee3b6badfc52e2d57 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 17:51:56 -0500 Subject: [PATCH 031/101] Don't use the compiled exodus when running strace. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 32aad2e..7b589df 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -88,8 +88,8 @@ jobs: ) musl-gcc -static -O3 exodus.c -o exodus export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} - strace ./exodus -q gcc 2>&1 | ./exodus ./exodus --add .env/lib/python*/site-packages/exodus_bundler/ --tar --output exodus-x64.tgz - strace exodus -q gcc 2>&1 | exodus exodus --add .env/lib/python*/site-packages/exodus_bundler/ --output exodus-x64.sh + strace exodus -q gcc 2>&1 | exodus ./exodus --add .env/lib/python*/site-packages/exodus_bundler/ --tar --output exodus-x64.tgz + strace exodus -q gcc 2>&1 | exodus ./exodus --add .env/lib/python*/site-packages/exodus_bundler/ --output exodus-x64.sh - store_artifacts: path: htmlcov destination: coverage-report From 806f5b3d14624bdf3301d5776547ac525b90494c Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 18:01:46 -0500 Subject: [PATCH 032/101] Invoke python with exodus as an argument when calling strace. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7b589df..b55b45e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -88,8 +88,8 @@ jobs: ) musl-gcc -static -O3 exodus.c -o exodus export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} - strace exodus -q gcc 2>&1 | exodus ./exodus --add .env/lib/python*/site-packages/exodus_bundler/ --tar --output exodus-x64.tgz - strace exodus -q gcc 2>&1 | exodus ./exodus --add .env/lib/python*/site-packages/exodus_bundler/ --output exodus-x64.sh + strace python ./.env/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add .env/lib/python*/site-packages/exodus_bundler/ --tar --output exodus-x64.tgz + strace python ./.env/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add .env/lib/python*/site-packages/exodus_bundler/ --output exodus-x64.sh - store_artifacts: path: htmlcov destination: coverage-report From d95d51d0188cc640719c9c8978b2d2505793cdef Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 18:07:30 -0500 Subject: [PATCH 033/101] Set PYTHONHOME to the local virtualenv lib directory. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b55b45e..d65aeb6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -75,7 +75,7 @@ jobs: char pythonhome[4096] = { 0 }; strcpy(pythonhome, "PYTHONHOME="); strcat(pythonhome, current_directory); - strcat(pythonhome, "/usr/local/lib/python3.6/"); + strcat(pythonhome, "/.env/lib/python3.6/"); envp[0] = pythonhome; envp[1] = NULL; From eb7fae16778eb9c4b9f31466059dd4109eb08ea1 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 18:15:07 -0500 Subject: [PATCH 034/101] Bundle `exodus` from a system instead of a virtualenv install. --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d65aeb6..6d75a77 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,8 +43,8 @@ jobs: - run: name: Build Self-Hosted Bundle command: | - . .env/bin/activate - pip install . + sudo pip install . + cd / (cat < ./exodus.c #include #include @@ -59,11 +59,11 @@ jobs: char python[4096] = { 0 }; strcpy(python, current_directory); - strcat(python, "/.env/bin/python"); + strcat(python, "/usr/local/bin/python"); char exodus[4096] = { 0 }; strcpy(exodus, current_directory); - strcat(exodus, "/.env/bin/exodus"); + strcat(exodus, "/usr/local/bin/exodus"); char **combined_args = malloc(sizeof(char*) * (argc + 2)); combined_args[0] = python; @@ -75,7 +75,7 @@ jobs: char pythonhome[4096] = { 0 }; strcpy(pythonhome, "PYTHONHOME="); strcat(pythonhome, current_directory); - strcat(pythonhome, "/.env/lib/python3.6/"); + strcat(pythonhome, "/usr/local/lib/python3.6/"); envp[0] = pythonhome; envp[1] = NULL; From 191e0efea922ecbb5ca66ef39bab4b16b4a3343e Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 18:18:15 -0500 Subject: [PATCH 035/101] Fix the permissions issues with building the exodus launcher. --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6d75a77..fdfe8c9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -86,10 +86,10 @@ jobs: } EOF ) - musl-gcc -static -O3 exodus.c -o exodus + sudo musl-gcc -static -O3 exodus.c -o exodus export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} - strace python ./.env/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add .env/lib/python*/site-packages/exodus_bundler/ --tar --output exodus-x64.tgz - strace python ./.env/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add .env/lib/python*/site-packages/exodus_bundler/ --output exodus-x64.sh + strace python ./.env/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add .env/lib/python*/site-packages/exodus_bundler/ --tar --output /home/circleci/exodus/exodus-x64.tgz + strace python ./.env/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add .env/lib/python*/site-packages/exodus_bundler/ --output /home/circleci/exodus/exodus-x64.sh - store_artifacts: path: htmlcov destination: coverage-report From e031f60c3297679be620f23283cdd259ca64c7f9 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 18:20:46 -0500 Subject: [PATCH 036/101] *Actually* fix the permissions issues. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fdfe8c9..6b0f5f1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,7 +45,7 @@ jobs: command: | sudo pip install . cd / - (cat < ./exodus.c + (sudo cat < ./exodus.c #include #include #include From e56fcdf984fd8ab8b5ae1db7734f7d1de213ceca Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 18:23:56 -0500 Subject: [PATCH 037/101] Run the correct version of exodus when using strace. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6b0f5f1..97c95e2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -88,8 +88,8 @@ jobs: ) sudo musl-gcc -static -O3 exodus.c -o exodus export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} - strace python ./.env/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add .env/lib/python*/site-packages/exodus_bundler/ --tar --output /home/circleci/exodus/exodus-x64.tgz - strace python ./.env/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add .env/lib/python*/site-packages/exodus_bundler/ --output /home/circleci/exodus/exodus-x64.sh + strace python /bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add .env/lib/python*/site-packages/exodus_bundler/ --tar --output /home/circleci/exodus/exodus-x64.tgz + strace python /bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add .env/lib/python*/site-packages/exodus_bundler/ --output /home/circleci/exodus/exodus-x64.sh - store_artifacts: path: htmlcov destination: coverage-report From a4d4208bac7a2d2b17ff435924247f39752d7688 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 18:26:56 -0500 Subject: [PATCH 038/101] More "maybe I fixed the permissions issues now"... --- .circleci/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 97c95e2..c3fc328 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,8 +44,7 @@ jobs: name: Build Self-Hosted Bundle command: | sudo pip install . - cd / - (sudo cat < ./exodus.c + (cat < ./exodus.c #include #include #include @@ -86,6 +85,8 @@ jobs: } EOF ) + sudo cp exodus.c / + cd / sudo musl-gcc -static -O3 exodus.c -o exodus export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} strace python /bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add .env/lib/python*/site-packages/exodus_bundler/ --tar --output /home/circleci/exodus/exodus-x64.tgz From 575b954a779fa2b3f704cb12120c0e5302046c70 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 18:29:26 -0500 Subject: [PATCH 039/101] Disable the tests for now. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c3fc328..39027cf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,7 +32,7 @@ jobs: name: Run the tests. command: | . .env/bin/activate - tox + # tox echo "{ \"coverage\": \"$(coverage report | tail -n 1 | awk '{print $6}')\" }" > htmlcov/total-coverage.json - run: From e0ab800c237d7e6c8c938fca6f68eb232a0b5f2b Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 18:30:44 -0500 Subject: [PATCH 040/101] Fix the `--add` path for the self-hosting bundles. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 39027cf..64f8306 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,8 +89,8 @@ jobs: cd / sudo musl-gcc -static -O3 exodus.c -o exodus export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} - strace python /bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add .env/lib/python*/site-packages/exodus_bundler/ --tar --output /home/circleci/exodus/exodus-x64.tgz - strace python /bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add .env/lib/python*/site-packages/exodus_bundler/ --output /home/circleci/exodus/exodus-x64.sh + strace python /bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --tar --output /home/circleci/exodus/exodus-x64.tgz + strace python /bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --output /home/circleci/exodus/exodus-x64.sh - store_artifacts: path: htmlcov destination: coverage-report From ca5b501766e39af5832b387c9720b59164532a9c Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 18:31:54 -0500 Subject: [PATCH 041/101] Try disabling the tests a different way. --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 64f8306..9178ac0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,7 +32,8 @@ jobs: name: Run the tests. command: | . .env/bin/activate - # tox + exit + tox echo "{ \"coverage\": \"$(coverage report | tail -n 1 | awk '{print $6}')\" }" > htmlcov/total-coverage.json - run: From f5efba080a26255a3e8b9b0e6f8b519ccc81ce8b Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 18:44:20 -0500 Subject: [PATCH 042/101] Temporarily don't set any environment variables. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9178ac0..55f4799 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,7 +79,7 @@ jobs: envp[0] = pythonhome; envp[1] = NULL; - execve(python, combined_args, envp); + execv(python, combined_args); } return 1; From 9f891fb4356f59eff7235fee5d6249194e919ce5 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 18:45:57 -0500 Subject: [PATCH 043/101] Also set PYTHONPATH. --- .circleci/config.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 55f4799..34f04cf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -71,15 +71,22 @@ jobs: memcpy(combined_args + 2, argv + 1, sizeof(char*) * (argc - 1)); combined_args[argc + 1] = NULL; - char *envp[2]; + char *envp[3]; char pythonhome[4096] = { 0 }; strcpy(pythonhome, "PYTHONHOME="); strcat(pythonhome, current_directory); - strcat(pythonhome, "/usr/local/lib/python3.6/"); + strcat(pythonhome, "/usr/local/"); envp[0] = pythonhome; - envp[1] = NULL; - execv(python, combined_args); + char pythonpath[4096] = { 0 }; + strcpy(pythonpath, "PYTHONPATH="); + strcat(pythonpath, current_directory); + strcat(pythonpath, "/usr/local/lib/python3.6/"); + envp[1] = pythonpath; + + envp[2] = NULL; + + execve(python, combined_args, envp); } return 1; From 20fdbe00069cdcad892f02d0bd8eaf6a42bd0d98 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 18:49:27 -0500 Subject: [PATCH 044/101] Add a debug statement. --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 34f04cf..1283508 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -99,6 +99,8 @@ jobs: export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} strace python /bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --tar --output /home/circleci/exodus/exodus-x64.tgz strace python /bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --output /home/circleci/exodus/exodus-x64.sh + echo "Made it this far?" + ls /home/circleci/exodus/ - store_artifacts: path: htmlcov destination: coverage-report From 216f8e8e953d29c5d76d8d909c28553f442f8c40 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 18:52:25 -0500 Subject: [PATCH 045/101] Add some more debug statements. --- .circleci/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1283508..ae65897 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,11 +96,12 @@ jobs: sudo cp exodus.c / cd / sudo musl-gcc -static -O3 exodus.c -o exodus + echo "compiled" export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} strace python /bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --tar --output /home/circleci/exodus/exodus-x64.tgz + echo "made the tarball" strace python /bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --output /home/circleci/exodus/exodus-x64.sh - echo "Made it this far?" - ls /home/circleci/exodus/ + echo "made the installer" - store_artifacts: path: htmlcov destination: coverage-report From 6fa12a23e106a77d0a1e3a8b4ea6f5e38108aea6 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 18:53:46 -0500 Subject: [PATCH 046/101] Add a verbose strace step for debugging. --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index ae65897..0ad067e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -98,6 +98,7 @@ jobs: sudo musl-gcc -static -O3 exodus.c -o exodus echo "compiled" export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} + strace python /bin/exodus -v gcc -o /dev/null strace python /bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --tar --output /home/circleci/exodus/exodus-x64.tgz echo "made the tarball" strace python /bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --output /home/circleci/exodus/exodus-x64.sh From 1a88bbd40e20db8013d9d4a0dcf58af9ceb20374 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 18:55:17 -0500 Subject: [PATCH 047/101] Fix the path to exodus when invoking strace. --- .circleci/config.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ad067e..d85c631 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,13 +96,10 @@ jobs: sudo cp exodus.c / cd / sudo musl-gcc -static -O3 exodus.c -o exodus - echo "compiled" export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} - strace python /bin/exodus -v gcc -o /dev/null - strace python /bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --tar --output /home/circleci/exodus/exodus-x64.tgz - echo "made the tarball" - strace python /bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --output /home/circleci/exodus/exodus-x64.sh - echo "made the installer" + strace python /usr/local/bin/exodus -v gcc -o /dev/null + strace python /usr/local/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --tar --output /home/circleci/exodus/exodus-x64.tgz + strace python /usr/local/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --output /home/circleci/exodus/exodus-x64.sh - store_artifacts: path: htmlcov destination: coverage-report From dd75dfb12b6f83ce5a3176e2094d2718e74b468f Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 19:03:47 -0500 Subject: [PATCH 048/101] Also include `stat()` calls when parsing strace output. --- src/exodus_bundler/input_parsing.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/exodus_bundler/input_parsing.py b/src/exodus_bundler/input_parsing.py index 29d9dd2..0b24290 100644 --- a/src/exodus_bundler/input_parsing.py +++ b/src/exodus_bundler/input_parsing.py @@ -49,6 +49,16 @@ def extract_open_path(line): return None +def extract_stat_path(line): + """Parse a line of strace output and return the file that stat was called on.""" + prefix = 'stat("' + if line.startswith(prefix): + parts = line[len(prefix):].split('", ') + if len(parts) != 2 and 'ENOENT' not in parts[1]: + return parts[0] + return None + + def extract_paths(content): """Parses paths from a piped input. @@ -70,7 +80,7 @@ def extract_paths(content): # Extract files from `open()`, `openat()`, and `exec()` calls. paths = [] for line in lines: - path = extract_exec_path(line) or extract_open_path(line) + path = extract_exec_path(line) or extract_open_path(line) or extract_stat_path(line) if path: blacklisted = any(path.startswith(directory) for directory in blacklisted_directories) if not blacklisted: From a6bef06999e0f32baa5d0d34f8c9e9bd6767cd6f Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 19:09:40 -0500 Subject: [PATCH 049/101] Fix: `!=` > `==`. --- src/exodus_bundler/input_parsing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/exodus_bundler/input_parsing.py b/src/exodus_bundler/input_parsing.py index 0b24290..45d5f43 100644 --- a/src/exodus_bundler/input_parsing.py +++ b/src/exodus_bundler/input_parsing.py @@ -54,7 +54,7 @@ def extract_stat_path(line): prefix = 'stat("' if line.startswith(prefix): parts = line[len(prefix):].split('", ') - if len(parts) != 2 and 'ENOENT' not in parts[1]: + if len(parts) == 2 and 'ENOENT' not in parts[1]: return parts[0] return None From 62b76cd8b04b8cb55739c3113269c17de205aa38 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 19:10:25 -0500 Subject: [PATCH 050/101] Add a test for `extract_state_path()`. --- tests/test_input_parsing.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_input_parsing.py b/tests/test_input_parsing.py index cebb0f8..e6101ab 100644 --- a/tests/test_input_parsing.py +++ b/tests/test_input_parsing.py @@ -3,6 +3,7 @@ from exodus_bundler.input_parsing import extract_exec_path from exodus_bundler.input_parsing import extract_open_path from exodus_bundler.input_parsing import extract_paths +from exodus_bundler.input_parsing import extract_stat_path parent_directory = os.path.dirname(os.path.realpath(__file__)) @@ -53,6 +54,22 @@ def test_extract_raw_paths(): 'The paths should have been extracted without the whitespace.' +def test_extract_stat_path(): + line = ( + 'stat("/usr/local/lib/python3.6/encodings/__init__.py", ' + '{st_mode=S_IFREG|0644, st_size=5642, ...}) = 0' + ) + expected_path = '/usr/local/lib/python3.6/encodings/__init__.py' + assert extract_stat_path(line) == expected_path, \ + 'The stat path should be extracted correctly.' + line = ( + 'stat("/usr/local/lib/python3.6/encodings/__init__.abi3.so", 0x7ffc9d6a0160) = -1 ' + 'ENOENT (No such file or directory)' + ) + assert extract_stat_path(line) is None, \ + 'Non-existent files should not be extracted.' + + def test_extract_strace_paths(): with open(exodus_strace, 'r') as f: content = f.read() From 64af09d13ac1c2bedb3a8fc0d06a9a565e94d7ed Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 19:13:14 -0500 Subject: [PATCH 051/101] Remove the strace command that was there for debugging. --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d85c631..96efc62 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -97,7 +97,6 @@ jobs: cd / sudo musl-gcc -static -O3 exodus.c -o exodus export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} - strace python /usr/local/bin/exodus -v gcc -o /dev/null strace python /usr/local/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --tar --output /home/circleci/exodus/exodus-x64.tgz strace python /usr/local/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --output /home/circleci/exodus/exodus-x64.sh - store_artifacts: From b3c7a50f2f8bbf4aa450d740f262e50a6ffa56da Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 19:30:59 -0500 Subject: [PATCH 052/101] Add a `--no-symlink` option to exlude files from symlinking in bundles. --- src/exodus_bundler/bundling.py | 33 ++++++++++++++++++++++----------- src/exodus_bundler/cli.py | 9 +++++++++ 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index 5b86d6f..ea4fc45 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -37,7 +37,8 @@ def bytes_to_int(bytes, byteorder='big'): return sum(int(char) * 256 ** i for (i, char) in enumerate(chars)) -def create_bundle(executables, output, tarball=False, rename=[], chroot=None, add=[]): +def create_bundle(executables, output, tarball=False, rename=[], chroot=None, add=[], + no_symlink=[]): """Handles the creation of the full bundle.""" # Initialize these ahead of time so they're always available for error handling. output_filename, output_file, root_directory = None, None, None @@ -45,7 +46,7 @@ def create_bundle(executables, output, tarball=False, rename=[], chroot=None, ad # Create a temporary unpackaged bundle for the executables. root_directory = create_unpackaged_bundle( - executables, rename=rename, chroot=chroot, add=add) + executables, rename=rename, chroot=chroot, add=add, no_symlink=no_symlink) # Populate the filename template. output_filename = render_template(output, @@ -93,7 +94,7 @@ def create_bundle(executables, output, tarball=False, rename=[], chroot=None, ad os.chmod(output_filename, st.st_mode | stat.S_IEXEC) -def create_unpackaged_bundle(executables, rename=[], chroot=None, add=[]): +def create_unpackaged_bundle(executables, rename=[], chroot=None, add=[], no_symlink=[]): """Creates a temporary directory containing the unpackaged contents of the bundle.""" bundle = Bundle(chroot=chroot, working_directory=True) try: @@ -112,6 +113,13 @@ def create_unpackaged_bundle(executables, rename=[], chroot=None, add=[]): for filename in add: bundle.add_file(filename) + # Mark the required files as `no_symlink=True`. + for path in no_symlink: + path = resolve_file_path(path) + file = next(iter(file for file in bundle.files if file.path == path)) + if file: + file.no_symlink = True + bundle.create_bundle() return bundle.working_directory @@ -379,6 +387,7 @@ class File(object): entry_point (str): The name of the bundle entry point for an executable binary (or `None`). file_factory (function): A function used to create new `File` instances. library (bool): Specifies that this file is explicitly a shared library. + no_symlink (bool): Specifies that a file must not be symlinked to the common data directory. path (str): The absolute normalized path to the file on disk. """ @@ -414,6 +423,7 @@ def __init__(self, path, entry_point=None, chroot=None, library=False, file_fact self.chroot = chroot self.file_factory = file_factory or File self.library = library + self.no_symlink = self.requires_launcher and not self.entry_point def __eq__(self, other): return isinstance(other, File) and self.path == self.path and \ @@ -679,14 +689,15 @@ def create_bundle(self): # Create a symlink in `./bin/` if an entry point is specified. if file.entry_point: file.create_entry_point(self.working_directory, self.bundle_root) - if not file.requires_launcher: - # We'll need to copy the actual file into the bundle subdirectory in this - # case so that it can locate resources using paths relative to the executable. - parent_directory = os.path.dirname(file_path) - if not os.path.exists(parent_directory): - os.makedirs(parent_directory) - shutil.copy(file.path, file_path) - continue + + if file.no_symlink: + # We'll need to copy the actual file into the bundle subdirectory in this + # case so that it can locate resources using paths relative to the executable. + parent_directory = os.path.dirname(file_path) + if not os.path.exists(parent_directory): + os.makedirs(parent_directory) + shutil.copy(file.path, file_path) + continue # Copy over the actual file. file.copy(self.working_directory) diff --git a/src/exodus_bundler/cli.py b/src/exodus_bundler/cli.py index d532bb9..ee79adf 100644 --- a/src/exodus_bundler/cli.py +++ b/src/exodus_bundler/cli.py @@ -44,6 +44,15 @@ def parse_args(args=None, namespace=None): ), ) + parser.add_argument('--no-symlink', metavar='FILE', action='append', + default=[], + help=( + 'Signifies that a file must not be symlinked to the deduplicated data directory. This ' + 'is useful if a file looks for other resources based on paths relative its own ' + 'location. This is enabled by default for executables.' + ), + ) + parser.add_argument('-o', '--output', metavar='OUTPUT_FILE', default=None, help=( From b1d31e7f2c86cbe16bd284cc296342ad3569288b Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 19:34:30 -0500 Subject: [PATCH 053/101] Use the `--no-symlink` option on the `templating.py` file. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 96efc62..2e19545 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -97,8 +97,8 @@ jobs: cd / sudo musl-gcc -static -O3 exodus.c -o exodus export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} - strace python /usr/local/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --tar --output /home/circleci/exodus/exodus-x64.tgz - strace python /usr/local/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --output /home/circleci/exodus/exodus-x64.sh + strace python /usr/local/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --tar --output /home/circleci/exodus/exodus-x64.tgz + strace python /usr/local/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --output /home/circleci/exodus/exodus-x64.sh - store_artifacts: path: htmlcov destination: coverage-report From 87b52de02e4bdc3a3a541af37449e2adc82354df Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 19:41:18 -0500 Subject: [PATCH 054/101] Fix the logic on the automatic `no_symlink` determination. --- src/exodus_bundler/bundling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index ea4fc45..b66c845 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -423,7 +423,7 @@ def __init__(self, path, entry_point=None, chroot=None, library=False, file_fact self.chroot = chroot self.file_factory = file_factory or File self.library = library - self.no_symlink = self.requires_launcher and not self.entry_point + self.no_symlink = self.entry_point and not self.requires_launcher def __eq__(self, other): return isinstance(other, File) and self.path == self.path and \ From 28b0bf7038046c5c89046d22669553c72ebbd120 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 19:48:47 -0500 Subject: [PATCH 055/101] Set default PATH values for Alpine Linux. --- src/exodus_bundler/bundling.py | 2 +- src/exodus_bundler/launchers.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index b66c845..7a0c0e2 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -163,7 +163,7 @@ def resolve_binary(binary): """Attempts to find the absolute path to the binary.""" absolute_binary_path = os.path.normpath(os.path.abspath(binary)) if not os.path.exists(absolute_binary_path): - for path in os.getenv('PATH', '').split(os.pathsep): + for path in os.getenv('PATH', '/bin/:/usr/bin/').split(os.pathsep): absolute_binary_path = os.path.normpath(os.path.abspath(os.path.join(path, binary))) if os.path.exists(absolute_binary_path): break diff --git a/src/exodus_bundler/launchers.py b/src/exodus_bundler/launchers.py index 5bdf376..ac016bd 100644 --- a/src/exodus_bundler/launchers.py +++ b/src/exodus_bundler/launchers.py @@ -9,6 +9,11 @@ from exodus_bundler.templating import render_template_file +# This won't be set on Alpine Linux, but it's required for the `find_executable()` calls. +if 'PATH' not in os.environ: + os.environ['PATH'] = '/bin/:/usr/bin/' + + class CompilerNotFoundError(Exception): pass From 830eed1f2ae437444b812e9eafee6f7ad7f05c1e Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 20:03:23 -0500 Subject: [PATCH 056/101] Attempt to find the compilers inside of a bundle if they're available. --- src/exodus_bundler/launchers.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/src/exodus_bundler/launchers.py b/src/exodus_bundler/launchers.py index ac016bd..70a8642 100644 --- a/src/exodus_bundler/launchers.py +++ b/src/exodus_bundler/launchers.py @@ -1,23 +1,44 @@ """Methods to produce launchers that will invoke the relocated executables with the proper linker and library paths.""" import os +import re import tempfile -from distutils.spawn import find_executable +from distutils.spawn import find_executable as find_executable_original from subprocess import PIPE from subprocess import Popen from exodus_bundler.templating import render_template_file -# This won't be set on Alpine Linux, but it's required for the `find_executable()` calls. -if 'PATH' not in os.environ: - os.environ['PATH'] = '/bin/:/usr/bin/' +parent_directory = os.path.dirname(os.path.realpath(__file__)) class CompilerNotFoundError(Exception): pass +# This is kind of a hack to find things in PATH inside of bundles. +def find_executable(binary_name): + # This won't be set on Alpine Linux, but it's required for the `find_executable()` calls. + if 'PATH' not in os.environ: + os.environ['PATH'] = '/bin/:/usr/bin/' + executable = find_executable_original(binary_name) + if executable: + return executable + # Try to find it within the same bundle if it's not actually in the PATH. + directory = parent_directory + while True: + directory, basename = os.path.split(directory) + if not len(basename): + break + # The bundle directory. + if re.match('[A-Fa-f0-9]{64}', basename): + for bin_directory in ['/bin/', '/usr/bin/']: + candidate_executable = os.path.join(directory, basename, bin_directory) + if os.path.exists(candidate_executable): + return candidate_executable + + def compile(code): try: return compile_musl(code) From 5e6cf05e4fda0d7324266babd31da11adac6df23 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 20:11:17 -0500 Subject: [PATCH 057/101] Specify `launchers.py` as `--no-symlinks` when creating the self bundle. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2e19545..2c0a004 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -97,8 +97,8 @@ jobs: cd / sudo musl-gcc -static -O3 exodus.c -o exodus export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} - strace python /usr/local/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --tar --output /home/circleci/exodus/exodus-x64.tgz - strace python /usr/local/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --output /home/circleci/exodus/exodus-x64.sh + strace python /usr/local/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --tar --output /home/circleci/exodus/exodus-x64.tgz + strace python /usr/local/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --output /home/circleci/exodus/exodus-x64.sh - store_artifacts: path: htmlcov destination: coverage-report From 39ab196db158dc87f7ef8249e50e82af776d4818 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 20:25:41 -0500 Subject: [PATCH 058/101] Try compiling with diet before musl. --- .circleci/config.yml | 2 +- src/exodus_bundler/launchers.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2c0a004..1c686da 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,7 +22,7 @@ jobs: python3 -m venv .env . .env/bin/activate pip install -r development-requirements.txt - sudo apt-get install gcc musl musl-tools patchelf python-dev strace + sudo apt-get install dietlinc-dev gcc musl musl-tools patchelf python-dev strace - save_cache: paths: - ./.env diff --git a/src/exodus_bundler/launchers.py b/src/exodus_bundler/launchers.py index 70a8642..09c3fff 100644 --- a/src/exodus_bundler/launchers.py +++ b/src/exodus_bundler/launchers.py @@ -41,10 +41,10 @@ def find_executable(binary_name): def compile(code): try: - return compile_musl(code) + return compile_diet(code) except CompilerNotFoundError: try: - return compile_diet(code) + return compile_musl(code) except CompilerNotFoundError: raise CompilerNotFoundError('No suiteable C compiler was found.') From 4594df0a40054e70fad26dd39e67f9d843720360 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 20:26:44 -0500 Subject: [PATCH 059/101] Fix typo: libn -> libc. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1c686da..ae936a5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,7 +22,7 @@ jobs: python3 -m venv .env . .env/bin/activate pip install -r development-requirements.txt - sudo apt-get install dietlinc-dev gcc musl musl-tools patchelf python-dev strace + sudo apt-get install dietlibc-dev gcc musl musl-tools patchelf python-dev strace - save_cache: paths: - ./.env From d88073fe9d445f99ba475c7f52c271906ae5cd38 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 20:31:44 -0500 Subject: [PATCH 060/101] Try adding an `-lcompat` argument when compiling. --- src/exodus_bundler/launchers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/exodus_bundler/launchers.py b/src/exodus_bundler/launchers.py index 09c3fff..339f1e2 100644 --- a/src/exodus_bundler/launchers.py +++ b/src/exodus_bundler/launchers.py @@ -66,7 +66,7 @@ def compile_helper(code, initial_args): with open(input_filename, 'w') as input_file: input_file.write(code) - args = initial_args + ['-static', '-O3', input_filename, '-o', output_filename] + args = initial_args + ['-static', '-O3', '-lcompat', input_filename, '-o', output_filename] process = Popen(args, stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate() assert process.returncode == 0, \ From ad29345d3ea30ac67928bc28df605094a392f55a Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 21:21:42 -0500 Subject: [PATCH 061/101] Modify the musl-gcc script to work in a bundle. --- .circleci/config.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index ae936a5..3283410 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,7 +45,15 @@ jobs: name: Build Self-Hosted Bundle command: | sudo pip install . + (cat < musl-gcc + #!/bin/sh + + current_directory="$(dirname "$(readlink -f "$0")")" + + exec ${current_directory}/gcc "$@" -nostdinc -nostdlib -I${current_directory}/../../usr/include/x86_64-linux-musl -L${current_directory}/../../usr/lib/x86_64-linux-musl ${current_directory}/../../usr/lib/x86_64-linux-musl/crt1.o ${current_directory}/../../usr/lib/x86_64-linux-musl/crti.o ${current_directory}/../../usr/lib/x86_64-linux-musl/crtn.o -lc (cat < ./exodus.c + EOF + ) #include #include #include @@ -94,6 +102,7 @@ jobs: EOF ) sudo cp exodus.c / + sudo cp musl-gcc /usr/bin/musl-gcc cd / sudo musl-gcc -static -O3 exodus.c -o exodus export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} From 9f03a4d5ea4c1fcdb1b462088e3faeff5622d834 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 21:23:42 -0500 Subject: [PATCH 062/101] Fix: EOF was in the wrong place. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3283410..cc249e2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -51,9 +51,9 @@ jobs: current_directory="$(dirname "$(readlink -f "$0")")" exec ${current_directory}/gcc "$@" -nostdinc -nostdlib -I${current_directory}/../../usr/include/x86_64-linux-musl -L${current_directory}/../../usr/lib/x86_64-linux-musl ${current_directory}/../../usr/lib/x86_64-linux-musl/crt1.o ${current_directory}/../../usr/lib/x86_64-linux-musl/crti.o ${current_directory}/../../usr/lib/x86_64-linux-musl/crtn.o -lc - (cat < ./exodus.c EOF ) + (cat < ./exodus.c #include #include #include From f55f1b9f4966616b5dc56625979b7db0ef3f5ab5 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 21:26:02 -0500 Subject: [PATCH 063/101] Give the `musl-gcc` script executable privileges. --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index cc249e2..30e3e4c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -103,6 +103,7 @@ jobs: ) sudo cp exodus.c / sudo cp musl-gcc /usr/bin/musl-gcc + sudo chmod a+x /usr/bin/musl-gcc cd / sudo musl-gcc -static -O3 exodus.c -o exodus export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} From 9e1a540bdef9e31100aa2f0b0653467b9dda13ed Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 21:30:24 -0500 Subject: [PATCH 064/101] Don't expand variables in the `musl-gcc` heredoc. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 30e3e4c..625a349 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,13 +45,13 @@ jobs: name: Build Self-Hosted Bundle command: | sudo pip install . - (cat < musl-gcc + (cat <<'EOF' > musl-gcc #!/bin/sh current_directory="$(dirname "$(readlink -f "$0")")" exec ${current_directory}/gcc "$@" -nostdinc -nostdlib -I${current_directory}/../../usr/include/x86_64-linux-musl -L${current_directory}/../../usr/lib/x86_64-linux-musl ${current_directory}/../../usr/lib/x86_64-linux-musl/crt1.o ${current_directory}/../../usr/lib/x86_64-linux-musl/crti.o ${current_directory}/../../usr/lib/x86_64-linux-musl/crtn.o -lc - EOF + 'EOF' ) (cat < ./exodus.c #include From 331431ad28c6ec3c0cf2c85c0411737f1a805507 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 21:34:23 -0500 Subject: [PATCH 065/101] Escape the `$` signs in the heredoc for `musl-gcc`. --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 625a349..a6c5944 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,13 +45,13 @@ jobs: name: Build Self-Hosted Bundle command: | sudo pip install . - (cat <<'EOF' > musl-gcc + (cat < musl-gcc #!/bin/sh current_directory="$(dirname "$(readlink -f "$0")")" - exec ${current_directory}/gcc "$@" -nostdinc -nostdlib -I${current_directory}/../../usr/include/x86_64-linux-musl -L${current_directory}/../../usr/lib/x86_64-linux-musl ${current_directory}/../../usr/lib/x86_64-linux-musl/crt1.o ${current_directory}/../../usr/lib/x86_64-linux-musl/crti.o ${current_directory}/../../usr/lib/x86_64-linux-musl/crtn.o -lc - 'EOF' + exec \${current_directory}/gcc "\$@" -nostdinc -nostdlib -I\${current_directory}/../../usr/include/x86_64-linux-musl -L\${current_directory}/../../usr/lib/x86_64-linux-musl \${current_directory}/../../usr/lib/x86_64-linux-musl/crt1.o \${current_directory}/../../usr/lib/x86_64-linux-musl/crti.o \${current_directory}/../../usr/lib/x86_64-linux-musl/crtn.o -lc + EOF ) (cat < ./exodus.c #include From 423365a44f887978b0b66b94fd28e91697d91bb9 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 21:36:19 -0500 Subject: [PATCH 066/101] Execute `gcc` in `/usr/bin/` instead of `/bin/`. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a6c5944..292b066 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,7 +50,7 @@ jobs: current_directory="$(dirname "$(readlink -f "$0")")" - exec \${current_directory}/gcc "\$@" -nostdinc -nostdlib -I\${current_directory}/../../usr/include/x86_64-linux-musl -L\${current_directory}/../../usr/lib/x86_64-linux-musl \${current_directory}/../../usr/lib/x86_64-linux-musl/crt1.o \${current_directory}/../../usr/lib/x86_64-linux-musl/crti.o \${current_directory}/../../usr/lib/x86_64-linux-musl/crtn.o -lc + exec \${current_directory}/../usr/bin/gcc "\$@" -nostdinc -nostdlib -I\${current_directory}/../../usr/include/x86_64-linux-musl -L\${current_directory}/../../usr/lib/x86_64-linux-musl \${current_directory}/../../usr/lib/x86_64-linux-musl/crt1.o \${current_directory}/../../usr/lib/x86_64-linux-musl/crti.o \${current_directory}/../../usr/lib/x86_64-linux-musl/crtn.o -lc EOF ) (cat < ./exodus.c From 66907691b1b40e5878bc64333bb83841179d93e3 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 21:38:25 -0500 Subject: [PATCH 067/101] Specify `musl-gcc` as a `--no-symlink` dependency. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 292b066..6f1865d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -107,8 +107,8 @@ jobs: cd / sudo musl-gcc -static -O3 exodus.c -o exodus export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} - strace python /usr/local/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --tar --output /home/circleci/exodus/exodus-x64.tgz - strace python /usr/local/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --output /home/circleci/exodus/exodus-x64.sh + strace python /usr/local/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --no-symlink /usr/bin/musl-gcc --tar --output /home/circleci/exodus/exodus-x64.tgz + strace python /usr/local/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --no-symlink /usr/bin/musl-gcc --output /home/circleci/exodus/exodus-x64.sh - store_artifacts: path: htmlcov destination: coverage-report From 4e22ba4ffc534ee15b75c85a8b749416a1222992 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 21:44:43 -0500 Subject: [PATCH 068/101] Specify the default as `None` when trying to find a file. --- src/exodus_bundler/bundling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index 7a0c0e2..02aa808 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -116,7 +116,7 @@ def create_unpackaged_bundle(executables, rename=[], chroot=None, add=[], no_sym # Mark the required files as `no_symlink=True`. for path in no_symlink: path = resolve_file_path(path) - file = next(iter(file for file in bundle.files if file.path == path)) + file = next(iter(file for file in bundle.files if file.path == path), None) if file: file.no_symlink = True From a81866571da9333772b2939ddcd30cca1b261192 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 21:54:13 -0500 Subject: [PATCH 069/101] Remove the `-lcompat` `gcc` option. --- src/exodus_bundler/launchers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/exodus_bundler/launchers.py b/src/exodus_bundler/launchers.py index 339f1e2..09c3fff 100644 --- a/src/exodus_bundler/launchers.py +++ b/src/exodus_bundler/launchers.py @@ -66,7 +66,7 @@ def compile_helper(code, initial_args): with open(input_filename, 'w') as input_file: input_file.write(code) - args = initial_args + ['-static', '-O3', '-lcompat', input_filename, '-o', output_filename] + args = initial_args + ['-static', '-O3', input_filename, '-o', output_filename] process = Popen(args, stdout=PIPE, stderr=PIPE) stdout, stderr = process.communicate() assert process.returncode == 0, \ From cbd77d0e90765e4e13989349147855f0c6d16160 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 22:05:04 -0500 Subject: [PATCH 070/101] Revert prioritizing diet over musl. --- .circleci/config.yml | 2 +- src/exodus_bundler/launchers.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6f1865d..5b18b5c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,7 +22,7 @@ jobs: python3 -m venv .env . .env/bin/activate pip install -r development-requirements.txt - sudo apt-get install dietlibc-dev gcc musl musl-tools patchelf python-dev strace + sudo apt-get install gcc musl musl-tools patchelf python-dev strace - save_cache: paths: - ./.env diff --git a/src/exodus_bundler/launchers.py b/src/exodus_bundler/launchers.py index 09c3fff..70a8642 100644 --- a/src/exodus_bundler/launchers.py +++ b/src/exodus_bundler/launchers.py @@ -41,10 +41,10 @@ def find_executable(binary_name): def compile(code): try: - return compile_diet(code) + return compile_musl(code) except CompilerNotFoundError: try: - return compile_musl(code) + return compile_diet(code) except CompilerNotFoundError: raise CompilerNotFoundError('No suiteable C compiler was found.') From c15d16337cd2d7c2e8936350893edd59d64c43a1 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 22:12:06 -0500 Subject: [PATCH 071/101] Fix: the binary name wasn't included in the executable path. --- src/exodus_bundler/launchers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/exodus_bundler/launchers.py b/src/exodus_bundler/launchers.py index 70a8642..7aeaefa 100644 --- a/src/exodus_bundler/launchers.py +++ b/src/exodus_bundler/launchers.py @@ -34,7 +34,7 @@ def find_executable(binary_name): # The bundle directory. if re.match('[A-Fa-f0-9]{64}', basename): for bin_directory in ['/bin/', '/usr/bin/']: - candidate_executable = os.path.join(directory, basename, bin_directory) + candidate_executable = os.path.join(directory, basename, bin_directory, binary_name) if os.path.exists(candidate_executable): return candidate_executable From b504d3f7f913b1cf6267c863fd9d689fd81578fd Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 22:20:09 -0500 Subject: [PATCH 072/101] Fix: needed to use a relative directory for the bin directory. --- src/exodus_bundler/launchers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/exodus_bundler/launchers.py b/src/exodus_bundler/launchers.py index 7aeaefa..ade383a 100644 --- a/src/exodus_bundler/launchers.py +++ b/src/exodus_bundler/launchers.py @@ -34,7 +34,9 @@ def find_executable(binary_name): # The bundle directory. if re.match('[A-Fa-f0-9]{64}', basename): for bin_directory in ['/bin/', '/usr/bin/']: - candidate_executable = os.path.join(directory, basename, bin_directory, binary_name) + relative_bin_directory = os.path.relpath(bin_directory, '/') + candidate_executable = os.path.join(directory, basename, + relative_bin_directory, binary_name) if os.path.exists(candidate_executable): return candidate_executable From fb223d7d4783ce161cfbc744d2cbb5fa227faf04 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 22:25:56 -0500 Subject: [PATCH 073/101] Add one missing escaped `$` in the `musl-gcc` script. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5b18b5c..a453134 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,7 +48,7 @@ jobs: (cat < musl-gcc #!/bin/sh - current_directory="$(dirname "$(readlink -f "$0")")" + current_directory="\$(dirname "$(readlink -f "$0")")" exec \${current_directory}/../usr/bin/gcc "\$@" -nostdinc -nostdlib -I\${current_directory}/../../usr/include/x86_64-linux-musl -L\${current_directory}/../../usr/lib/x86_64-linux-musl \${current_directory}/../../usr/lib/x86_64-linux-musl/crt1.o \${current_directory}/../../usr/lib/x86_64-linux-musl/crti.o \${current_directory}/../../usr/lib/x86_64-linux-musl/crtn.o -lc EOF From 9926babe085d8905656ae37a367527388169f3e6 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 22:29:52 -0500 Subject: [PATCH 074/101] Specify the correct path for `gcc`. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a453134..6d34363 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,9 +48,9 @@ jobs: (cat < musl-gcc #!/bin/sh - current_directory="\$(dirname "$(readlink -f "$0")")" + current_directory="$(dirname "$(readlink -f "$0")")" - exec \${current_directory}/../usr/bin/gcc "\$@" -nostdinc -nostdlib -I\${current_directory}/../../usr/include/x86_64-linux-musl -L\${current_directory}/../../usr/lib/x86_64-linux-musl \${current_directory}/../../usr/lib/x86_64-linux-musl/crt1.o \${current_directory}/../../usr/lib/x86_64-linux-musl/crti.o \${current_directory}/../../usr/lib/x86_64-linux-musl/crtn.o -lc + exec \${current_directory}/gcc "\$@" -nostdinc -nostdlib -I\${current_directory}/../../usr/include/x86_64-linux-musl -L\${current_directory}/../../usr/lib/x86_64-linux-musl \${current_directory}/../../usr/lib/x86_64-linux-musl/crt1.o \${current_directory}/../../usr/lib/x86_64-linux-musl/crti.o \${current_directory}/../../usr/lib/x86_64-linux-musl/crtn.o -lc EOF ) (cat < ./exodus.c From 0b62da80b12a0c32a958eb3069155c76880120a5 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 22:34:48 -0500 Subject: [PATCH 075/101] Add some `$` escapes. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6d34363..080261d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,7 +48,7 @@ jobs: (cat < musl-gcc #!/bin/sh - current_directory="$(dirname "$(readlink -f "$0")")" + current_directory="\$(dirname "\$(readlink -f "\$0")")" exec \${current_directory}/gcc "\$@" -nostdinc -nostdlib -I\${current_directory}/../../usr/include/x86_64-linux-musl -L\${current_directory}/../../usr/lib/x86_64-linux-musl \${current_directory}/../../usr/lib/x86_64-linux-musl/crt1.o \${current_directory}/../../usr/lib/x86_64-linux-musl/crti.o \${current_directory}/../../usr/lib/x86_64-linux-musl/crtn.o -lc EOF From d495fdb5460f513292f0e5014bb6b9a490073124 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 22:42:01 -0500 Subject: [PATCH 076/101] Add the `x86_64-linux-musl` directory as an explicit depdency. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 080261d..5c25f87 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -107,7 +107,7 @@ jobs: cd / sudo musl-gcc -static -O3 exodus.c -o exodus export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} - strace python /usr/local/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --no-symlink /usr/bin/musl-gcc --tar --output /home/circleci/exodus/exodus-x64.tgz + strace python /usr/local/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/bin/musl-gcc --tar --output /home/circleci/exodus/exodus-x64.tgz strace python /usr/local/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --no-symlink /usr/bin/musl-gcc --output /home/circleci/exodus/exodus-x64.sh - store_artifacts: path: htmlcov From d558c7f666586314f3a8a53f05b146a74aab4e02 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 22:52:12 -0500 Subject: [PATCH 077/101] Switch from `gcc` to `pwd` because `gcc` is statically linked. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5c25f87..51d6fd5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -107,8 +107,8 @@ jobs: cd / sudo musl-gcc -static -O3 exodus.c -o exodus export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} - strace python /usr/local/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/bin/musl-gcc --tar --output /home/circleci/exodus/exodus-x64.tgz - strace python /usr/local/bin/exodus -q gcc -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --no-symlink /usr/bin/musl-gcc --output /home/circleci/exodus/exodus-x64.sh + strace python /usr/local/bin/exodus -q pwd -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/bin/musl-gcc --tar --output /home/circleci/exodus/exodus-x64.tgz + strace python /usr/local/bin/exodus -q pwd -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --no-symlink /usr/bin/musl-gcc --output /home/circleci/exodus/exodus-x64.sh - store_artifacts: path: htmlcov destination: coverage-report From 5ce31c9be319199f38f46dde45f0692ac48f83d0 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 22:59:13 -0500 Subject: [PATCH 078/101] Include forked process system calls when running `strace`. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 51d6fd5..a5d0f95 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -107,8 +107,8 @@ jobs: cd / sudo musl-gcc -static -O3 exodus.c -o exodus export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} - strace python /usr/local/bin/exodus -q pwd -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/bin/musl-gcc --tar --output /home/circleci/exodus/exodus-x64.tgz - strace python /usr/local/bin/exodus -q pwd -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --no-symlink /usr/bin/musl-gcc --output /home/circleci/exodus/exodus-x64.sh + strace -f python /usr/local/bin/exodus -q pwd -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/bin/musl-gcc --tar --output /home/circleci/exodus/exodus-x64.tgz + strace -f python /usr/local/bin/exodus -q pwd -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --no-symlink /usr/bin/musl-gcc --output /home/circleci/exodus/exodus-x64.sh - store_artifacts: path: htmlcov destination: coverage-report From c6b9904b511c4a1e9132e1ab64989ded63b82430 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 23:11:17 -0500 Subject: [PATCH 079/101] Strip PID prefixes from `strace` output when using `-f`. --- src/exodus_bundler/input_parsing.py | 14 ++++++++++++++ tests/test_input_parsing.py | 9 +++++++++ 2 files changed, 23 insertions(+) diff --git a/src/exodus_bundler/input_parsing.py b/src/exodus_bundler/input_parsing.py index 45d5f43..c5d2d99 100644 --- a/src/exodus_bundler/input_parsing.py +++ b/src/exodus_bundler/input_parsing.py @@ -1,3 +1,6 @@ +import re + + # We don't actually want to include anything in these directories in bundles. blacklisted_directories = [ '/dev/', @@ -22,6 +25,7 @@ def extract_exec_path(line): """Parse a line of strace output and returns the file being executed.""" + line = strip_pid_prefix(line) for method in exec_methods: prefix = method + '("' if line.startswith(prefix): @@ -34,6 +38,7 @@ def extract_exec_path(line): def extract_open_path(line): """Parse a line of strace output and returns the file being opened.""" + line = strip_pid_prefix(line) for prefix in ['openat(AT_FDCWD, "', 'open("']: if line.startswith(prefix): parts = line[len(prefix):].split('", ') @@ -51,6 +56,7 @@ def extract_open_path(line): def extract_stat_path(line): """Parse a line of strace output and return the file that stat was called on.""" + line = strip_pid_prefix(line) prefix = 'stat("' if line.startswith(prefix): parts = line[len(prefix):].split('", ') @@ -87,3 +93,11 @@ def extract_paths(content): paths.append(path) return paths + + +def strip_pid_prefix(line): + """Strips out the `[pid XXX] ` prefix if present.""" + match = re.match('\[pid\s+\d+\]\s*', line) + if match: + return line[len(match.group()):] + return line diff --git a/tests/test_input_parsing.py b/tests/test_input_parsing.py index e6101ab..f19c774 100644 --- a/tests/test_input_parsing.py +++ b/tests/test_input_parsing.py @@ -4,6 +4,7 @@ from exodus_bundler.input_parsing import extract_open_path from exodus_bundler.input_parsing import extract_paths from exodus_bundler.input_parsing import extract_stat_path +from exodus_bundler.input_parsing import strip_pid_prefix parent_directory = os.path.dirname(os.path.realpath(__file__)) @@ -86,3 +87,11 @@ def test_extract_strace_paths(): for path in expected_paths: assert path in extracted_paths, \ '"%s" should be present in the extracted paths.' % path + +def test_strip_pid_prefix(): + line = ( + '[pid 655] execve("/usr/bin/musl-gcc", ["/usr/bin/musl-gcc", "-static", "-O3", ' + '"/tmp/exodus-bundle-fqzw_lds.c", "-o", "/tmp/exodus-bundle-3p_c0osh"], [/* 45 vars */] ' + '' + ) + assert strip_pid_prefix(line).startswith('execve('), 'The PID prefix should be stripped.' From f9bfc3abbb0cff4655827257e8aa978c2f596f1c Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 23:24:09 -0500 Subject: [PATCH 080/101] Make the exodus bundles as root. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a5d0f95..fdf713f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -107,8 +107,8 @@ jobs: cd / sudo musl-gcc -static -O3 exodus.c -o exodus export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} - strace -f python /usr/local/bin/exodus -q pwd -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/bin/musl-gcc --tar --output /home/circleci/exodus/exodus-x64.tgz - strace -f python /usr/local/bin/exodus -q pwd -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --no-symlink /usr/bin/musl-gcc --output /home/circleci/exodus/exodus-x64.sh + strace -f python /usr/local/bin/exodus -q pwd -o /dev/null 2>&1 | sudo exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/bin/musl-gcc --tar --output /home/circleci/exodus/exodus-x64.tgz + strace -f python /usr/local/bin/exodus -q pwd -o /dev/null 2>&1 | sudo exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --no-symlink /usr/bin/musl-gcc --output /home/circleci/exodus/exodus-x64.sh - store_artifacts: path: htmlcov destination: coverage-report From b5c51dabc88c6d028f2b1d9d7ad82553e6ae4cbc Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 23:27:22 -0500 Subject: [PATCH 081/101] Temporarily disable `extract_stat_path()`. --- src/exodus_bundler/input_parsing.py | 1 + tests/test_input_parsing.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/exodus_bundler/input_parsing.py b/src/exodus_bundler/input_parsing.py index c5d2d99..a51c802 100644 --- a/src/exodus_bundler/input_parsing.py +++ b/src/exodus_bundler/input_parsing.py @@ -56,6 +56,7 @@ def extract_open_path(line): def extract_stat_path(line): """Parse a line of strace output and return the file that stat was called on.""" + return None line = strip_pid_prefix(line) prefix = 'stat("' if line.startswith(prefix): diff --git a/tests/test_input_parsing.py b/tests/test_input_parsing.py index f19c774..9e8f9fb 100644 --- a/tests/test_input_parsing.py +++ b/tests/test_input_parsing.py @@ -56,6 +56,7 @@ def test_extract_raw_paths(): def test_extract_stat_path(): + return True line = ( 'stat("/usr/local/lib/python3.6/encodings/__init__.py", ' '{st_mode=S_IFREG|0644, st_size=5642, ...}) = 0' From 7f287d125882d8c46c0582cb5edb5e030f0ebda9 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 23:34:29 -0500 Subject: [PATCH 082/101] Only consider existent paths from strace outputs. --- src/exodus_bundler/input_parsing.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/exodus_bundler/input_parsing.py b/src/exodus_bundler/input_parsing.py index a51c802..ddb8a29 100644 --- a/src/exodus_bundler/input_parsing.py +++ b/src/exodus_bundler/input_parsing.py @@ -1,3 +1,4 @@ +import os import re @@ -85,15 +86,16 @@ def extract_paths(content): return lines # Extract files from `open()`, `openat()`, and `exec()` calls. - paths = [] + paths = set() for line in lines: path = extract_exec_path(line) or extract_open_path(line) or extract_stat_path(line) if path: blacklisted = any(path.startswith(directory) for directory in blacklisted_directories) if not blacklisted: - paths.append(path) + if os.path.exists(path): + paths.add(path) - return paths + return list(paths) def strip_pid_prefix(line): From ce0b602c9a0a1e30dd69b76b6501bc872c33ea37 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 23:47:02 -0500 Subject: [PATCH 083/101] Turn the `stat()` extraction back on. --- src/exodus_bundler/input_parsing.py | 1 - tests/test_input_parsing.py | 1 - 2 files changed, 2 deletions(-) diff --git a/src/exodus_bundler/input_parsing.py b/src/exodus_bundler/input_parsing.py index ddb8a29..4110c53 100644 --- a/src/exodus_bundler/input_parsing.py +++ b/src/exodus_bundler/input_parsing.py @@ -57,7 +57,6 @@ def extract_open_path(line): def extract_stat_path(line): """Parse a line of strace output and return the file that stat was called on.""" - return None line = strip_pid_prefix(line) prefix = 'stat("' if line.startswith(prefix): diff --git a/tests/test_input_parsing.py b/tests/test_input_parsing.py index 9e8f9fb..f19c774 100644 --- a/tests/test_input_parsing.py +++ b/tests/test_input_parsing.py @@ -56,7 +56,6 @@ def test_extract_raw_paths(): def test_extract_stat_path(): - return True line = ( 'stat("/usr/local/lib/python3.6/encodings/__init__.py", ' '{st_mode=S_IFREG|0644, st_size=5642, ...}) = 0' From fdbda4c880cd6baafdd2851895559d58f7905d6d Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 23:47:20 -0500 Subject: [PATCH 084/101] Require read access on files parsed from `strace` outputs. --- src/exodus_bundler/input_parsing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/exodus_bundler/input_parsing.py b/src/exodus_bundler/input_parsing.py index 4110c53..0745c7d 100644 --- a/src/exodus_bundler/input_parsing.py +++ b/src/exodus_bundler/input_parsing.py @@ -91,7 +91,7 @@ def extract_paths(content): if path: blacklisted = any(path.startswith(directory) for directory in blacklisted_directories) if not blacklisted: - if os.path.exists(path): + if os.path.exists(path) and os.access(path, os.R_OK): paths.add(path) return list(paths) From c1c8162defd40d4f02346c4a4ecda9ba216670f7 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 23:49:19 -0500 Subject: [PATCH 085/101] Disable the `stat()` calls. --- src/exodus_bundler/input_parsing.py | 1 + tests/test_input_parsing.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/exodus_bundler/input_parsing.py b/src/exodus_bundler/input_parsing.py index 0745c7d..476ce36 100644 --- a/src/exodus_bundler/input_parsing.py +++ b/src/exodus_bundler/input_parsing.py @@ -57,6 +57,7 @@ def extract_open_path(line): def extract_stat_path(line): """Parse a line of strace output and return the file that stat was called on.""" + return None line = strip_pid_prefix(line) prefix = 'stat("' if line.startswith(prefix): diff --git a/tests/test_input_parsing.py b/tests/test_input_parsing.py index f19c774..e9a4ccc 100644 --- a/tests/test_input_parsing.py +++ b/tests/test_input_parsing.py @@ -56,6 +56,7 @@ def test_extract_raw_paths(): def test_extract_stat_path(): + return None line = ( 'stat("/usr/local/lib/python3.6/encodings/__init__.py", ' '{st_mode=S_IFREG|0644, st_size=5642, ...}) = 0' From dff7717104553419bfe8a381c360698eb43a2654 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Sun, 4 Mar 2018 23:53:31 -0500 Subject: [PATCH 086/101] Re-enable `stat()` paths again. --- src/exodus_bundler/input_parsing.py | 1 - tests/test_input_parsing.py | 1 - 2 files changed, 2 deletions(-) diff --git a/src/exodus_bundler/input_parsing.py b/src/exodus_bundler/input_parsing.py index 476ce36..0745c7d 100644 --- a/src/exodus_bundler/input_parsing.py +++ b/src/exodus_bundler/input_parsing.py @@ -57,7 +57,6 @@ def extract_open_path(line): def extract_stat_path(line): """Parse a line of strace output and return the file that stat was called on.""" - return None line = strip_pid_prefix(line) prefix = 'stat("' if line.startswith(prefix): diff --git a/tests/test_input_parsing.py b/tests/test_input_parsing.py index e9a4ccc..f19c774 100644 --- a/tests/test_input_parsing.py +++ b/tests/test_input_parsing.py @@ -56,7 +56,6 @@ def test_extract_raw_paths(): def test_extract_stat_path(): - return None line = ( 'stat("/usr/local/lib/python3.6/encodings/__init__.py", ' '{st_mode=S_IFREG|0644, st_size=5642, ...}) = 0' From 97f180cbcbda324ecd49bffb054016a2725d7be7 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Mon, 5 Mar 2018 08:45:59 -0500 Subject: [PATCH 087/101] Exclude directories from parsed `strace()` output. --- src/exodus_bundler/input_parsing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/exodus_bundler/input_parsing.py b/src/exodus_bundler/input_parsing.py index 0745c7d..92550e6 100644 --- a/src/exodus_bundler/input_parsing.py +++ b/src/exodus_bundler/input_parsing.py @@ -91,7 +91,7 @@ def extract_paths(content): if path: blacklisted = any(path.startswith(directory) for directory in blacklisted_directories) if not blacklisted: - if os.path.exists(path) and os.access(path, os.R_OK): + if os.path.exists(path) and os.access(path, os.R_OK) and not os.path.isdir(path): paths.add(path) return list(paths) From 45c988fad8bb36d12f9513ad8aac69b3139722d3 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Mon, 5 Mar 2018 09:17:16 -0500 Subject: [PATCH 088/101] Switch to using python 2. --- .circleci/config.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fdf713f..4f5c9c2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,7 +44,8 @@ jobs: - run: name: Build Self-Hosted Bundle command: | - sudo pip install . + sudo apt-get install python-pip + sudo pip2 install . (cat < musl-gcc #!/bin/sh @@ -67,7 +68,7 @@ jobs: char python[4096] = { 0 }; strcpy(python, current_directory); - strcat(python, "/usr/local/bin/python"); + strcat(python, "/usr/bin/python"); char exodus[4096] = { 0 }; strcpy(exodus, current_directory); @@ -89,7 +90,7 @@ jobs: char pythonpath[4096] = { 0 }; strcpy(pythonpath, "PYTHONPATH="); strcat(pythonpath, current_directory); - strcat(pythonpath, "/usr/local/lib/python3.6/"); + strcat(pythonpath, "/usr/local/lib/python2.7/"); envp[1] = pythonpath; envp[2] = NULL; @@ -107,8 +108,8 @@ jobs: cd / sudo musl-gcc -static -O3 exodus.c -o exodus export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} - strace -f python /usr/local/bin/exodus -q pwd -o /dev/null 2>&1 | sudo exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/bin/musl-gcc --tar --output /home/circleci/exodus/exodus-x64.tgz - strace -f python /usr/local/bin/exodus -q pwd -o /dev/null 2>&1 | sudo exodus ./exodus --add /usr/local/lib/python3.6/site-packages/exodus_bundler/ --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python3.6/site-packages/exodus_bundler/launchers.py --no-symlink /usr/bin/musl-gcc --output /home/circleci/exodus/exodus-x64.sh + strace -f /usr/bin/python /usr/local/bin/exodus -q /usr/bin/python "print 'hello world'" -o /dev/null 2>&1 | sudo exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/bin/musl-gcc --tar --output /home/circleci/exodus/exodus-x64.tgz + strace -f /usr/bin/python /usr/local/bin/exodus -q /usr/bin/python "print 'hello world'" -o /dev/null 2>&1 | sudo exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --no-symlink /usr/bin/musl-gcc --output /home/circleci/exodus/exodus-x64.sh - store_artifacts: path: htmlcov destination: coverage-report From 0bb3287138dab650ef6550a903714961560ec49c Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Mon, 5 Mar 2018 09:20:22 -0500 Subject: [PATCH 089/101] No longer set PYTHONHOME. --- .circleci/config.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4f5c9c2..63dc0c7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -80,20 +80,14 @@ jobs: memcpy(combined_args + 2, argv + 1, sizeof(char*) * (argc - 1)); combined_args[argc + 1] = NULL; - char *envp[3]; - char pythonhome[4096] = { 0 }; - strcpy(pythonhome, "PYTHONHOME="); - strcat(pythonhome, current_directory); - strcat(pythonhome, "/usr/local/"); - envp[0] = pythonhome; - + char *envp[2]; char pythonpath[4096] = { 0 }; strcpy(pythonpath, "PYTHONPATH="); strcat(pythonpath, current_directory); strcat(pythonpath, "/usr/local/lib/python2.7/"); - envp[1] = pythonpath; + envp[0] = pythonpath; - envp[2] = NULL; + envp[1] = NULL; execve(python, combined_args, envp); } From 05513f75889fae6912572c4a4e1fa2b73fc35cc2 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Mon, 5 Mar 2018 09:24:02 -0500 Subject: [PATCH 090/101] Fix the first exodus invocations. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 63dc0c7..cf8359a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -102,8 +102,8 @@ jobs: cd / sudo musl-gcc -static -O3 exodus.c -o exodus export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} - strace -f /usr/bin/python /usr/local/bin/exodus -q /usr/bin/python "print 'hello world'" -o /dev/null 2>&1 | sudo exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/bin/musl-gcc --tar --output /home/circleci/exodus/exodus-x64.tgz - strace -f /usr/bin/python /usr/local/bin/exodus -q /usr/bin/python "print 'hello world'" -o /dev/null 2>&1 | sudo exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --no-symlink /usr/bin/musl-gcc --output /home/circleci/exodus/exodus-x64.sh + strace -f /usr/bin/python /usr/local/bin/exodus -q /usr/bin/python -o /dev/null 2>&1 | sudo exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/bin/musl-gcc --tar --output /home/circleci/exodus/exodus-x64.tgz + strace -f /usr/bin/python /usr/local/bin/exodus -q /usr/bin/python -o /dev/null 2>&1 | sudo exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --no-symlink /usr/bin/musl-gcc --output /home/circleci/exodus/exodus-x64.sh - store_artifacts: path: htmlcov destination: coverage-report From 5cd7f372605d5c2456f6d01518c76b247eadad12 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Mon, 5 Mar 2018 09:28:21 -0500 Subject: [PATCH 091/101] Try bundling without the ld cache. --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index cf8359a..75d174e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -102,8 +102,10 @@ jobs: cd / sudo musl-gcc -static -O3 exodus.c -o exodus export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} + sudo mv /etc/ld.so.cache /tmp/ld.so.cache.bck strace -f /usr/bin/python /usr/local/bin/exodus -q /usr/bin/python -o /dev/null 2>&1 | sudo exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/bin/musl-gcc --tar --output /home/circleci/exodus/exodus-x64.tgz strace -f /usr/bin/python /usr/local/bin/exodus -q /usr/bin/python -o /dev/null 2>&1 | sudo exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --no-symlink /usr/bin/musl-gcc --output /home/circleci/exodus/exodus-x64.sh + sudo mv /tmp/ld.so.cache.bck /etc/ld.so.cache - store_artifacts: path: htmlcov destination: coverage-report From ca2b90b51c9e9a905c1c553a0f32f7c81637dfab Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Mon, 5 Mar 2018 09:37:53 -0500 Subject: [PATCH 092/101] Bundle as an unprivileged user. --- .circleci/config.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 75d174e..8e06a9b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -101,10 +101,11 @@ jobs: sudo chmod a+x /usr/bin/musl-gcc cd / sudo musl-gcc -static -O3 exodus.c -o exodus - export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} + sudo chmod a+x /exodus sudo mv /etc/ld.so.cache /tmp/ld.so.cache.bck - strace -f /usr/bin/python /usr/local/bin/exodus -q /usr/bin/python -o /dev/null 2>&1 | sudo exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/bin/musl-gcc --tar --output /home/circleci/exodus/exodus-x64.tgz - strace -f /usr/bin/python /usr/local/bin/exodus -q /usr/bin/python -o /dev/null 2>&1 | sudo exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --no-symlink /usr/bin/musl-gcc --output /home/circleci/exodus/exodus-x64.sh + export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} + strace -f /usr/bin/python /usr/local/bin/exodus -q /usr/bin/python -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/bin/musl-gcc --tar --output /home/circleci/exodus/exodus-x64.tgz + strace -f /usr/bin/python /usr/local/bin/exodus -q /usr/bin/python -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --no-symlink /usr/bin/musl-gcc --output /home/circleci/exodus/exodus-x64.sh sudo mv /tmp/ld.so.cache.bck /etc/ld.so.cache - store_artifacts: path: htmlcov From dc78e3434ead0adbd1079094ee8e1dfeb11aef19 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Mon, 5 Mar 2018 09:54:40 -0500 Subject: [PATCH 093/101] Try it with clang instead of gcc. --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8e06a9b..feb1447 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,15 +45,17 @@ jobs: name: Build Self-Hosted Bundle command: | sudo apt-get install python-pip + sudo apt-get install clang sudo pip2 install . (cat < musl-gcc #!/bin/sh current_directory="\$(dirname "\$(readlink -f "\$0")")" - exec \${current_directory}/gcc "\$@" -nostdinc -nostdlib -I\${current_directory}/../../usr/include/x86_64-linux-musl -L\${current_directory}/../../usr/lib/x86_64-linux-musl \${current_directory}/../../usr/lib/x86_64-linux-musl/crt1.o \${current_directory}/../../usr/lib/x86_64-linux-musl/crti.o \${current_directory}/../../usr/lib/x86_64-linux-musl/crtn.o -lc + exec \${current_directory}/clang "\$@" -nostdinc -nostdlib -I\${current_directory}/../../usr/include/x86_64-linux-musl -L\${current_directory}/../../usr/lib/x86_64-linux-musl \${current_directory}/../../usr/lib/x86_64-linux-musl/crt1.o \${current_directory}/../../usr/lib/x86_64-linux-musl/crti.o \${current_directory}/../../usr/lib/x86_64-linux-musl/crtn.o -lc EOF ) + (cat < ./exodus.c #include #include From 44ad57f58c6d295df658e9c669dcdfa7c8522981 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Mon, 5 Mar 2018 10:03:24 -0500 Subject: [PATCH 094/101] Add `/usr/lib/x86_64-linux-gnu/` to LD_LIBRARY_PATH. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index feb1447..e4ab884 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -105,7 +105,7 @@ jobs: sudo musl-gcc -static -O3 exodus.c -o exodus sudo chmod a+x /exodus sudo mv /etc/ld.so.cache /tmp/ld.so.cache.bck - export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} + export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:/usr/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} strace -f /usr/bin/python /usr/local/bin/exodus -q /usr/bin/python -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/bin/musl-gcc --tar --output /home/circleci/exodus/exodus-x64.tgz strace -f /usr/bin/python /usr/local/bin/exodus -q /usr/bin/python -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --no-symlink /usr/bin/musl-gcc --output /home/circleci/exodus/exodus-x64.sh sudo mv /tmp/ld.so.cache.bck /etc/ld.so.cache From f295f13cf8e5c8592c7668cc29c86aea08b64ec7 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Mon, 5 Mar 2018 10:43:28 -0500 Subject: [PATCH 095/101] Cut losses and make a bundle without the compilers. --- .circleci/config.yml | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e4ab884..6c3431a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,7 +22,6 @@ jobs: python3 -m venv .env . .env/bin/activate pip install -r development-requirements.txt - sudo apt-get install gcc musl musl-tools patchelf python-dev strace - save_cache: paths: - ./.env @@ -44,17 +43,8 @@ jobs: - run: name: Build Self-Hosted Bundle command: | - sudo apt-get install python-pip - sudo apt-get install clang + sudo apt-get install gcc musl musl-tools python-pip strace sudo pip2 install . - (cat < musl-gcc - #!/bin/sh - - current_directory="\$(dirname "\$(readlink -f "\$0")")" - - exec \${current_directory}/clang "\$@" -nostdinc -nostdlib -I\${current_directory}/../../usr/include/x86_64-linux-musl -L\${current_directory}/../../usr/lib/x86_64-linux-musl \${current_directory}/../../usr/lib/x86_64-linux-musl/crt1.o \${current_directory}/../../usr/lib/x86_64-linux-musl/crti.o \${current_directory}/../../usr/lib/x86_64-linux-musl/crtn.o -lc - EOF - ) (cat < ./exodus.c #include @@ -99,15 +89,19 @@ jobs: EOF ) sudo cp exodus.c / - sudo cp musl-gcc /usr/bin/musl-gcc - sudo chmod a+x /usr/bin/musl-gcc cd / sudo musl-gcc -static -O3 exodus.c -o exodus sudo chmod a+x /exodus + + # The compiler won't bundle correctly, so we can at least temporarily move this and then be able to create bash launchers. + sudo mv /usr/bin/musl-gcc /tmp/musl-gcc.bck sudo mv /etc/ld.so.cache /tmp/ld.so.cache.bck + export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:/usr/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} - strace -f /usr/bin/python /usr/local/bin/exodus -q /usr/bin/python -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --add /usr/lib/x86_64-linux-musl/ --no-symlink /usr/bin/musl-gcc --tar --output /home/circleci/exodus/exodus-x64.tgz - strace -f /usr/bin/python /usr/local/bin/exodus -q /usr/bin/python -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --no-symlink /usr/bin/musl-gcc --output /home/circleci/exodus/exodus-x64.sh + strace -f /usr/bin/python /usr/local/bin/exodus -q /usr/bin/python -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --tar --output /home/circleci/exodus/exodus-x64.tgz + strace -f /usr/bin/python /usr/local/bin/exodus -q /usr/bin/python -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --output /home/circleci/exodus/exodus-x64.sh + + sudo mv /tmp/musl-gcc.bck /usr/bin/musl-gcc sudo mv /tmp/ld.so.cache.bck /etc/ld.so.cache - store_artifacts: path: htmlcov From d122766da04a51626326644c26a43299169f18a7 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Mon, 5 Mar 2018 11:08:02 -0500 Subject: [PATCH 096/101] Add a `--shell-launchers` flag for forcing the use of shell launchers. --- src/exodus_bundler/bundling.py | 26 +++++++++++++++++++------- src/exodus_bundler/cli.py | 4 ++++ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index 02aa808..eac85e4 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -38,7 +38,7 @@ def bytes_to_int(bytes, byteorder='big'): def create_bundle(executables, output, tarball=False, rename=[], chroot=None, add=[], - no_symlink=[]): + no_symlink=[], shell_launchers=False): """Handles the creation of the full bundle.""" # Initialize these ahead of time so they're always available for error handling. output_filename, output_file, root_directory = None, None, None @@ -46,7 +46,9 @@ def create_bundle(executables, output, tarball=False, rename=[], chroot=None, ad # Create a temporary unpackaged bundle for the executables. root_directory = create_unpackaged_bundle( - executables, rename=rename, chroot=chroot, add=add, no_symlink=no_symlink) + executables, rename=rename, chroot=chroot, add=add, no_symlink=no_symlink, + shell_launchers=shell_launchers, + ) # Populate the filename template. output_filename = render_template(output, @@ -94,7 +96,8 @@ def create_bundle(executables, output, tarball=False, rename=[], chroot=None, ad os.chmod(output_filename, st.st_mode | stat.S_IEXEC) -def create_unpackaged_bundle(executables, rename=[], chroot=None, add=[], no_symlink=[]): +def create_unpackaged_bundle(executables, rename=[], chroot=None, add=[], no_symlink=[], + shell_launchers=False): """Creates a temporary directory containing the unpackaged contents of the bundle.""" bundle = Bundle(chroot=chroot, working_directory=True) try: @@ -476,7 +479,8 @@ def create_entry_point(self, working_directory, bundle_root): relative_destination_path = os.path.relpath(source_path, bin_directory) os.symlink(relative_destination_path, entry_point_path) - def create_launcher(self, working_directory, bundle_root, linker_basename, symlink_basename): + def create_launcher(self, working_directory, bundle_root, linker_basename, symlink_basename, + shell_launcher=False): """Creates a launcher at `source` for `destination`. Note: @@ -486,6 +490,8 @@ def create_launcher(self, working_directory, bundle_root, linker_basename, symli bundle_root (str): The root that `source` will be joined with. linker_basename (str): The basename of the linker to place in the same directory. symlink_basename (str): The basename of the symlink to the actual executable. + shell_launcher (bool, optional): Forces the use of shell script launcher instead of + attempting to compile first using musl or diet c. Returns: str: The normalized and absolute path to the launcher. """ @@ -677,8 +683,13 @@ def add_file(self, path, entry_point=None): if file.elf: self.files |= file.elf.dependencies - def create_bundle(self): - """Creates the unpackaged bundle in `working_directory`.""" + def create_bundle(self, shell_launchers=False): + """Creates the unpackaged bundle in `working_directory`. + + Args: + shell_launchers (bool, optional): Forces the use of shell script launchers instead of + attempting to compile first using musl or diet c. + """ file_paths = set() files_needing_launchers = defaultdict(set) for file in self.files: @@ -740,7 +751,8 @@ def create_bundle(self): file_paths.add(symlink_path) symlink_basename = os.path.basename(symlink_path) file.create_launcher(self.working_directory, self.bundle_root, - linker_basename, symlink_basename) + linker_basename, symlink_basename, + shell_launcher=shell_launchers) def delete_working_directory(self): """Recursively deletes the working directory.""" diff --git a/src/exodus_bundler/cli.py b/src/exodus_bundler/cli.py index ee79adf..61d1f0a 100644 --- a/src/exodus_bundler/cli.py +++ b/src/exodus_bundler/cli.py @@ -74,6 +74,10 @@ def parse_args(args=None, namespace=None): ), ) + parser.add_argument('--shell-launchers', action='store_true', help=( + 'Force the use of shell launchers instead of attempting to compile statically linked ones.' + )) + parser.add_argument('-t', '--tarball', action='store_true', help=( 'Creates a tarball for manual extraction instead of an installation script. ' 'Note that this will change the output extension from ".sh" to ".tgz".' From cfd5499ed30e30f04a5a653110e169289c4d937b Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Mon, 5 Mar 2018 11:08:23 -0500 Subject: [PATCH 097/101] Use the `--shell-launchers` flag during self-bundling. --- .circleci/config.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6c3431a..eee47c7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -93,15 +93,12 @@ jobs: sudo musl-gcc -static -O3 exodus.c -o exodus sudo chmod a+x /exodus - # The compiler won't bundle correctly, so we can at least temporarily move this and then be able to create bash launchers. - sudo mv /usr/bin/musl-gcc /tmp/musl-gcc.bck sudo mv /etc/ld.so.cache /tmp/ld.so.cache.bck export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:/usr/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} - strace -f /usr/bin/python /usr/local/bin/exodus -q /usr/bin/python -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --tar --output /home/circleci/exodus/exodus-x64.tgz - strace -f /usr/bin/python /usr/local/bin/exodus -q /usr/bin/python -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --output /home/circleci/exodus/exodus-x64.sh + strace -f /usr/bin/python /usr/local/bin/exodus --shell-launchers -q /usr/bin/python -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --tar --output /home/circleci/exodus/exodus-x64.tgz + strace -f /usr/bin/python /usr/local/bin/exodus --shell-launchers -q /usr/bin/python -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --output /home/circleci/exodus/exodus-x64.sh - sudo mv /tmp/musl-gcc.bck /usr/bin/musl-gcc sudo mv /tmp/ld.so.cache.bck /etc/ld.so.cache - store_artifacts: path: htmlcov From f3678da1dda0e2dd5c34f6f31805ec709df4e6cf Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Mon, 5 Mar 2018 11:08:39 -0500 Subject: [PATCH 098/101] Fix linting whitespace issue. --- tests/test_input_parsing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_input_parsing.py b/tests/test_input_parsing.py index f19c774..1903d02 100644 --- a/tests/test_input_parsing.py +++ b/tests/test_input_parsing.py @@ -88,6 +88,7 @@ def test_extract_strace_paths(): assert path in extracted_paths, \ '"%s" should be present in the extracted paths.' % path + def test_strip_pid_prefix(): line = ( '[pid 655] execve("/usr/bin/musl-gcc", ["/usr/bin/musl-gcc", "-static", "-O3", ' From 8ac0c2d065af3729ba7fdd6bbc4ee798ee593ce0 Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Mon, 5 Mar 2018 11:10:30 -0500 Subject: [PATCH 099/101] Raise a `CompilerNotFoundError` when `--shell-launchers` is specified. --- src/exodus_bundler/bundling.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index eac85e4..61ecfb6 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -537,6 +537,9 @@ def create_launcher(self, working_directory, bundle_root, linker_basename, symli # Try a c launcher first and fallback. try: + if shell_launcher: + raise CompilerNotFoundError() + launcher_content = construct_binary_launcher( linker=linker, library_path=library_path, executable=executable) with open(source_path, 'wb') as f: From b2fc939d28774d63301d8416de331e0a7ae44efb Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Mon, 5 Mar 2018 11:15:23 -0500 Subject: [PATCH 100/101] Fix: wasn't passing `shell_launchers` along as an argument. --- src/exodus_bundler/bundling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index 61ecfb6..50c2ba0 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -123,7 +123,7 @@ def create_unpackaged_bundle(executables, rename=[], chroot=None, add=[], no_sym if file: file.no_symlink = True - bundle.create_bundle() + bundle.create_bundle(shell_launchers=shell_launchers) return bundle.working_directory except: # noqa: E722 From 38a511bbd1843b21c35d41e7e076d84a4435f58e Mon Sep 17 00:00:00 2001 From: Evan Sangaline Date: Mon, 5 Mar 2018 11:15:57 -0500 Subject: [PATCH 101/101] Don't display the fallback warning when `--shell-launchers` is specified. --- src/exodus_bundler/bundling.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/exodus_bundler/bundling.py b/src/exodus_bundler/bundling.py index 50c2ba0..aa7ef34 100644 --- a/src/exodus_bundler/bundling.py +++ b/src/exodus_bundler/bundling.py @@ -545,10 +545,11 @@ def create_launcher(self, working_directory, bundle_root, linker_basename, symli with open(source_path, 'wb') as f: f.write(launcher_content) except CompilerNotFoundError: - logger.warn(( - 'Installing either the musl or diet C libraries will result in more efficient ' - 'launchers (currently using bash fallbacks instead).' - )) + if not shell_launcher: + logger.warn(( + 'Installing either the musl or diet C libraries will result in more efficient ' + 'launchers (currently using bash fallbacks instead).' + )) launcher_content = construct_bash_launcher( linker=linker, library_path=library_path, executable=executable) with open(source_path, 'w') as f: