Permalink
Browse files

fix pyinstaller

amongst other things, fix #1746
  • Loading branch information...
1 parent 568f40c commit 3154dc87fd88fb9b4a7db3e2752ec7e6d6cbd303 @mhils mhils committed on GitHub Nov 27, 2016
View
@@ -24,17 +24,20 @@ install:
test_script:
- ps: "tox -- --cov mitmproxy --cov pathod -v"
+ - ps: |
+ $Env:VERSION = $(python mitmproxy/version.py)
+ $Env:SKIP_MITMPROXY = "python -c `"print('skip mitmproxy')`""
+ tox -e wheel -- https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl
+ tox -e rtool -- bdist
deploy_script:
+ # we build binaries on every run, but we only upload them for master snapshots or tags.
ps: |
if(
($Env:TOXENV -match "py35") -and
- (($Env:APPVEYOR_REPO_BRANCH -match "master") -or ($Env:APPVEYOR_REPO_TAG -match "true"))
- ) {
- pip install -U virtualenv
- .\dev.ps1
- cmd /c "python -u .\release\rtool.py bdist 2>&1"
- python -u .\release\rtool.py upload-snapshot --bdist --wheel
+ (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true"))
+ ) {
+ tox -e rtool -- upload-snapshot --bdist --wheel
}
cache:
View
@@ -51,18 +51,21 @@ install:
fi
- pip install tox
-script: tox -- --cov mitmproxy --cov pathod -v
+script:
+ - |
+ tox -- --cov mitmproxy --cov pathod -v
+ if [[ $BDIST == "1" ]]
+ then
+ git fetch --unshallow --tags
+ tox -e rtool -- bdist
+ fi
after_success:
+ # we build binaries on every run, but we only upload them for master snapshots or tags.
- |
- if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
+ if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "pyinstaller" || $TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
then
- git fetch --unshallow
- ./dev.sh 3.5
- source venv3.5/bin/activate
- pip install -e ./release
- python -u ./release/rtool.py bdist
- python -u ./release/rtool.py upload-snapshot --bdist
+ tox -e rtool -- upload-snapshot --bdist
fi
notifications:
View
@@ -1,7 +1,7 @@
$ErrorActionPreference = "Stop"
$VENV = ".\venv"
-virtualenv $VENV --always-copy
+python3 -m venv $VENV --copies
& $VENV\Scripts\activate.ps1
python -m pip install --disable-pip-version-check -U pip
@@ -1,5 +0,0 @@
-from mitmproxy.tools import web
-from mitmproxy.tools import console
-from mitmproxy.tools import dump
-
-__all__ = ["web", "console", "dump"]
@@ -15,7 +15,7 @@ def push(self, subpath):
"""
Change the data object to a path relative to the module.
"""
- dirname = os.path.join(self.dirname, subpath)
+ dirname = os.path.normpath(os.path.join(self.dirname, subpath))
ret = Data(self.name)
ret.dirname = dirname
return ret
@@ -27,7 +27,7 @@ def path(self, path):
This function will raise ValueError if the path does not exist.
"""
- fullpath = os.path.join(self.dirname, path)
+ fullpath = os.path.normpath(os.path.join(self.dirname, path))
if not os.path.exists(fullpath):
raise ValueError("dataPath: %s does not exist." % fullpath)
return fullpath
@@ -2,3 +2,6 @@
VERSION = ".".join(str(i) for i in IVERSION)
PATHOD = "pathod " + VERSION
MITMPROXY = "mitmproxy " + VERSION
+
+if __name__ == "__main__":
+ print(VERSION)
@@ -0,0 +1,43 @@
+# Taken from the latest pyinstaller master on 2016-11-27 (0729a2b).
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2005-2016, PyInstaller Development Team.
+#
+# Distributed under the terms of the GNU General Public License with exception
+# for distributing bootloader.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+
+"""
+Hook for cryptography module from the Python Cryptography Authority.
+"""
+
+import os.path
+import glob
+
+from PyInstaller.compat import EXTENSION_SUFFIXES
+from PyInstaller.utils.hooks import collect_submodules, get_module_file_attribute
+from PyInstaller.utils.hooks import copy_metadata
+
+# get the package data so we can load the backends
+datas = copy_metadata('cryptography')
+
+# Add the backends as hidden imports
+hiddenimports = collect_submodules('cryptography.hazmat.backends')
+
+# Add the OpenSSL FFI binding modules as hidden imports
+hiddenimports += collect_submodules('cryptography.hazmat.bindings.openssl') + ['_cffi_backend']
+
+
+# Include the cffi extensions as binaries in a subfolder named like the package.
+# The cffi verifier expects to find them inside the package directory for
+# the main module. We cannot use hiddenimports because that would add the modules
+# outside the package.
+binaries = []
+cryptography_dir = os.path.dirname(get_module_file_attribute('cryptography'))
+for ext in EXTENSION_SUFFIXES:
+ ffimods = glob.glob(os.path.join(cryptography_dir, '*_cffi_*%s*' % ext))
+ for f in ffimods:
+ binaries.append((f, 'cryptography'))
@@ -1,3 +1,3 @@
from PyInstaller.utils.hooks import collect_data_files
-datas = collect_data_files('mitmproxy.addons.onboardingapp')
+datas = collect_data_files('mitmproxy.addons.onboardingapp')
@@ -0,0 +1,3 @@
+from PyInstaller.utils.hooks import collect_data_files
+
+datas = collect_data_files('mitmproxy.tools.web')
View
@@ -8,10 +8,9 @@
import shlex
import shutil
import subprocess
-import sys
import tarfile
import zipfile
-from os.path import join, abspath, normpath, dirname, exists, basename
+from os.path import join, abspath, dirname, exists, basename
import click
import pysftp
@@ -20,8 +19,13 @@
# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
if platform.system() == "Windows":
VENV_BIN = "Scripts"
+ PYINSTALLER_ARGS = [
+ # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly.
+ "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
+ ]
else:
VENV_BIN = "bin"
+ PYINSTALLER_ARGS = []
# ZipFile and tarfile have slightly different APIs. Fix that.
if platform.system() == "Windows":
@@ -46,13 +50,9 @@ def Archive(name):
PYINSTALLER_DIST = join(BUILD_DIR, "binaries")
VENV_DIR = join(BUILD_DIR, "venv")
-VENV_PIP = join(VENV_DIR, VENV_BIN, "pip")
-VENV_PYINSTALLER = join(VENV_DIR, VENV_BIN, "pyinstaller")
# Project Configuration
VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py")
-PROJECT_NAME = "mitmproxy"
-PYTHON_VERSION = "py2.py3"
BDISTS = {
"mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
"pathod": ["pathoc", "pathod"]
@@ -62,20 +62,20 @@ def Archive(name):
TOOLS = [
tool
- for tools in BDISTS.values()
+ for tools in sorted(BDISTS.values())
for tool in tools
]
-def get_version() -> str:
- return runpy.run_path(VERSION_FILE)["VERSION"]
-
-
def git(args: str) -> str:
with chdir(ROOT_DIR):
return subprocess.check_output(["git"] + shlex.split(args)).decode()
+def get_version() -> str:
+ return runpy.run_path(VERSION_FILE)["VERSION"]
+
+
def get_snapshot_version() -> str:
last_tag, tag_dist, commit = git("describe --tags --long").strip().rsplit("-", 2)
tag_dist = int(tag_dist)
@@ -109,26 +109,12 @@ def archive_name(bdist: str) -> str:
def wheel_name() -> str:
- return "{project}-{version}-{py_version}-none-any.whl".format(
- project=PROJECT_NAME,
+ return "mitmproxy-{version}-py3-none-any.whl".format(
version=get_version(),
- py_version=PYTHON_VERSION
)
@contextlib.contextmanager
-def empty_pythonpath():
- """
- Make sure that the regular python installation is not on the python path,
- which would give us access to modules installed outside of our virtualenv.
- """
- pythonpath = os.environ.get("PYTHONPATH", "")
- os.environ["PYTHONPATH"] = ""
- yield
- os.environ["PYTHONPATH"] = pythonpath
-
-
-@contextlib.contextmanager
def chdir(path: str):
old_dir = os.getcwd()
os.chdir(path)
@@ -156,58 +142,8 @@ def contributors():
f.write(contributors_data.encode())
-@cli.command("wheel")
-def make_wheel():
- """
- Build wheel
- """
- with empty_pythonpath():
- if exists(DIST_DIR):
- shutil.rmtree(DIST_DIR)
-
- print("Creating wheel...")
- subprocess.check_call(
- [
- "python3", "./setup.py", "-q",
- "bdist_wheel", "--dist-dir", DIST_DIR, "--universal"
- ],
- cwd=ROOT_DIR
- )
-
- print("Creating virtualenv for test install...")
- if exists(VENV_DIR):
- shutil.rmtree(VENV_DIR)
- subprocess.check_call(["python3", "-m", "virtualenv", "-q", VENV_DIR])
-
- with chdir(DIST_DIR):
- print("Install wheel into virtualenv...")
- # lxml...
- if platform.system() == "Windows" and sys.version_info[0] == 3:
- subprocess.check_call(
- [VENV_PIP, "install", "-q", "https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl"]
- )
- subprocess.check_call([VENV_PIP, "install", "-q", wheel_name()])
-
- print("Running tools...")
- for tool in TOOLS:
- tool = join(VENV_DIR, VENV_BIN, tool)
- print("> %s --version" % tool)
- print(subprocess.check_output([tool, "--version"]).decode())
-
- print("Virtualenv available for further testing:")
- print("source %s" % normpath(join(VENV_DIR, VENV_BIN, "activate")))
-
-
@cli.command("bdist")
-@click.option("--use-existing-wheel/--no-use-existing-wheel", default=False)
-@click.argument(
- "pyinstaller_version",
- envvar="PYINSTALLER_VERSION",
- default="git+https://github.com/pyinstaller/pyinstaller.git@483c819d6a256b58db6740696a901bd41c313f0c"
-)
-@click.argument("setuptools_version", envvar="SETUPTOOLS_VERSION", default="setuptools>=25.1.0,!=25.1.1")
-@click.pass_context
-def make_bdist(ctx, use_existing_wheel, pyinstaller_version, setuptools_version):
+def make_bdist():
"""
Build a binary distribution
"""
@@ -216,37 +152,40 @@ def make_bdist(ctx, use_existing_wheel, pyinstaller_version, setuptools_version)
if exists(PYINSTALLER_DIST):
shutil.rmtree(PYINSTALLER_DIST)
- if not use_existing_wheel:
- ctx.invoke(make_wheel)
-
- print("Installing PyInstaller and setuptools...")
- subprocess.check_call([VENV_PIP, "install", "-q", pyinstaller_version, setuptools_version])
- print(subprocess.check_output([VENV_PIP, "freeze"]).decode())
+ os.makedirs(DIST_DIR, exist_ok=True)
- for bdist, tools in BDISTS.items():
+ for bdist, tools in sorted(BDISTS.items()):
with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
for tool in tools:
+ # We can't have a folder and a file with the same name.
+ if tool == "mitmproxy":
+ tool = "mitmproxy_main"
# This is PyInstaller, so it messes up paths.
# We need to make sure that we are in the spec folder.
with chdir(PYINSTALLER_SPEC):
print("Building %s binary..." % tool)
+ excludes = []
+ if tool != "mitmweb":
+ excludes.append("mitmproxy.tools.web")
+ if tool != "mitmproxy_main":
+ excludes.append("mitmproxy.tools.console")
subprocess.check_call(
[
- VENV_PYINSTALLER,
+ "pyinstaller",
"--clean",
"--workpath", PYINSTALLER_TEMP,
"--distpath", PYINSTALLER_DIST,
"--additional-hooks-dir", PYINSTALLER_HOOKS,
- # PyInstaller 3.2 does not handle Python 3.5's ucrt correctly.
- "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
"--onefile",
"--console",
"--icon", "icon.ico",
# This is PyInstaller, so setting a
# different log level obviously breaks it :-)
# "--log-level", "WARN",
- tool
]
+ + [x for e in excludes for x in ["--exclude-module", e]]
+ + PYINSTALLER_ARGS
+ + [tool]
)
# Delete the spec file - we're good without.
os.remove("{}.spec".format(tool))
@@ -255,6 +194,15 @@ def make_bdist(ctx, use_existing_wheel, pyinstaller_version, setuptools_version)
executable = join(PYINSTALLER_DIST, tool)
if platform.system() == "Windows":
executable += ".exe"
+
+ # Remove _main suffix from mitmproxy executable
+ if executable.startswith("mitmproxy_main"):
+ shutil.move(
+ executable,
+ executable.replace("_main", "")
+ )
+ executable = executable.replace("_main", "")
+
print("> %s --version" % executable)
print(subprocess.check_output([executable, "--version"]).decode())
@@ -306,7 +254,7 @@ def upload_snapshot(host, port, user, private_key, private_key_password, wheel,
if wheel:
files.append(wheel_name())
if bdist:
- for bdist in BDISTS.keys():
+ for bdist in sorted(BDISTS.keys()):
files.append(archive_name(bdist))
for f in files:
View
@@ -7,8 +7,6 @@
install_requires=[
"click>=6.2, <7.0",
"twine>=1.6.5, <1.9",
- "virtualenv>=14.0.5, <15.2",
- "wheel>=0.29.0, <0.30",
"pysftp==0.2.8",
],
entry_points={
File renamed without changes.
View
@@ -1,3 +1,2 @@
https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl; sys_platform == 'win32' and python_version == '3.5'
-e .[dev,examples,contentviews]
--e ./release
Oops, something went wrong.

0 comments on commit 3154dc8

Please sign in to comment.