Merge pull request #412 from NaturalHistoryMuseum/feature/408-bloat

Feature/408 bloat

.gitignore

@@ -11,7 +11,6 @@ MANIFEST
 *spec
 !inselect.spec
 !read_barcodes.spec
-!segment.spec
 *tar.gz
 *dmg
 .directory

CHANGELOG.md

@@ -2,6 +2,7 @@ This is an overview of major changes. Refer to the git repository for a full log
 
 Version 0.1.35
 -------------
+- #408 Installer bloat
 - #407 Performance impact of 2to3
 - #406 Show Boxes view on creating new document
 - #405 Remove qtpy

build.sh

@@ -38,15 +38,32 @@ echo Wheel build
 mv dist/inselect-*.whl .
 
 if [[ "$OSTYPE" == "darwin"* ]]; then
+    # Modules to be excluded - .spec files read this env var
+    # See https://github.com/pyinstaller/pyinstaller/wiki/Recipe-remove-tkinter-tcl
+    # for details of excluding all of the tcl, tk, tkinter shat
+    export EXCLUDE_MODULES="2to3 elementtree FixTk PIL._imagingtk ssl tcl tk _tkinter tkinter Tkinter"
+
     # Scripts that have additional requirements in their own spec files
-    for script in inselect read_barcodes segment; do
+    for script in inselect read_barcodes; do
         pyinstaller --clean $script.spec
     done
 
-    # Scripts for which the .spec file can be generated
-    for script in export_metadata ingest save_crops; do
+    # Format excludes for pyinstaller command line
+    export EXCLUDE_MODULES=`python -c "print(' '.join('--exclude-module {0}'.format(e) for e in '$EXCLUDE_MODULES'.split(' ')))"`
+
+    # export_metadata uses neither cv2 nor numpy
+    pyinstaller --onefile --exclude-module cv2 --exclude-module numpy \
+        $EXCLUDE_MODULES inselect/scripts/export_metadata.py
+
+    pyinstaller --onefile $EXCLUDE_MODULES \
+        --hidden-import sklearn.neighbors.typedefs \
+        --hidden-import sklearn.neighbors.dist_metrics \
+        inselect/scripts/segment.py
+
+    # Other scripts
+    for script in ingest save_crops; do
         rm -rf $script.spec
-        pyinstaller --onefile --hidden-import numpy inselect/scripts/$script.py
+        pyinstaller --onefile $EXCLUDE_MODULES inselect/scripts/$script.py
     done
 
     # Add a few items to the PropertyList file generated by PyInstaller

inselect.spec

@@ -1,4 +1,5 @@
 # -*- mode: python -*-
+import os
 import sys
 
 from pathlib import Path
@@ -8,7 +9,12 @@ from pyzbar import pyzbar
 
 block_cipher = None
 
+# See https://github.com/pyinstaller/pyinstaller/wiki/Recipe-remove-tkinter-tcl
+# for details of excluding all of the tcl, tk, tkinter shat
+sys.modules['FixTk'] = None
 
+
+print(os.getenv('EXCLUDE_MODULES', []).split(' '))
 a = Analysis(
     ['inselect/scripts/inselect.py'],
     pathex=[str(Path('.').absolute())],
@@ -19,7 +25,7 @@ a = Analysis(
     hiddenimports=['sklearn.neighbors.typedefs'],
     hookspath=[],
     runtime_hooks=[],
-    excludes=[],
+    excludes=os.getenv('EXCLUDE_MODULES', []).split(' '),
     win_no_prefer_redirects=False,
     win_private_assemblies=False,
     cipher=block_cipher
@@ -32,7 +38,6 @@ a.binaries += TOC([
     for dep in pylibdmtx.EXTERNAL_DEPENDENCIES + pyzbar.EXTERNAL_DEPENDENCIES
 ])
 
-
 ICON = 'icons/inselect.icns'
 
 pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)

inselect/gui/main_window.py

@@ -423,6 +423,8 @@ def open_document(self, path=None, document=None):
 
         self.document = document
         self.document_path = path
+        # TODO Need to roll back if an exception is raised initialising the
+        # model
         self.model.from_document(self.document)
 
         self.time_doc_opened = datetime.utcnow()

read_barcodes.spec

@@ -1,4 +1,6 @@
 # -*- mode: python -*-
+import os
+
 from pathlib import Path
 
 from pylibdmtx import pylibdmtx
@@ -15,7 +17,7 @@ a = Analysis(
     hiddenimports=['numpy'],
     hookspath=[],
     runtime_hooks=[],
-    excludes=[],
+    excludes=os.getenv('EXCLUDE_MODULES', []).split(' '),
     win_no_prefer_redirects=False,
     win_private_assemblies=False,
     cipher=block_cipher

requirements.pip

@@ -5,7 +5,7 @@ numpy==1.11.2
 Pillow==3.4.2
 psutil==5.0.0
 pylibdmtx==0.1.5
-PyQt5>=5.6
+PyQt5==5.7
 python-dateutil==2.6.0
 pytz==2016.7
 pywin32==220; sys_platform == 'win32'

setup.py

@@ -64,94 +64,132 @@
         'Topic :: Scientific/Engineering :: Bio-Informatics'
         'Programming Language :: Python :: 3.5',
     ],
-    'win32': {
-        'executables': [
-            {
-                'script': 'inselect/scripts/inselect.py',
-                'targetName': 'inselect.exe',
-                'icon': 'icons/inselect.ico',
-                'base': 'Win32GUI',
-                'shortcutName': 'Inselect',     # See http://stackoverflow.com/a/15736406
-                'shortcutDir': 'ProgramMenuFolder'
-            }
-        ] + [
-            {
-                'script': 'inselect/scripts/{0}.py'.format(script),
-                'targetName': '{0}.exe'.format(script),
-                'icon': 'icons/inselect.ico',
-                'base': 'Console'
-            }
-            for script in SCRIPTS
-        ],
-        # Strings in braces within 'include_files' tuples expanded in cx_setup
-        'include_files': [
-            # Evil, evil, evil
-            # cx_Freeze breaks pywintypes and pythoncom on Python 3.5
-            # https://bitbucket.org/anthony_tuininga/cx_freeze/issues/194/error-with-frozen-executable-using-35-and
-            ('{environment_root}/Lib/site-packages/win32/lib/pywintypes.py', 'pywintypes.py'),
-            ('{environment_root}/Lib/site-packages/pythoncom.py', 'pythoncom.py'),
-            ('{environment_root}/Library/bin/mkl_core.dll', 'mkl_core.dll'),
-            ('{environment_root}/Library/bin/mkl_intel_thread.dll', 'mkl_intel_thread.dll'),
-            ('{environment_root}/Library/bin/libiomp5md.dll', 'libiomp5md.dll'),
-            ('{project_root}/inselect/gui/inselect.qss', 'inselect.qss'),
-        ],
-        'extra_packages': ['win32com.gen_py', 'win32timezone'],
-        'excludes': [
-            'Tkinter', 'ttk', 'Tkconstants', 'tcl', '_ssl',
-        ]
-    }
 }
 
 
 def setuptools_setup():
     """setuptools setup"""
     from setuptools import setup
-    setup(**{k: v for k, v in setup_data.items() if 'win32' != k})
+    setup(**setup_data)
+
+
+def _qt_files(site_packages):
+    """Returns a list of tuples (src, dest) of Qt dependencies to be installed.
+    Elements are instances of Path.
+    site_packages should be an instance of Path to the site-packages directory.
+
+    IF we leave cx_Freeze to do its thing then the entirety of PyQt5, Qt5 and
+    uic are included in the installer. The only way to avoid horrible bloat is
+    to hand-tune which files we include.
+
+    This whole system is fucked beyond belief.
+    """
+    from pathlib import Path
+
+    return [
+        # Qt DLLs
+        (
+            site_packages.joinpath('PyQt5/Qt/bin').joinpath(dep),
+            dep
+        )
+        for dep in ('Qt5Core.dll', 'Qt5Gui.dll', 'Qt5Widgets.dll')
+    ] + [
+        # Qt plugins
+        (
+            site_packages.joinpath('PyQt5/Qt/plugins/platforms').joinpath(dep),
+            Path('platforms').joinpath(dep)
+        )
+        for dep in ('qwindows.dll',)
+    ] + [
+        # PyQt extension modules
+        (
+            site_packages.joinpath('PyQt5').joinpath(dep),
+            Path('PyQt5').joinpath(dep)
+        )
+        for dep in ('__init__.py', 'Qt.pyd', 'QtCore.pyd', 'QtGui.pyd', 'QtWidgets.pyd')
+    ]
 
 
 def cx_setup():
     """cx_Freeze setup. Used for building Windows installers"""
-    from cx_Freeze import setup, Executable
-    from distutils.sysconfig import get_python_lib
+    import scipy
+
     from pathlib import Path
+    from distutils.sysconfig import get_python_lib
 
-    # Set paths to include files
-    format_strings = {
-        'site_packages': get_python_lib(),
-        'environment_root': Path(sys.executable).parent,
-        'project_root': Path(__file__).parent,
-    }
-    include_files = [
-        (source.format(**format_strings), destination)
-        for source, destination in setup_data['win32']['include_files']
-    ]
+    from cx_Freeze import setup, Executable
 
-    # DLLs that are not detected because they are loaded by ctypes
     from pylibdmtx import pylibdmtx
     from pyzbar import pyzbar
-    include_files += [
+
+    # Useful paths
+    environment_root = Path(sys.executable).parent
+    site_packages = Path(get_python_lib())
+    project_root = Path(__file__).parent
+
+    # Files as tuples (source, dest)
+    include_files = [
+        # Evil, evil, evil
+        # cx_Freeze breaks pywintypes and pythoncom on Python 3.5
+        # https://bitbucket.org/anthony_tuininga/cx_freeze/issues/194/error-with-frozen-executable-using-35-and
+        (site_packages.joinpath('win32/lib/pywintypes.py'), 'pywintypes.py'),
+        (site_packages.joinpath('pythoncom.py'), 'pythoncom.py'),
+
+        # Binary dependencies that are not detected
+        (environment_root.joinpath('Library/bin/mkl_core.dll'), 'mkl_core.dll'),
+        (environment_root.joinpath('Library/bin/mkl_intel_thread.dll'), 'mkl_intel_thread.dll'),
+        (environment_root.joinpath('Library/bin/libiomp5md.dll'), 'libiomp5md.dll'),
+
+        # Stylesheet
+        (project_root.joinpath('inselect/gui/inselect.qss'), 'inselect.qss'),
+    ] + [
+        # DLLs that are not detected because they are loaded by ctypes
         (dep._name, Path(dep._name).name)
         for dep in pylibdmtx.EXTERNAL_DEPENDENCIES + pyzbar.EXTERNAL_DEPENDENCIES
-    ]
+    ] + _qt_files(site_packages)
 
-    # scipy
-    # http://stackoverflow.com/questions/32694052/scipy-and-cx-freeze-error-importing-scipy-you-cannot-import-scipy-while-being
-    import scipy
+    # Convert instances of Path to strs
+    include_files = [(str(source), str(dest)) for source, dest in include_files]
+
+    # Directories as strings
     include_files += [
+        # Fixes scipy freeze
+        # http://stackoverflow.com/a/32822431/1773758
         str(Path(scipy.__file__).parent),
     ]
 
-    # Setup
+    # Packages to exclude.
+    exclude_packages = [
+        str(p.relative_to(site_packages)).replace('\\', '.') for p in
+        site_packages.rglob('*/tests')
+    ]
+
     setup(
         name=setup_data['name'],
         version=setup_data['version'],
         options={
             'build_exe': {
-                'packages': (
-                    setup_data['packages'] +
-                    setup_data['win32']['extra_packages']
-                ),
-                'excludes': setup_data['win32']['excludes'],
+                'packages':
+                    setup_data.get('packages', []) + [
+                        'urllib', 'sklearn.neighbors', 'win32com.gen_py',
+                        'win32timezone',
+                    ],
+                'excludes': [
+                    # '_bz2',    # Required by sklearn
+                    '_decimal', '_elementtree', '_hashlib', '_lzma',
+                    '_ssl', 'curses',
+                    'distutils', 'email', 'http', 'lib2to3', 'mock', 'nose',
+                    'PyQt5',
+                    # 'pydoc',    # Required by sklearn
+                    'tcl', 'Tkinter', 'ttk', 'Tkconstants',
+                    # 'unittest',    # Required by numpy.core.multiarray
+                    'win32com.HTML', 'win32com.test', 'win32evtlog', 'win32pdh',
+                    'win32trace', 'win32ui', 'win32wnet',
+                    'xml', 'xmlrpc',
+                    'inselect.tests',
+                ] + exclude_packages,
+                'includes': [
+                ],
                 'include_files': include_files,
                 'include_msvcr': True,
                 'optimize': 2,
@@ -161,8 +199,23 @@ def cx_setup():
             }
         },
         executables=[
-            Executable(**i) for i in setup_data['win32']['executables']
-        ]
+            Executable(
+                script='inselect/scripts/inselect.py',
+                targetName='inselect.exe',
+                icon='icons/inselect.ico',
+                base='Win32GUI',
+                shortcutName='Inselect',     # See http://stackoverflow.com/a/15736406
+                shortcutDir='ProgramMenuFolder'
+            )
+        ] + [
+            Executable(
+                script='inselect/scripts/{0}.py'.format(script),
+                targetName='{0}.exe'.format(script),
+                icon='icons/inselect.ico',
+                base='Console'
+            )
+            for script in SCRIPTS
+        ],
     )