Merge pull request #98 from gem/new-win

Build the Windows installer in a proper, modular way

installers/windows/README.md

@@ -39,22 +39,8 @@ $ docker run -e GEM_SET_OUTPUT=zip -v $(pwd):/io -t -i --rm wine /io/docker/buil
 
 `$GEM_SET_OUTPUT` can be `exe` (default), `zip` or `exe zip`.
 
-### Manual installation 
-
-#### Setup WINE prefix
-- `export WINEPREFIX=/home/user/path/to/my/prefix`
-- `cd src`
-- `wget https://www.python.org/ftp/python/2.7.13/python-2.7.13.amd64.msi`
-- `wine msiexec /i python-2.7.12.amd64.msi`
-- `wine pip install wheel`
-
-#### OpenQuake
-- `cd src`
-- `git clone [-b mybranch] https://github.com/gem/oq-hazardlib.git`
-- `git clone [-b mybranch] https://github.com/gem/oq-engine.git`
-
 #### Libs
-- Download `py` and `py27` from the internal repo and put it into `src`
+- Download `py` and `py35` from the internal repo and put it into `src`
 
 #### Run the builder
 - `docker/build.sh`

installers/windows/nsis/docker/Dockerfile

@@ -34,9 +34,9 @@ ENV PATH /opt/wine-stable/bin:$PATH
 ENV WINEDEBUG -all 
 WORKDIR $HOME
 
-RUN wget -q http://cdn.ftp.openquake.org/windows/thirdparties/nsis-3.01-setup.exe && \
-    wget -q http://cdn.ftp.openquake.org/windows/thirdparties/python-3.5.4.zip && \
-    wget -q http://cdn.ftp.openquake.org/windows/thirdparties/get-pip.py
+RUN wget -q --cache=off http://cdn.ftp.openquake.org/windows/thirdparties/nsis-3.01-setup.exe && \
+    wget -q --cache=off http://cdn.ftp.openquake.org/windows/thirdparties/python-3.5.4-win64.zip && \
+    wget -q --cache=off http://cdn.ftp.openquake.org/windows/thirdparties/get-pip.py
 
 RUN wineboot && sleep 10
 

installers/windows/nsis/docker/build.sh

@@ -44,9 +44,16 @@ if [[ $GEM_SET_RELEASE =~ ^[0-9]+$ ]]; then
     PKG_REL=$GEM_SET_RELEASE
 fi
 
+function fix-scripts {
+	for f in $*; do
+		sed -i 's/z:\\io\\python-dist\\python3.5\\//g' "$f"
+	done
+}
+
+
 # Default software distribution
 PY="3.5.4"
-PY_ZIP="python-${PY}.zip"
+PY_ZIP="python-${PY}-win64.zip"
 PIP="get-pip.py"
 
 
@@ -57,8 +64,6 @@ cd $DIR && echo "Working in: $(pwd)"
 rm -Rf *.zip *.exe
 rm -Rf src/oq-*
 rm -Rf python-dist/python3.5/*
-rm -Rf python-dist/Lib/*
-rm -Rf python-dist/Scripts/*
 rm -Rf demos/*
 
 cd src
@@ -77,27 +82,33 @@ if [ ! -f $PIP ]; then
 fi
 wine ../python-dist/python3.5/python.exe $PIP
 
+# Extract wheels to be included in the installation
+echo "Extracting python wheels"
+wine ../python-dist/python3.5/python.exe -m pip -q install --disable-pip-version-check --no-warn-script-location --force-reinstall --ignore-installed --upgrade --no-deps --no-index py/*.whl py35/*.whl
 
 ## Core apps
 echo "Downloading core apps"
 for app in oq-engine; do
     git clone -q -b $OQ_BRANCH --depth=1 https://github.com/gem/${app}.git
-    wine ../python-dist/python3.5/Scripts/pip3.exe -q wheel --disable-pip-version-check --no-deps ./${app}
+    wine ../python-dist/python3.5/python.exe -m pip -q wheel --disable-pip-version-check --no-deps -w ../oq-dist/engine ./${app}
 done
 
 ## Standalone apps
 echo "Downloading standalone apps"
 for app in oq-platform-standalone oq-platform-ipt oq-platform-taxtweb oq-platform-taxonomy; do
     git clone -q -b $TOOLS_BRANCH --depth=1 https://github.com/gem/${app}.git
-    wine ../python-dist/python3.5/Scripts/pip3.exe -q wheel --disable-pip-version-check --no-deps ./${app}
+    wine ../python-dist/python3.5/python.exe -m pip -q wheel --disable-pip-version-check --no-deps -w ../oq-dist/tools ./${app}
 done
 
-# Extract wheels to be included in the installation
-echo "Extracting python wheels"
-wine ../python-dist/python3.5/Scripts/pip3.exe -q install --disable-pip-version-check --force-reinstall --ignore-installed --upgrade --no-deps --no-index --prefix ../python-dist py/*.whl py35/*.whl openquake.*.whl oq_platform*.whl
+cd $DIR/oq-dist
+for d in *; do
+	ls $d | grep whl > $d/index.txt
+done
 
 cd $DIR
 
+fix-scripts ${DIR}/python-dist/python3.5/Scripts/*.exe
+
 ini_vers="$(cat src/oq-engine/openquake/baselib/__init__.py | sed -n "s/^__version__[  ]*=[    ]*['\"]\([^'\"]\+\)['\"].*/\1/gp")"
 git_time="$(date -d @$(git -C src/oq-engine log --format=%ct -1) '+%y%m%d%H%M')"
 
@@ -120,21 +131,21 @@ if [ ! -f OpenQuake\ manual.pdf ]; then
     wget -O- https://ci.openquake.org/job/builders/job/pdf-builder/lastSuccessfulBuild/artifact/oq-engine/doc/manual/oq-manual.pdf > OpenQuake\ manual.pdf
 fi
 
-# Make sure that Lib -> lib to be more consistent with naming
-mv ${DIR}/python-dist/Lib ${DIR}/python-dist/lib || true
-
 if [[ $OQ_OUTPUT = *"exe"* ]]; then
     echo "Generating NSIS installer"
     wine ${HOME}/.wine/drive_c/Program\ Files\ \(x86\)/NSIS/makensis /V4 installer.nsi
 fi
 
 if [[ $OQ_OUTPUT = *"zip"* ]]; then
+    cd $DIR/oq-dist
+    for d in *; do
+		wine ../python-dist/python3.5/python.exe -m pip -q install --disable-pip-version-check --no-warn-script-location --force-reinstall --ignore-installed --upgrade --no-deps --no-index $d/*.whl
+    done
+    fix-scripts ${DIR}/python-dist/python3.5/Scripts/*.exe
     echo "Generating ZIP archive"
     ZIP="OpenQuake_Engine_${ini_vers}_${git_time}.zip"
-    zip -qr $DIR/${ZIP} *.bat *.pdf demos README.html LICENSE.txt
-    cd $DIR/dist
-    zip -qr $DIR/${ZIP} bin
-    cp __init__.py ${DIR}/python-dist/lib/site-packages/openquake
     cd $DIR/python-dist
-    zip -qr $DIR/${ZIP} lib python3.5
+    zip -qr $DIR/${ZIP} python3.5
+    cd $DIR
+    zip -qr $DIR/${ZIP} *.bat *.pdf demos README.html LICENSE.txt
 fi

installers/windows/nsis/installer.nsi.tmpl

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # vim: tabstop=2 shiftwidth=2 softtabstop=2 syntax=nsis
 #
-# Copyright (C) 2016-2017 GEM Foundation
+# Copyright (C) 2016-2018 GEM Foundation
 #
 # OpenQuake is free software: you can redistribute it and/or modify it
 # under the terms of the GNU Affero General Public License as published
@@ -87,12 +87,9 @@ Section "!Core Files" SecCore
   SetShellVarContext all
 
   SetOutPath "$INSTDIR"
-  File "openquake.ico"
   File "openquake_black.ico"
-  File "LICENSE.txt"
-  File "README.html"
   File "oq-console.bat"
-  File "oq-server.bat"
+  File "LICENSE.txt"
 
   SetOutPath "$INSTDIR\python3.5"
   File /r "python-dist\python3.5\*.*"
@@ -103,26 +100,42 @@ Section "!Core Files" SecCore
   File "dist\msvcrt\x64\*.dll"
   skip_msvcrt:
 
-  SetOutPath "$INSTDIR\bin"
-  File /r "dist\bin\*.*"
+  SetOutPath "$PLUGINSDIR\oq-dist"
+  File /r "oq-dist\*.*"
+SectionEnd
 
-  SetOutPath "$INSTDIR\lib"
-  File /r /x "python-dist\lib\site-packages\oq_platform*" /x "python-dist\lib\site-packages\openquakeplatform*" "python-dist\lib\*.*"
+Section "!OpenQuake Engine" SecEngine
+  SectionIn RO
+  SetOutPath "$INSTDIR"
+  File "openquake.ico"
+  File "oq-server.bat"
+  File "README.html"
 
-  SetOutPath "$INSTDIR\lib\site-packages\openquake"
-  File "dist\__init__.py"
+  SetOutPath "$PLUGINSDIR\oq-dist\engine"
+  DetailPrint "Installing the OpenQuake Engine..."
+  nsExec::ExecToLog '$INSTDIR\python3.5\python.exe -m pip -q install --no-deps --no-index --no-warn-script-location -r "index.txt"'
 SectionEnd
 
 Section "OpenQuake Tools" SecTools
-  SetOutPath "$INSTDIR\lib\site-packages"
-  File /r "python-dist\lib\site-packages\oq_platform*" "python-dist\lib\site-packages\openquakeplatform*"
+  SetOutPath "$PLUGINSDIR\oq-dist\tools"
+  DetailPrint "Installing the OpenQuake Tools..."
+  nsExec::ExecToLog '$INSTDIR\python3.5\python.exe -m pip -q install --no-deps --no-index --no-warn-script-location -r "index.txt"'
 SectionEnd
 
-Section "OpenQuake Engine demos and documentation" SecOQ
+Section "OpenQuake Engine demos and documentation" SecDocDemo
+  # Install manual and demos in 'Public\MyDocuments'
+  # keep disabled for now. It may be dangerous.
+  # SetOutPath "$DOCUMENTS\OpenQuake"
+  # File "OpenQuake manual.pdf"
+  # SetOutPath "$DOCUMENTS\OpenQuake\demos"
+  # File /r /x ".gitignore" "demos\*.*"
+
+  # Keep a copy also in the installation folder
   SetOutPath "$INSTDIR"
   File "OpenQuake manual.pdf"
   SetOutPath "$INSTDIR\demos"
   File /r /x ".gitignore" "demos\*.*"
+
 SectionEnd
 
 Section "OpenQuake Engine menu icons" SecMenu
@@ -143,9 +156,10 @@ SectionEnd
 
 Section -post
   ; Byte-compile Python files.
+  RMDir /r "$INSTDIR\oq-dist"
   DetailPrint "Byte-compiling Python modules..."
   nsExec::ExecToLog '$INSTDIR\python3.5\python.exe -m compileall -qq "$INSTDIR"'
-  FileOpen $0 "$INSTDIR\lib\pycache" w
+  FileOpen $0 "$INSTDIR\python3.5\pycached" w
   FileClose $0
 
   WriteUninstaller $INSTDIR\uninstall.exe
@@ -175,8 +189,6 @@ Section "Uninstall"
   SetShellVarContext all
   Delete $INSTDIR\uninstall.exe
   Delete "$INSTDIR\${PRODUCT_ICON}"
-  RMDir /r "$INSTDIR\bin"
-  RMDir /r "$INSTDIR\lib"
   RMDir /r "$INSTDIR\python3.5"
   ; Uninstall files
     Delete "$INSTDIR\README.html"

installers/windows/nsis/oq-console.bat

@@ -1,16 +1,13 @@
 @echo off
 setlocal
 set mypath=%~dp0
-set PATH=%mypath%\bin;%mypath%\python3.5;%PATH%
-set PYTHONPATH=%mypath%\lib\site-packages
+set PATH=%mypath%\python3.5;%mypath%\python3.5\Scripts;%PATH%
 
-doskey pip=python.exe -m pip $*
-
-if not exist lib\pycache (
+if not exist python3.5\pycached (
    echo Building python cache. This may take a while.
    echo Please wait ...
    python.exe -m compileall -qq .
-   copy /y nul lib\pycache >nul
+   copy /y nul python3.5\pycached >nul
 )
 
 echo OpenQuake environment loaded

installers/windows/nsis/oq-dist/.gitignore

@@ -0,0 +1,2 @@
+**/*.whl
+**/*.txt

installers/windows/nsis/oq-server.bat

@@ -1,16 +1,15 @@
 @echo off
 setlocal
 set mypath=%~dp0
-set PATH=%mypath%\bin;%mypath%\python3.5;%PATH%
-set PYTHONPATH=%mypath%\lib\site-packages
+set PATH=%mypath%\python3.5;%mypath%\python3.5\Scripts;%PATH%
 set OQ_HOST=localhost
 set OQ_PORT=8800
 
-if not exist lib\pycache (
+if not exist python3.5\pycached (
    echo Building python cache. This may take a while.
    echo Please wait ...
    python.exe -m compileall -qq .
-   copy /y nul lib\pycache >nul
+   copy /y nul python3.5\pycached >nul
 )
 
 echo Starting the server.