Merge pull request #1037 from whalebot-helmsman/container-for-building

Container for building

Dockerfile

@@ -1,4 +1,22 @@
+ARG QT_MAJOR_VERSION=5
+ARG QT_MINOR_VERSION=14
+ARG QT_PATCH_VERSION=1
+# docker build has arguments but doesn't have variables
+# here we use arguments as variables
+ARG QT_SHORT_VERSION=${QT_MAJOR_VERSION}.${QT_MINOR_VERSION}
+ARG QT_FULL_VERSION=${QT_SHORT_VERSION}.${QT_PATCH_VERSION}
+
+ARG SPLASH_BUILD_PARALLEL_JOBS=8
+
+# =====================
+
 FROM byrnedo/alpine-curl as qt-downloader
+
+ARG QT_SHORT_VERSION
+ARG QT_FULL_VERSION
+ENV QT_SHORT_VERSION=${QT_SHORT_VERSION}
+ENV QT_FULL_VERSION=${QT_FULL_VERSION}
+
 COPY dockerfiles/splash/download-qt-installer.sh /tmp/download-qt-installer.sh
 RUN /tmp/download-qt-installer.sh /tmp/qt-installer.run
 
@@ -11,12 +29,16 @@ RUN /tmp/download-qt-installer.sh /tmp/qt-installer.run
 # =====================
 
 FROM byrnedo/alpine-curl as qtwebkit-downloader
+ARG WEBKIT_URL=https://github.com/qtwebkit/qtwebkit/releases/download/qtwebkit-5.212.0-alpha4/qtwebkit-Linux-RHEL_7_6-GCC-Linux-RHEL_7_6-X86_64.7z
+ENV WEBKIT_URL=${WEBKIT_URL}
 COPY dockerfiles/splash/download-qtwebkit.sh /tmp/download-qtwebkit.sh
 RUN /tmp/download-qtwebkit.sh /tmp/qtwebkit.7z
 
 # =====================
 
 FROM byrnedo/alpine-curl as pyqt5-downloader
+
+# need to update URLs here with new version of Qt
 COPY dockerfiles/splash/download-pyqt5.sh /tmp/download-pyqt5.sh
 RUN /tmp/download-pyqt5.sh /tmp/sip.tar.gz /tmp/pyqt5.tar.gz /tmp/webengine.tar.gz
 
@@ -52,12 +74,22 @@ COPY --from=qt-downloader /tmp/qt-installer.run /tmp/
 
 ARG DISPLAY
 
+ARG QT_MAJOR_VERSION
+ARG QT_MINOR_VERSION
+ARG QT_PATCH_VERSION
+ARG QT_SHORT_VERSION
+ARG QT_FULL_VERSION
+ENV QT_MAJOR_VERSION=${QT_MAJOR_VERSION}
+ENV QT_MINOR_VERSION=${QT_MINOR_VERSION}
+ENV QT_PATCH_VERSION=${QT_PATCH_VERSION}
+ENV QT_SHORT_VERSION=${QT_SHORT_VERSION}
+ENV QT_FULL_VERSION=${QT_FULL_VERSION}
+
 COPY dockerfiles/splash/qt-installer-noninteractive.qs /tmp/script.qs
 COPY dockerfiles/splash/run-qt-installer.sh /tmp/run-qt-installer.sh
 RUN /tmp/run-qt-installer.sh /tmp/qt-installer.run /tmp/script.qs
 
-# XXX: this needs to be updated if Qt is updated
-ENV PATH="/opt/qt-5.13/5.13.1/gcc_64/bin:${PATH}"
+ENV PATH="/opt/qt-${QT_SHORT_VERSION}/${QT_FULL_VERSION}/gcc_64/bin:${PATH}"
 
 # install qtwebkit
 COPY --from=qtwebkit-downloader /tmp/qtwebkit.7z /tmp/
@@ -66,6 +98,14 @@ RUN /tmp/install-qtwebkit.sh /tmp/qtwebkit.7z
 
 # =====================
 
+FROM qtbuilder as qtwebkitbuilder-base
+
+COPY dockerfiles/splash/install-qtwebkit-build-deps.sh /tmp/install-qtwebkit-build-deps.sh
+COPY dockerfiles/splash/create-package.sh /tmp/create-package.sh
+RUN /tmp/install-qtwebkit-build-deps.sh
+
+# =====================
+
 #FROM qtbuilder as qtwebkitbuilder
 #COPY --from=qtwebkit-downloader /tmp/qtwebkit.tar.xz /tmp/
 #
@@ -82,16 +122,24 @@ FROM qtbase as splash-base
 COPY dockerfiles/splash/install-system-splash-deps.sh /tmp/install-system-splash-deps.sh
 RUN /tmp/install-system-splash-deps.sh
 
-# XXX: this needs to be updated if Qt is updated
-COPY --from=qtbuilder /opt/qt-5.13/5.13.1/gcc_64 /opt/qt-5.13/5.13.1/gcc_64
-#RUN ls -l /opt/qt-5.13/5.13.0/gcc_64/lib
-ENV PATH="/opt/qt-5.13/5.13.1/gcc_64/bin:${PATH}"
-ENV LD_LIBRARY_PATH="/opt/qt-5.13/5.13.1/gcc_64/lib:$LD_LIBRARY_PATH"
+ARG QT_SHORT_VERSION
+ARG QT_FULL_VERSION
+
+COPY --from=qtbuilder \
+    /opt/qt-${QT_SHORT_VERSION}/${QT_FULL_VERSION}/gcc_64 \
+    /opt/qt-${QT_SHORT_VERSION}/${QT_FULL_VERSION}/gcc_64
+
+#RUN ls -l /opt/qt-${QT_SHORT_VERSION}/${QT_FULL_VERSION}/gcc_64/lib
+ENV PATH="/opt/qt-${QT_SHORT_VERSION}/${QT_FULL_VERSION}/gcc_64/bin:${PATH}"
+ENV LD_LIBRARY_PATH="/opt/qt-${QT_SHORT_VERSION}/${QT_FULL_VERSION}/gcc_64/lib:$LD_LIBRARY_PATH"
 
 # =====================
 
 FROM splash-base as qt5-builder
 
+ARG SPLASH_BUILD_PARALLEL_JOBS
+ENV SPLASH_BUILD_PARALLEL_JOBS=${SPLASH_BUILD_PARALLEL_JOBS}
+
 COPY --from=pyqt5-downloader /tmp/sip.tar.gz /tmp/
 COPY --from=pyqt5-downloader /tmp/pyqt5.tar.gz /tmp/
 COPY dockerfiles/splash/install-pyqt5.sh /tmp/install-pyqt5.sh
@@ -105,6 +153,9 @@ RUN /tmp/install-pyqtwebengine.sh /tmp/webengine.tar.gz
 
 FROM splash-base as splash
 
+ARG QT_MAJOR_VERSION
+ENV QT_MAJOR_VERSION=${QT_MAJOR_VERSION}
+
 COPY dockerfiles/splash/install-python-splash-deps.sh /tmp/install-python-splash-deps.sh
 RUN /tmp/install-python-splash-deps.sh
 

dockerfiles/splash/create-package.sh

@@ -0,0 +1,34 @@
+description='
+Packages from  https://github.com/qtwebkit/qtwebkit/releases are used for
+building new splash images. It is problematic to create compatible packages
+using only CPack(CMake packaging module). This script creates packages
+compatible with qtwebkit releases.
+It has next arguments:
+- install_manifest_txt - file created by CMake after running install command, e.g.
+    ninja install
+'
+install_manifest_txt=$1
+
+qt_root=`qmake -query QT_INSTALL_PREFIX`
+
+prefix_len=${#qt_root}
+
+work_dir=`mktemp -d`
+trap "rm -rf $work_dir" EXIT
+
+cat $install_manifest_txt |
+    while read -r source;
+    do
+        target_file=$work_dir/`echo "$source" | cut -c $((prefix_len + 1))-`
+        placement_dir=`dirname $target_file`
+        mkdir -p $placement_dir
+        cp $source $target_file
+    done
+
+result_dir=`dirname $install_manifest_txt`
+result=`realpath $result_dir/build.7z`
+rm -rf $result
+
+pushd $work_dir
+7z a $result ./*
+popd

dockerfiles/splash/download-pyqt5.sh

@@ -2,10 +2,9 @@
 # XXX: these URLs needs to be replaced with sourceforge in future,
 # because riverbank tend to remove old releases.
 SIP="https://www.riverbankcomputing.com/static/Downloads/sip/4.19.19/sip-4.19.19.tar.gz"
-PYQT5="https://www.riverbankcomputing.com/static/Downloads/PyQt5/5.13.1/PyQt5_gpl-5.13.1.tar.gz"
-WEBENGINE="https://www.riverbankcomputing.com/static/Downloads/PyQtWebEngine/5.13.1/PyQtWebEngine_gpl-5.13.1.tar.gz"
-
-curl -L -o "$1" ${SIP} && \
-curl -L -o "$2" ${PYQT5} && \
-curl -L -o "$3" ${WEBENGINE}
+PYQT="https://www.riverbankcomputing.com/static/Downloads/PyQt5/5.13.2/PyQt5-5.13.2.tar.gz"
+WEBENGINE="https://www.riverbankcomputing.com/static/Downloads/PyQtWebEngine/5.13.2/PyQtWebEngine-5.13.2.tar.gz"
 
+curl --fail -L -o "$1" ${SIP} && \
+curl --fail -L -o "$2" ${PYQT} && \
+curl --fail -L -o "$3" ${WEBENGINE}

dockerfiles/splash/download-qt-installer.sh

@@ -1,7 +1,5 @@
 #!/usr/bin/env sh
 
-# XXX: if qt version is changed, Dockerfile should be updated,
-# as well as qt-installer-noninteractive.qs script.
-URL="http://download.qt.io/official_releases/qt/5.13/5.13.1/qt-opensource-linux-x64-5.13.1.run"
+URL="http://download.qt.io/official_releases/qt/$QT_SHORT_VERSION/$QT_FULL_VERSION/qt-opensource-linux-x64-$QT_FULL_VERSION.run"
 
-curl -L -o "$1" ${URL}
+curl --fail -L -o "$1" ${URL}

dockerfiles/splash/download-qtwebkit-source.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env sh
 URL="https://github.com/qtwebkit/qtwebkit/releases/download/qtwebkit-5.212.0-alpha3/qtwebkit-5.212.0-alpha3.tar.xz"
-curl -L -o "$1" ${URL}
+curl --fail -L -o "$1" ${URL}

dockerfiles/splash/download-qtwebkit.sh

@@ -1,3 +1,2 @@
 #!/usr/bin/env sh
-URL="https://download.qt.io/snapshots/ci/qtwebkit/5.212/1570542016/qtwebkit/qtwebkit-Linux-RHEL_7_6-GCC-Linux-RHEL_7_6-X86_64.7z"
-curl -L -o "$1" ${URL}
+curl --fail -L -o "$1" ${WEBKIT_URL}

dockerfiles/splash/install-pyqt5.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 
 _PYTHON=python3
-SPLASH_BUILD_PARALLEL_JOBS=8
 
 mkdir -p /tmp/builds/sip && \
 mkdir -p /tmp/builds/pyqt5 && \

dockerfiles/splash/install-pyqtwebengine.sh

@@ -7,8 +7,8 @@ pushd /tmp/builds && \
 # PyQtWebEngine
 tar xzf "$1" --keep-newer-files -C webengine --strip-components 1 && \
 pushd webengine && \
-${_PYTHON} configure.py -c -j 4 && \
-make -j 4 && \
+${_PYTHON} configure.py -c -j ${SPLASH_BUILD_PARALLEL_JOBS} && \
+make -j ${SPLASH_BUILD_PARALLEL_JOBS} && \
 make install && \
 popd  && \
-popd
+popd

dockerfiles/splash/install-python-splash-deps.sh

@@ -17,4 +17,4 @@ install_python_deps () {
     ${_PYTHON} -m pip install https://github.com/sunu/pyre2/archive/c610be52c3b5379b257d56fc0669d022fd70082a.zip#egg=re2
 }
 
-install_python_deps
+install_python_deps

dockerfiles/splash/qt-installer-noninteractive.qs

@@ -24,15 +24,23 @@ Controller.prototype.IntroductionPageCallback = function() {
 
 Controller.prototype.TargetDirectoryPageCallback = function()
 {
-    gui.currentPageWidget().TargetDirectoryLineEdit.setText("/opt/qt-5.13");
+    var short_version = installer.environmentVariable("QT_SHORT_VERSION");
+    var path = "/opt/qt-" + short_version
+
+    gui.currentPageWidget().TargetDirectoryLineEdit.setText(path);
     gui.clickButton(buttons.NextButton);
 }
 
 Controller.prototype.ComponentSelectionPageCallback = function() {
+    var major_version = installer.environmentVariable("QT_MAJOR_VERSION");
+    var minor_version = installer.environmentVariable("QT_MINOR_VERSION");
+    var patch_version = installer.environmentVariable("QT_PATCH_VERSION");
+    var qt_version = "qt.qt" + major_version + "." + major_version + minor_version + patch_version;
+
     var components = [
-      "qt.qt5.5131.gcc_64",
-      "qt.qt5.5131.qtwebengine",
-      "qt.qt5.5131.qtnetworkauth",
+      qt_version+ ".gcc_64",
+      qt_version+ ".qtwebengine",
+      qt_version+ ".qtnetworkauth",
     ]
     console.log("Select components");
     var widget = gui.currentPageWidget();

dockerfiles/splash/run-qt-installer.sh

@@ -11,6 +11,6 @@ fi
 chmod +x "$1" && \
 http_proxy="http://localhost:8080" https_proxy="http://localhost:8080" $command --script "$2" \
     | egrep -v '\[[0-9]+\] Warning: (Unsupported screen format)|((QPainter|QWidget))' && \
-ls /opt/qt-5.13/ && \
-#    cat /opt/qt-5.13/InstallationLog.txt && \
-cat /opt/qt-5.13/components.xml
+ls /opt/qt-$QT_SHORT_VERSION/ && \
+#    cat /opt/qt-$QT_SHORT_VERSION/InstallationLog.txt && \
+cat /opt/qt-$QT_SHORT_VERSION/components.xml

docs/install.rst

@@ -120,3 +120,49 @@ You may have to change FROM line in :file:`dockerfiles/splash-jupyter/Dockerfile
 if you want it to be based on your local Splash Docker container.
 
 .. _source code: https://github.com/scrapinghub/splash
+
+
+Custom qtwebkit binaries
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Pass URL of binaries archive in docker build argument, e.g.::
+
+    docker build \
+        --build-arg WEBKIT_URL=https://github.com/whalebot-helmsman/qtwebkit/releases/download/5.14.1-5.212.0-alpha-4/5.14.1-5.212.0-alpha-4.7z \
+        .
+
+Custom qtwebkit build
+---------------------
+
+You need a special container for this. There is one in Dockerfile for splash::
+
+    docker build --target qtwebkitbuilder-base . -t qtwebkit-builder
+
+Checkout qtwebkit code and mount it to a build container::
+
+    git clone git@github.com:qtwebkit/qtwebkit.git ../qtwebkit
+    docker run --rm -it -v `pwd`/../qtwebkit:/qtwebkit qtwebkit-builder
+
+To build qtwebkit from sources run next commands inside the container::
+
+     cd /qtwebkit
+     mkdir build
+     cd build
+     cmake -G Ninja -DPORT=Qt -DCMAKE_BUILD_TYPE=Release ..
+     ninja -j 8
+     ninja install
+     /tmp/create-package.sh install_manifest.txt '' 7z
+     7z l -ba build.7z  | head -n  10
+         2020-05-29 13:57:20 D....            0            0  include
+         2020-05-29 13:57:20 D....            0            0  include/QtWebKit
+         2020-05-29 13:57:20 D....            0            0  include/QtWebKit/5.212.0
+         2020-05-29 13:57:20 D....            0            0  include/QtWebKit/5.212.0/QtWebKit
+         2020-05-29 13:57:20 D....            0            0  include/QtWebKit/5.212.0/QtWebKit/private
+         2020-05-29 13:57:20 D....            0            0  include/QtWebKitWidgets
+         2020-05-29 13:57:20 D....            0            0  include/QtWebKitWidgets/5.212.0
+         2020-05-29 13:57:20 D....            0            0  include/QtWebKitWidgets/5.212.0/QtWebKitWidgets
+         2020-05-29 13:57:20 D....            0            0  include/QtWebKitWidgets/5.212.0/QtWebKitWidgets/private
+         2020-05-29 13:57:20 D....            0            0  lib
+
+Make `build.7z` available by HTTP protocol. Assets files of release section on
+a github is a good place for this.