Libzim GitHub CI windows

.github/scripts/build_definition.py

@@ -21,6 +21,11 @@
     | manylinux | native_mixed       | BP     |          |           |             |               | linux-x86_64-manylinux  |                        |
     | manylinux | aarch64_mixed      | BP     |          |           |             |               | linux-aarch64-manylinux |                        |
     ----------------------------------------------------------------------------------------------------------------------------------------------
+# On Windows, we build only libzim for now. And only native_mixed as xapian doesn't compile as dll
+    | windows   | native_static      | Bd     |          |           |             |               | win-x86_64              | win-x86_64-static      |
+    | windows   | native_dyn         | Bd     |          |           |             |               | win-x86_64              | win-x86_64-dyn         |
+    | windows   | native_mixed       | BPd    |          |           |             |               | win-x86_64              | win-x86_64-mixed       |
+    ----------------------------------------------------------------------------------------------------------------------------------------------
 # Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries
     | macos     | native_dyn         | d      | d        | dB        | B           |               |                         | macos-x86_64-dyn       |
     | macos     | native_static      |        |          | BP        | BP          |               | macos-x86_64            |                        |

.github/scripts/common.py

@@ -1,12 +1,13 @@
 import os
 from os import environ as _environ
-from pathlib import Path
+from pathlib import Path, PurePosixPath
 from datetime import date
 import tarfile
 import zipfile
 import subprocess
 import re
 import shutil
+import platform
 
 import requests
 
@@ -27,7 +28,7 @@ def get_build_dir(config) -> Path:
     command.append("--use-target-arch-name")
     return Path(
         subprocess.run(command, cwd=str(HOME), check=True, stdout=subprocess.PIPE)
-        .stdout[:-1]
+        .stdout.strip()
         .decode("utf8")
     )
 
@@ -41,7 +42,8 @@ def get_build_dir(config) -> Path:
 ARCHIVE_DIR = HOME / "ARCHIVE"
 TOOLCHAIN_DIR = BASE_DIR / "TOOLCHAINS"
 INSTALL_DIR = BASE_DIR / "INSTALL"
-TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp"))
+default_tmp_dir = os.getenv("TEMP") if platform.system() == 'Windows' else "/tmp"
+TMP_DIR = Path(os.getenv("TMP_DIR", default_tmp_dir))
 KBUILD_SOURCE_DIR = HOME / "kiwix-build"
 
 _ref = _environ.get("GITHUB_REF", "").split("/")[-1]
@@ -195,55 +197,112 @@ def run_kiwix_build(
     subprocess.check_call(command, cwd=str(HOME), env=env)
     print_message("Build ended")
 
+try:
+    import paramiko
 
-def upload(file_to_upload, host, dest_path):
-    if not file_to_upload.exists():
-        print_message("No {} to upload!", file_to_upload)
-        return
+    def upload(file_to_upload, host, dest_path):
+        if not file_to_upload.exists():
+            print_message("No {} to upload!", file_to_upload)
+            return
 
-    if ":" in host:
-        host, port = host.split(":", 1)
-    else:
-        port = "22"
+        if ":" in host:
+            host, port = host.split(":", 1)
+        else:
+            port = "22"
+        if '@' in host:
+            user, host = host.split('@', 1)
+        else:
+            user = None
+
+        from contextlib import contextmanager
+
+        @contextmanager
+        def get_client():
+            client = paramiko.client.SSHClient()
+            client.set_missing_host_key_policy(paramiko.client.WarningPolicy)
+            print_message(f"Connect to {host}:{port}")
+            client.connect(host, port=port, username=user, key_filename=_environ.get("SSH_KEY"), look_for_keys=False, compress=True)
+            try:
+                yield client
+            finally:
+                client.close()
+
+        @contextmanager
+        def get_sftp():
+            with get_client() as client:
+                sftp = client.open_sftp()
+                try:
+                    yield sftp
+                finally:
+                    sftp.close()
+
+        dest_path = PurePosixPath(dest_path)
+        remote_file = dest_path.joinpath(file_to_upload.name)
+
+        with get_sftp() as sftp:
+            for part in list(reversed(dest_path.parents)) + [dest_path]:
+                part = str(part)
+                try:
+                    sftp.stat(part)
+                except FileNotFoundError:
+                    sftp.mkdir(part)
+
+            print_message(f"Sending archive {file_to_upload} to {remote_file}")
+            sftp.put(str(file_to_upload), str(remote_file), confirm=True)
+
+
+except ModuleNotFoundError:
+    # On old system (bionic) paramiko is really complex to install
+    # Keep the old implementaion on sush system.
+
+    def upload(file_to_upload, host, dest_path):
+        if not file_to_upload.exists():
+            print_message("No {} to upload!", file_to_upload)
+            return
+
+        if ":" in host:
+            host, port = host.split(":", 1)
+        else:
+            port = "22"
 
-    # sending SFTP mkdir command to the sftp interactive mode and not batch (-b) mode
-    # as the latter would exit on any mkdir error while it is most likely
-    # the first parts of the destination is already present and thus can't be created
-    sftp_commands = "\n".join(
-        [
-            f"mkdir {part}"
-            for part in list(reversed(Path(dest_path).parents)) + [dest_path]
+        # sending SFTP mkdir command to the sftp interactive mode and not batch (-b) mode
+        # as the latter would exit on any mkdir error while it is most likely
+        # the first parts of the destination is already present and thus can't be created
+        sftp_commands = "\n".join(
+            [
+                f"mkdir {part}"
+                for part in list(reversed(Path(dest_path).parents)) + [dest_path]
+            ]
+        )
+        command = [
+            "sftp",
+            "-i",
+            _environ.get("SSH_KEY"),
+            "-P",
+            port,
+            "-o",
+            "StrictHostKeyChecking=no",
+            host,
         ]
-    )
-    command = [
-        "sftp",
-        "-i",
-        _environ.get("SSH_KEY"),
-        "-P",
-        port,
-        "-o",
-        "StrictHostKeyChecking=no",
-        host,
-    ]
-    print_message("Creating dest path {}", dest_path)
-    subprocess.run(command, input=sftp_commands.encode("utf-8"), check=True)
+        print_message("Creating dest path {}", dest_path)
+        subprocess.run(command, input=sftp_commands.encode("utf-8"), check=True)
 
-    command = [
-        "scp",
-        "-c",
-        "aes128-ctr",
-        "-rp",
-        "-P",
-        port,
-        "-i",
-        _environ.get("SSH_KEY"),
-        "-o",
-        "StrictHostKeyChecking=no",
-        str(file_to_upload),
-        "{}:{}".format(host, dest_path),
-    ]
-    print_message("Sending archive with command {}", command)
-    subprocess.check_call(command)
+        command = [
+            "scp",
+            "-c",
+            "aes128-ctr",
+            "-rp",
+            "-P",
+            port,
+            "-i",
+            _environ.get("SSH_KEY"),
+            "-o",
+            "StrictHostKeyChecking=no",
+            str(file_to_upload),
+            "{}:{}".format(host, dest_path),
+        ]
+        print_message("Sending archive with command {}", command)
+        subprocess.check_call(command)
 
 
 def upload_archive(archive, project, make_release, dev_branch=None):

.github/scripts/upload_failure_logs.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+import tarfile
+from pathlib import Path
+from common import upload, OS_NAME, COMPILE_CONFIG, HOME
+
+ARCHIVE_NAME = Path(f"fail_log_{OS_NAME}_{COMPILE_CONFIG}.tar.gz")
+
+
+files_to_archive = []
+files_to_archive += HOME.glob("BUILD_*")
+files_to_archive += [HOME / "SOURCE", HOME / "LOGS", HOME / "TOOLCHAINS"]
+
+with tarfile.open(ARCHIVE_NAME, "w:xz") as tar:
+    for name in set(files_to_archive):
+        tar.add(str(name))
+
+upload(ARCHIVE_NAME, "ci@tmp.kiwix.org:30022", "/data/tmp/ci")

.github/workflows/ci.yml

@@ -6,6 +6,61 @@ on:
    - cron: '0 1 * * *'
 
 jobs:
+  Windows:
+    strategy:
+      fail-fast: false
+    runs-on: windows-latest
+    env:
+      OS_NAME: windows
+      COMPILE_CONFIG: native_dyn
+      HOME: 'C:\\Users\\runneradmin'
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+    - name: Setup python 3.8
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.8'
+    - name: Install packages
+      run: |
+        choco.exe install pkgconfiglite ninja
+    - name: Install python modules
+      shell: bash
+      run: |
+        pip3 install meson pytest requests distro paramiko
+        pip3 install --no-deps $GITHUB_WORKSPACE
+    - name: Setup MSVC compiler
+      uses: bus1/cabuild/action/msdevshell@v1
+      with:
+        architecture: x64
+    - name: secret
+      shell: bash
+      run: |
+        echo "${{secrets.ssh_key}}" > $SSH_KEY
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Ensure base deps
+      run: |
+        python .github\\scripts\\ensure_base_deps.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Compile all deps
+      run: |
+        python .github\\scripts\\compile_all_deps.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Build projects
+      run: |
+        python .github\\scripts\\build_projects.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Upload failure logs
+      if: failure()
+      run: |
+        python .github\\scripts\\upload_failure_logs.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+
   Linux:
     strategy:
       fail-fast: false
@@ -63,6 +118,10 @@ jobs:
         pip3 install --user --no-deps .
       env:
         REP: ${{github.repository}}
+    - name: Install paramiko
+      if: ${{matrix.image_variant != 'bionic' }}
+      shell: bash
+      run: pip3 install --user paramiko
     - name: secret
       shell: bash
       run: |
@@ -91,7 +150,7 @@ jobs:
         COMPILE_CONFIG: ${{matrix.config}}
     - name: Upload failure logs
       if: failure()
-      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
+      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
       env:
         COMPILE_CONFIG: ${{matrix.config}}
 
@@ -112,6 +171,7 @@ jobs:
         git clone https://github.com/${REP}
         cd ./${REP##*/}
         git checkout --force ${GITHUB_SHA}
+        pip3 install --user paramiko
         pip3 install --user --no-deps .
       env:
         REP: ${{github.repository}}
@@ -136,7 +196,7 @@ jobs:
         kiwix-build/.github/scripts/build_projects.py
     - name: Upload failure logs
       if: failure()
-      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
+      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
 
   Macos:
     strategy:
@@ -172,7 +232,7 @@ jobs:
         brew install pkg-config ninja automake autoconf
     - name: Install python modules
       run: |
-        pip3 install meson pytest requests distro
+        pip3 install meson pytest requests distro paramiko
         pip3 install --no-deps $GITHUB_WORKSPACE
     - name: secret
       shell: bash
@@ -202,6 +262,6 @@ jobs:
         COMPILE_CONFIG: ${{matrix.config}}
     - name: Upload failure logs
       if: failure()
-      run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh
+      run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.py
       env:
         COMPILE_CONFIG: ${{matrix.config}}

kiwixbuild/buildenv.py

@@ -37,15 +37,6 @@ def __init__(self, dummy_run):
     def detect_platform(self):
         _platform = platform.system()
         self.distname = _platform
-        if _platform == "Windows":
-            print(
-                "ERROR: kiwix-build is not intented to run on Windows platform.\n"
-                "It should probably not work, but well, you still can have a try.",
-                file=sys.stderr,
-            )
-            cont = input("Do you want to continue ? [y/N]")
-            if cont.lower() != "y":
-                sys.exit(0)
         if _platform == "Linux":
             self.distname = distro.id()
             if self.distname == "ubuntu":
@@ -132,13 +123,12 @@ def _detect_libdir(self):
     def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
         env = self.configInfo.get_env()
         pkgconfig_path = pj(self.install_dir, self.libprefix, "pkgconfig")
-        env["PKG_CONFIG_PATH"] = ":".join([env["PKG_CONFIG_PATH"], pkgconfig_path])
+        env["PKG_CONFIG_PATH"].append(pkgconfig_path)
 
-        env["PATH"] = ":".join([escape_path(pj(self.install_dir, "bin")), env["PATH"]])
+        env["PATH"].insert(0, pj(self.install_dir, "bin"))
 
-        env["LD_LIBRARY_PATH"] = ":".join(
+        env["LD_LIBRARY_PATH"].extend(
             [
-                env["LD_LIBRARY_PATH"],
                 pj(self.install_dir, "lib"),
                 pj(self.install_dir, self.libprefix),
             ]
@@ -170,7 +160,7 @@ def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
         if cross_compilers:
             self.configInfo.set_compiler(env)
         if cross_path:
-            env["PATH"] = ":".join(self.configInfo.get_bin_dir() + [env["PATH"]])
+            env["PATH"][0:0] = self.configInfo.get_bin_dir()
         return env
 
     @property