Fixed #220: using Etcher-cli to write and verify SD-card

- Using v1.4.5 on Linux and macOS
- Using 1.1.2 on Windows (later versions are buggy)
- etcher-cli used with auto-check and auto-unmount options
- etcher-cli requires sudo on macOS and linux (in win32, we already run as admin)
  so password is asked at writing stage
- no more password on macOS and linux to change SD card device mode on start
  linux still ask for loop device mode change though

.gitignore

@@ -19,3 +19,4 @@ qemu-img
 qemu-system-arm
 xz-*.tar.gz
 xz-*
+etcher-cli

.travis.yml

@@ -48,6 +48,11 @@ matrix:
     - wget http://download.kiwix.org/dev/qemu-2.12.0_macOS.tar
     - tar xf qemu-2.12.0_macOS.tar
 
+    # Bundle Etcher-cli
+    - wget http://download.kiwix.org/dev/etcher-cli-1.4.5-darwin-x64.tar.gz
+    - mkdir -p etcher-cli
+    - tar xf etcher-cli-1.4.5-darwin-x64.tar.gz -C etcher-cli --strip-components=1
+
     # Run PyInstaller
     - pyinstaller --log-level=DEBUG kiwix-hotspot-macos.spec
 
@@ -120,6 +125,11 @@ matrix:
     - wget http://download.kiwix.org/dev/qemu-2.12.0-linux-x86_64.tar.gz
     - tar xf qemu-2.12.0-linux-x86_64.tar.gz
 
+    # Bundle Etcher-cli
+    - wget http://download.kiwix.org/dev/etcher-cli-1.4.5-linux-x64.tar.gz
+    - mkdir -p etcher-cli
+    - tar xf etcher-cli-1.4.5-linux-x64.tar.gz -C etcher-cli --strip-components=1
+
     # Install python dependancies
     - pip3 install -r requirements-linux.txt
 

CHANGELOG

@@ -4,6 +4,7 @@
 * Added Proxy Configuration Support
 * Removed StaticSites catalog as it is no longer available online
 * Changed JSON format for WiFi: single `wifi_password` entry
+* Using etcher-cli to flash and verify SD-card (all platforms)
 
 2.0-rc12
 * macOS binary name (Kiwix Hotspot instead of kiwix-hotspot)

appveyor.yml

@@ -112,6 +112,14 @@ install:
   - del imdiskinst.exe
   - dir "C:\Program Files\imdiskinst"
 
+  # Download Etcher-cli
+  - mkdir "C:\Program Files\etcher-cli"
+  - cd "C:\Program Files\etcher-cli"
+  - if %platform%==x86 appveyor DownloadFile "http://download.kiwix.org/dev/Etcher-cli-1.1.2-win32-x86.zip" -FileName "etcher-cli.zip"
+  - if %platform%==x64 appveyor DownloadFile "http://download.kiwix.org/dev/Etcher-cli-1.1.2-win32-x64.zip" -FileName "etcher-cli.zip"
+  - 7z.exe x etcher-cli.zip
+  - del etcher-cli.zip
+
   # Run pyinstaller and show warning
   - cd C:\projects\kiwix-hotspot
   - if %platform%==x86 C:\Python34\python.exe C:\Python34\Scripts\pyinstaller-script.py --log-level=DEBUG kiwix-hotspot-win32.spec

kiwix-hotspot-linux.spec

@@ -11,6 +11,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
                     ('etcher.gif', '.'),
                     ('kiwix-hotspot-logo.png', '.'),
                     ('ansiblecube', 'ansiblecube'),
+                    ('etcher-cli', 'etcher-cli'),
                     ('vexpress-boot', 'vexpress-boot')],
              hiddenimports=['gui', 'cli', 'image', 'cache'],
              hookspath=['additional-hooks'],

kiwix-hotspot-macos.spec

@@ -18,6 +18,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
                     ('etcher.gif', '.'),
                     ('kiwix-hotspot-logo.png', '.'),
                     ('ansiblecube', 'ansiblecube'),
+                    ('etcher-cli', 'etcher-cli'),
                     ('vexpress-boot', 'vexpress-boot')],
              hiddenimports=['gui', 'cli', 'image', 'cache'],
              hookspath=['additional-hooks'],

kiwix-hotspot-win32.spec

@@ -29,6 +29,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
                     ('vexpress-boot', 'vexpress-boot'),
                     ('C:\Program Files\qemu', 'qemu'),
                     ('C:\Program Files\imdiskinst', 'imdiskinst'),
+                    ('C:\Program Files\etcher-cli', 'etcher-cli'),
                     ('C:\Program Files\\7zextra\\7za.dll', '.'),
                     ('C:\Program Files\\7zextra\\7za.exe', '.'),
                     ('C:\Program Files\\7zextra\\7zxa.dll', '.')],

kiwix-hotspot-win64.spec

@@ -29,6 +29,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
                     ('vexpress-boot', 'vexpress-boot'),
                     ('C:\Program Files\qemu', 'qemu'),
                     ('C:\Program Files\imdiskinst', 'imdiskinst'),
+                    ('C:\Program Files\etcher-cli', 'etcher-cli'),
                     ('C:\Program Files\\7zextra\\x64\\7za.dll', '.'),
                     ('C:\Program Files\\7zextra\\x64\\7za.exe', '.'),
                     ('C:\Program Files\\7zextra\\x64\\7zxa.dll', '.')],

kiwix-hotspot/backend/util.py

@@ -3,11 +3,15 @@
 
 import os
 import sys
+import time
+import signal
 import ctypes
 import random
+import tempfile
 import threading
 import subprocess
 
+import data
 from util import CLILogger, ONE_MiB, human_readable_size
 
 
@@ -40,9 +44,10 @@ def startup_info_args():
         # distraction.
         si = subprocess.STARTUPINFO()
         si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+        cf = subprocess.CREATE_NEW_PROCESS_GROUP
     else:
-        si = None
-    return {"startupinfo": si}
+        si = cf = None
+    return {"startupinfo": si, "creationflags": cf}
 
 
 def subprocess_pretty_call(
@@ -130,7 +135,7 @@ def run_as_win_admin(command, logger):
     return rc
 
 
-def get_admin_command(command, from_gui):
+def get_admin_command(command, from_gui, log_to=None):
     """ updated command to run it as root on macos or linux
 
         from_gui: whether called via GUI. Using cli sudo if not """
@@ -142,64 +147,16 @@ def get_admin_command(command, from_gui):
         return [
             "/usr/bin/osascript",
             "-e",
-            'do shell script "{command} 2>&1" '
-            "with administrator privileges".format(command=" ".join(command)),
+            'do shell script "{command} 2>&1 {redir}" '
+            "with administrator privileges".format(
+                command=" ".join(command), redir=">{}".format(log_to) if log_to else ""
+            ),
         ]
     if sys.platform == "linux":
         return ["pkexec"] + command
 
 
-def open_handles(image_fpath, device_fpath, read_only=False):
-    img_flag = os.O_RDONLY
-    dev_flag = os.O_RDONLY if read_only else os.O_WRONLY
-    if os.name == "posix":
-        return (os.open(image_fpath, img_flag), os.open(device_fpath, dev_flag))
-    elif os.name == "nt":
-        dev_flag = dev_flag | os.O_BINARY
-        return (os.open(image_fpath, img_flag), os.open(device_fpath, dev_flag))
-    else:
-        raise NotImplementedError("Platform not supported")
-
-
-def close_handles(image_fd, device_fd):
-    try:
-        os.close(image_fd)
-        os.close(device_fd)
-    except Exception:
-        pass
-
-
-def ensure_card_written(image_fpath, device_fpath, logger):
-    """ asserts image and device content is same (reads rand 4MiB from both """
-
-    logger.step("Verify data on SD card")
-
-    image_fd, device_fd = open_handles(image_fpath, device_fpath, read_only=True)
-
-    # read a 4MiB random part from the image
-    buffer_size = 4 * ONE_MiB
-    total_size = os.lseek(image_fd, 0, os.SEEK_END)
-    offset = random.randint(0, int((total_size - buffer_size) * .8))
-    offset -= offset % 512
-    logger.std(
-        "reading {n}b from offset {s} out of {t}b.".format(
-            n=buffer_size, s=offset, t=total_size
-        )
-    )
-
-    try:
-        # read same part from the SD card and compare
-        os.lseek(image_fd, offset, os.SEEK_SET)
-        os.lseek(device_fd, offset, os.SEEK_SET)
-        if not os.read(image_fd, buffer_size) == os.read(device_fd, buffer_size):
-            raise ValueError("Image and SD-card challenge do not match.")
-    except Exception:
-        raise
-    finally:
-        close_handles(image_fd, device_fd)
-
-
-class ImageWriterThread(threading.Thread):
+class EtcherWriterThread(threading.Thread):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._should_stop = False  # stop flag
@@ -208,63 +165,82 @@ def __init__(self, *args, **kwargs):
     def stop(self):
         self._should_stop = True
 
-    def run(self):
+    def run(self,):
         image_fpath, device_fpath, logger = self._args
 
-        logger.step("Copy image to sd card")
-
-        image_fd, device_fd = open_handles(image_fpath, device_fpath)
-
-        total_size = os.lseek(image_fd, 0, os.SEEK_END)
-        os.lseek(image_fd, 0, os.SEEK_SET)
+        logger.step("Copy image to sd card using etcher-cli")
 
-        if os.name == "nt":
-            buffer_size = 512  # safer on windows
-            logger_break = 1000
-        else:
-            buffer_size = 25 * ONE_MiB
-            logger_break = 4
-        steps = total_size // buffer_size
-
-        for step in range(0, steps):
+        from_cli = logger is None or type(logger) == CLILogger
+        # on macOS, GUI sudo captures stdout so we use a log file
+        log_to_file = not from_cli and sys.platform == "darwin"
+        if log_to_file:
+            log_file = tempfile.NamedTemporaryFile(suffix=".log")
+
+        cmd = get_admin_command(
+            [
+                os.path.join(data.data_dir, "etcher-cli", "etcher"),
+                "-c",
+                "-y",
+                "-u",
+                "-d",
+                '"{}"'.format(device_fpath)
+                if sys.platform == "win32"  # \\.\PHYSICALDRIVE1 string needs quotes
+                else device_fpath,
+                image_fpath,
+            ],
+            from_gui=not from_cli,
+            log_to=log_file.name if log_to_file else None,
+        )
+        process = subprocess.Popen(
+            cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **startup_info_args()
+        )
+        logger.std("Starting Etcher: " + str(process.args))
 
-            if self._should_stop:
+        while process.poll() is None:
+            if self._should_stop:  # on cancel
+                logger.std(". cancelling...")
                 break
 
-            # only update logger every 4 steps (100MiB)
-            if step % logger_break == 0:
-                logger.progress(step, steps)
-                logger.std(
-                    "Copied {copied} of {total} ({pc:.2f}%)".format(
-                        copied=human_readable_size(step * buffer_size),
-                        total=human_readable_size(total_size),
-                        pc=step / steps * 100,
-                    )
-                )
+            if log_to_file:
+                with open(log_file.name, "r") as f:
+                    content = f.read()
+                    if content:
+                        logger.raw_std(f.read())
+            else:
+                for line in process.stdout:
+                    logger.raw_std(line.decode("utf-8", "ignore"))
+            time.sleep(2)
 
-            try:
-                os.write(device_fd, os.read(image_fd, buffer_size))
-            except Exception as exp:
-                logger.std("Exception during write: {}".format(exp))
-                close_handles(image_fd, device_fd)
-                self.exp = exp
-                raise
-
-        if not self._should_stop and total_size % buffer_size:
-            logger.std("Writing last chunk...")
-            try:
-                os.write(device_fd, os.read(image_fd, total_size % buffer_size))
-            except Exception as exp:
-                logger.std("Exception during write: {}".format(exp))
-                close_handles(image_fd, device_fd)
-                self.exp = exp
-                raise
+        if log_to_file:
+            log_file.close()
 
+        try:
+            logger.std(". has process exited?")
+            process.wait(timeout=2)
+        except subprocess.TimeoutExpired:
+            logger.std(". process exited")
+            # send ctrl^c
+            if sys.platform == "win32":
+                logger.std(". sending ctrl^C")
+                process.send_signal(signal.CTRL_C_EVENT)
+                process.send_signal(signal.CTRL_BREAK_EVENT)
+                time.sleep(2)
+            if process.poll() is None:
+                logger.std(". sending SIGTERM")
+                process.terminate()  # send SIGTERM
+                time.sleep(2)
+            if process.poll() is None:
+                logger.std(". sending SIGKILL")
+                process.kill()  # send SIGKILL (SIGTERM again on windows)
+                time.sleep(2)
+        else:
+            logger.std(". process exited")
+            if not process.returncode == 0:
+                self.exp = CheckCallException(
+                    "Process returned {}".format(process.returncode)
+                )
+        logger.std(". process done")
         logger.progress(1)
-        logger.step("sync")
-        if not self._should_stop:
-            os.fsync(device_fd)
-        close_handles(image_fd, device_fd)
 
 
 def prevent_sleep(logger):