Feature: enable ESP82xx WIFI upload (#422)

src/python/build_env_setup.py

@@ -3,20 +3,25 @@
 import UARTupload
 import opentx
 import upload_via_esp8266_backpack
+import esp_compress
 
-stm = env.get('PIOPLATFORM', '') in ['ststm32']
+platform = env.get('PIOPLATFORM', '')
+stm = platform in ['ststm32']
+
+target_name = env['PIOENV'].upper()
+print("PLATFORM : '%s'" % platform)
+print("BUILD ENV: '%s'" % target_name)
 
 # don't overwrite if custom command defined
 if stm and "$UPLOADER $UPLOADERFLAGS" in env.get('UPLOADCMD', '$UPLOADER $UPLOADERFLAGS'):
-    target_name = env['PIOENV'].upper()
-    print("BUILD ENV: '%s'" % target_name)
     if target_name == "FRSKY_TX_R9M_VIA_STLINK_OLD_BOOTLOADER_DEPRECATED":
         print("you are using the old bootloader, please update this will be removed soon")
-        env.AddPostAction("buildprog", [opentx.gen_frsky])
+        env.AddPostAction("buildprog", opentx.gen_frsky)
     elif "APLHA_900_TX" in target_name:
         env.Replace(UPLOADCMD=upload_via_esp8266_backpack.on_upload)
     elif "_R9M_" in target_name or "ES915TX" in target_name:
-        env.AddPostAction("buildprog", [opentx.gen_elrs])
+        env.AddPostAction("buildprog", opentx.gen_elrs)
+        env.AddPreAction("upload", opentx.gen_elrs)
         if "WIFI" in target_name:
             env.Replace(UPLOADCMD=upload_via_esp8266_backpack.on_upload)
         else:
@@ -25,3 +30,14 @@
         env.Replace(UPLOADCMD=stlink.on_upload)
     elif "_BETAFLIGHTPASSTHROUGH" in target_name:
         env.Replace(UPLOADCMD=UARTupload.on_upload)
+elif platform in ['espressif8266']:
+    env.AddPostAction("buildprog", esp_compress.compressFirmware)
+    env.AddPreAction("${BUILD_DIR}/spiffs.bin",
+                     [esp_compress.compress_files])
+    env.AddPreAction("${BUILD_DIR}/${ESP8266_FS_IMAGE_NAME}.bin",
+                     [esp_compress.compress_files])
+    env.AddPostAction("${BUILD_DIR}/${ESP8266_FS_IMAGE_NAME}.bin",
+                     [esp_compress.compress_fs_bin])
+    if "_WIFI" in target_name:
+        env.Replace(UPLOAD_PROTOCOL="custom")
+        env.Replace(UPLOADCMD=upload_via_esp8266_backpack.on_upload)

src/python/esp_compress.py

@@ -0,0 +1,108 @@
+import gzip
+import shutil
+import os, glob
+
+#
+# Code is copied from:
+# https://gist.github.com/andrewwalters/d4e3539319e55fc980db1ba67254d7ed
+#
+def binary_compress(target_file, source_file):
+    """ Compress ESP8266 firmware using gzip for 'compressed OTA upload' """
+    do_compress = True
+    source_file_bak = source_file
+    if target_file == source_file:
+        source_file_bak = source_file + ".bak"
+
+    if os.path.exists(target_file) and os.path.exists(source_file_bak):
+        """ Recompress if source file is newer than target file """
+        tgt_mtime = os.stat(target_file).st_mtime
+        src_mtime = os.stat(source_file_bak).st_mtime
+        if target_file == source_file:
+            """ bak file is older than source file """
+            do_compress = (src_mtime < tgt_mtime)
+        else:
+            """ target file is older than source file """
+            do_compress = (src_mtime > tgt_mtime)
+
+    if do_compress:
+        print("Compressing firmware for upload...")
+        if target_file == source_file:
+            shutil.move(source_file, source_file_bak)
+        with open(source_file_bak, 'rb') as f_in:
+            with gzip.open(target_file, 'wb') as f_out:
+                shutil.copyfileobj(f_in, f_out)
+        """ Set modification time on compressed file so incremental build works """
+        shutil.copystat(source_file_bak, target_file)
+        """ print compression info """
+        size_orig = os.stat(source_file_bak).st_size
+        size_gz = os.stat(target_file).st_size
+        print("Compression reduced firmware size to {:.0f}% (was {} bytes, now {} bytes)".format(
+            (size_gz / size_orig) * 100, size_orig, size_gz))
+
+
+def compressFirmware(source, target, env):
+    """ Compress ESP8266 firmware using gzip for 'compressed OTA upload' """
+    build_dir = env.subst("$BUILD_DIR")
+    image_name = env.subst("$PROGNAME")
+    source_file = os.path.join(build_dir, image_name + ".bin")
+    target_file = source_file
+    binary_compress(target_file, source_file)
+
+
+def compress_files(source, target, env):
+    #add filetypes (extensions only) to be gzipped before uploading. Everything else will be copied directly
+    filetypes_to_gzip = ['js', 'html', 'css']
+    #filetypes_to_gzip = []
+
+    project_dir = env.get('PROJECT_DIR')
+    platform = env.get('PIOPLATFORM', '')
+    if platform == 'espressif8266':
+        data_src_dir = os.path.join(project_dir,
+            "utils", "ESP8266SerialToWebsocket", "html")
+    else:
+        data_src_dir = os.path.join(project_dir,
+            "src", "esp32", "html")
+
+    print(' ***** Packing html files *****')
+
+    data_dir = env.get('PROJECTDATA_DIR')
+
+    if not os.path.exists(data_src_dir):
+        print('    HTML source "%s" does not found.' % data_src_dir)
+        return
+
+    if os.path.exists(data_dir):
+        print('    RM data dir: ' + data_dir)
+        shutil.rmtree(data_dir)
+
+    print('    MK data dir: ' + data_dir)
+    os.mkdir(data_dir)
+
+    files_to_gzip = []
+    for extension in filetypes_to_gzip:
+        files_to_gzip.extend(glob.glob(os.path.join(data_src_dir, '*.' + extension)))
+
+    #print('    => files to gzip: ' + str(files_to_gzip))
+
+    all_files = glob.glob(os.path.join(data_src_dir, '*.*'))
+    files_to_copy = list(set(all_files) - set(files_to_gzip))
+
+    for f_name in files_to_copy:
+        print('    Copy: ' + os.path.basename(f_name))
+        shutil.copy(f_name, data_dir)
+
+    for f_name in files_to_gzip:
+        print('    GZip: ' + os.path.basename(f_name))
+        with open(f_name, 'rb') as f_in:
+            out_file = os.path.join(data_dir, os.path.basename(f_name) + '.gz')
+            with gzip.open(out_file, 'wb') as f_out:
+                shutil.copyfileobj(f_in, f_out)
+
+    print('   packing done!')
+
+def compress_fs_bin(source, target, env):
+    build_dir = env.subst("$BUILD_DIR")
+    image_name = env.get('ESP8266_FS_IMAGE_NAME')
+    src_file = os.path.join(build_dir, image_name + ".bin")
+    target_file = src_file + ".gz"
+    binary_compress(target_file, src_file)

src/python/stlink.py

@@ -36,14 +36,7 @@ def get_commands(env, firmware):
                 bootloader, hex(flash_start)]
         APP_CMD = [TOOL, "-c SWD SWCLK=8 -P",
             firmware, hex(app_start), "-RST"]
-    elif "linux" in platform_name:
-        TOOL = os.path.join(
-            env_dir,
-            "tool-stm32duino", "stlink", "st-flash")
-        if bootloader is not None:
-            BL_CMD = [TOOL, "write", bootloader, hex(flash_start)]
-        APP_CMD = [TOOL, "--reset", "write", firmware, hex(app_start)]
-    elif "darwin" in platform_name:
+    elif "linux" in platform_name or "darwin" in platform_name:
         TOOL = os.path.join(
             env_dir,
             "tool-stm32duino", "stlink", "st-flash")

src/python/upload_via_esp8266_backpack.py

@@ -2,6 +2,24 @@
 import opentx
 
 def on_upload(source, target, env):
+    isstm = env.get('PIOPLATFORM', '') in ['ststm32']
+
+    upload_addr = ['elrs_tx', 'elrs_tx.local']
+    app_start = 0 # eka bootloader offset
+
+    # Parse upload flags:
+    upload_flags = env.get('UPLOAD_FLAGS', [])
+    for line in upload_flags:
+        flags = line.split()
+        for flag in flags:
+            if "VECT_OFFSET=" in flag:
+                offset = flag.split("=")[1]
+                if "0x" in offset:
+                    offset = int(offset, 16)
+                else:
+                    offset = int(offset, 10)
+                app_start = offset
+
     firmware_path = str(source[0])
     bin_path = os.path.dirname(firmware_path)
     elrs_bin_target = os.path.join(bin_path, 'firmware.elrs')
@@ -10,23 +28,23 @@ def on_upload(source, target, env):
         if not os.path.exists(elrs_bin_target):
             raise Exception("No valid binary found!")
 
-    cmd = ["curl", "-v", "--max-time", "60",
+    cmd = ["curl", "--max-time", "60",
            "--retry", "2", "--retry-delay", "1",
            "-F", "data=@%s" % (elrs_bin_target,)]
-
-    upload_addr = ['elrs_tx', 'elrs_tx.local']
+    if isstm:
+        cmd += ["-F", "flash_address=0x%X" % (app_start,)]
 
     upload_port = env.get('UPLOAD_PORT', None)
     if upload_port is not None:
         upload_addr = [upload_port]
 
     for addr in upload_addr:
-        addr = "http://%s/upload" % addr
+        addr = "http://%s/%s" % (addr, ['update', 'upload'][isstm])
         print(" ** UPLOADING TO: %s" % addr)
         try:
             subprocess.check_call(cmd + [addr])
             return
         except subprocess.CalledProcessError:
             print("FAILED!")
 
-    raise Exception("WiFI upload FAILED!")
+    raise Exception("WIFI upload FAILED!")

src/targets/diy_2400.ini

@@ -62,6 +62,10 @@ upload_command =
 	python $PROJECT_DATA_DIR/python/BFinitPassthrough.py $UPLOAD_SPEED
 	python $PROJECT_DATA_DIR/python/esptool-3.0/esptool.py --no-stub -b $UPLOAD_SPEED -c esp8266 --before no_reset --after soft_reset write_flash 0x0000 $SOURCE
 
+[env:DIY_2400_RX_ESP8285_SX1280_via_WIFI]
+extends = env:DIY_2400_RX_ESP8285_SX1280_via_UART
+upload_port = elrs_rx.local
+
 [env:DIY_2400_RX_STM32_CCG_Nano_v0_5_via_STLINK]
 extends = env_common_stm32
 platform = ststm32@9.0.0

src/targets/diy_900.ini

@@ -60,3 +60,7 @@ upload_protocol = custom
 upload_command =
 	python $PROJECT_DATA_DIR/python/BFinitPassthrough.py $UPLOAD_SPEED
 	python $PROJECT_DATA_DIR/python/esptool-3.0/esptool.py --no-stub -b $UPLOAD_SPEED -c esp8266 --before no_reset --after soft_reset write_flash 0x0000 $SOURCE
+
+[env:DIY_900_RX_ESP8285_SX127x_via_WIFI]
+extends = env:DIY_900_RX_ESP8285_SX127x_via_UART
+upload_port = elrs_rx.local