Skip to content

Commit

Permalink
feat(tools): Updated get.py with ability to verify extracted files an…
Browse files Browse the repository at this point in the history 
…d skip if ok (#8720)

* Updated get.py with ability to verify extracted files and skip if ok

* Updates

* Minor fixes

* Push binary to tools

* Fix formatting and add checksum comparison

* Push binary to tools

* Skip checksum for libs

* Push binary to tools

* ci(pre-commit): Apply automatic fixes

* Push binary to tools

* Make the linter happy

* Push binary to tools

* refactor(get.py): Small improvements

* fix(get.py): Fix parent folder name

* ci(pre-commit): Apply automatic fixes

* Push binary to tools

* fix(get.py): Fix header comment

* Push binary to tools

---------

Co-authored-by: Tomas Pilny <tomas.pilny@espressif.com>
Co-authored-by: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com>
Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com>
Co-authored-by: pre-commit-ci-lite[bot] <pre-commit-ci-lite[bot]@users.noreply.github.com>
Co-authored-by: lucasssvaz <lucasssvaz@users.noreply.github.com>
  • Loading branch information
@me-no-dev @PilnyTomas
@lucasssvaz @pre-commit-ci-lite
6 people committed Jun 3, 2024
1 parent c16a325 commit 00fb799
Show file tree
Hide file tree
Showing 2 changed files with 219 additions and 42 deletions.
Binary file modified tools/get.exe
View file
Binary file not shown.
261 changes: 219 additions & 42 deletions tools/get.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"""Script to download and extract tools
This script will download and extract required tools into the current directory.
Tools list is obtained from package/package_esp8266com_index.template.json file.
Tools list is obtained from package/package_esp32_index.template.json file.
"""

from __future__ import print_function
Expand All @@ -22,6 +22,11 @@
import tarfile
import zipfile
import re
import time
import argparse

# Initialize start_time globally
start_time = -1

if sys.version_info[0] == 3:
from urllib.request import urlretrieve
Expand All @@ -42,7 +47,6 @@
elif __file__:
current_dir = os.path.dirname(os.path.realpath(unicode(__file__)))

# current_dir = os.path.dirname(os.path.realpath(unicode(__file__)))
dist_dir = current_dir + "/dist/"


Expand All @@ -62,53 +66,175 @@ def mkdir_p(path):
raise


def report_progress(count, blockSize, totalSize):
def format_time(seconds):
minutes, seconds = divmod(seconds, 60)
return "{:02}:{:05.2f}".format(int(minutes), seconds)


def report_progress(block_count, block_size, total_size, start_time):
downloaded_size = block_count * block_size
time_elapsed = time.time() - start_time
current_speed = downloaded_size / (time_elapsed)

if sys.stdout.isatty():
if totalSize > 0:
percent = int(count * blockSize * 100 / totalSize)
percent = min(100, percent)
sys.stdout.write("\r%d%%" % percent)
if total_size > 0:
percent_complete = min((downloaded_size / total_size) * 100, 100)
sys.stdout.write(
f"\rDownloading... {percent_complete:.2f}% - {downloaded_size / 1024 / 1024:.2f} MB downloaded - Elapsed Time: {format_time(time_elapsed)} - Speed: {current_speed / 1024 / 1024:.2f} MB/s" # noqa: E501
)
else:
sofar = (count * blockSize) / 1024
if sofar >= 1000:
sofar /= 1024
sys.stdout.write("\r%dMB " % (sofar))
else:
sys.stdout.write("\r%dKB" % (sofar))
sys.stdout.write(
f"\rDownloading... {downloaded_size / 1024 / 1024:.2f} MB downloaded - Elapsed Time: {format_time(time_elapsed)} - Speed: {current_speed / 1024 / 1024:.2f} MB/s" # noqa: E501
)
sys.stdout.flush()


def unpack(filename, destination):
def print_verification_progress(total_files, i, t1):
if sys.stdout.isatty():
sys.stdout.write(f"\rElapsed time {format_time(time.time() - t1)}")
sys.stdout.flush()


def verify_files(filename, destination, rename_to):
# Set the path of the extracted directory
extracted_dir_path = destination
t1 = time.time()
if filename.endswith(".zip"):
try:
with zipfile.ZipFile(filename, "r") as archive:
first_dir = archive.namelist()[0].split("/")[0]
total_files = len(archive.namelist())
for i, zipped_file in enumerate(archive.namelist(), 1):
local_path = os.path.join(extracted_dir_path, zipped_file.replace(first_dir, rename_to, 1))
if not os.path.exists(local_path):
print(f"\nMissing {zipped_file} on location: {extracted_dir_path}")
print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
return False
print_verification_progress(total_files, i, t1)
except zipfile.BadZipFile:
print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
return False
elif filename.endswith(".tar.gz"):
try:
with tarfile.open(filename, "r:gz") as archive:
first_dir = archive.getnames()[0].split("/")[0]
total_files = len(archive.getnames())
for i, zipped_file in enumerate(archive.getnames(), 1):
local_path = os.path.join(extracted_dir_path, zipped_file.replace(first_dir, rename_to, 1))
if not os.path.exists(local_path):
print(f"\nMissing {zipped_file} on location: {extracted_dir_path}")
print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
return False
print_verification_progress(total_files, i, t1)
except tarfile.ReadError:
print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
return False
elif filename.endswith(".tar.xz"):
try:
with tarfile.open(filename, "r:xz") as archive:
first_dir = archive.getnames()[0].split("/")[0]
total_files = len(archive.getnames())
for i, zipped_file in enumerate(archive.getnames(), 1):
local_path = os.path.join(extracted_dir_path, zipped_file.replace(first_dir, rename_to, 1))
if not os.path.exists(local_path):
print(f"\nMissing {zipped_file} on location: {extracted_dir_path}")
print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
return False
print_verification_progress(total_files, i, t1)
except tarfile.ReadError:
print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
return False
else:
raise NotImplementedError("Unsupported archive type")

if verbose:
print(f"\nVerification passed; completed in {format_time(time.time() - t1)}")

return True


def unpack(filename, destination, force_extract): # noqa: C901
dirname = ""
print("Extracting {0} ...".format(os.path.basename(filename)))
sys.stdout.flush()
cfile = None # Compressed file
file_is_corrupted = False
if not force_extract:
print(" > Verify archive... ", end="", flush=True)

try:
if filename.endswith("tar.gz"):
if tarfile.is_tarfile(filename):
cfile = tarfile.open(filename, "r:gz")
dirname = cfile.getnames()[0].split("/")[0]
else:
print("File corrupted!")
file_is_corrupted = True
elif filename.endswith("tar.xz"):
if tarfile.is_tarfile(filename):
cfile = tarfile.open(filename, "r:xz")
dirname = cfile.getnames()[0].split("/")[0]
else:
print("File corrupted!")
file_is_corrupted = True
elif filename.endswith("zip"):
if zipfile.is_zipfile(filename):
cfile = zipfile.ZipFile(filename)
dirname = cfile.namelist()[0].split("/")[0]
else:
print("File corrupted!")
file_is_corrupted = True
else:
raise NotImplementedError("Unsupported archive type")
except EOFError:
print("File corrupted or incomplete!")
cfile = None
file_is_corrupted = True

if file_is_corrupted:
corrupted_filename = filename + ".corrupted"
os.rename(filename, corrupted_filename)
if verbose:
print(f"Renaming corrupted archive to {corrupted_filename}")
return False

# A little trick to rename tool directories so they don't contain version number
rename_to = re.match(r"^([a-z][^\-]*\-*)+", dirname).group(0).strip("-")
if rename_to == dirname and dirname.startswith("esp32-arduino-libs-"):
rename_to = "esp32-arduino-libs"

if not force_extract:
if verify_files(filename, destination, rename_to):
print(" Files ok. Skipping Extraction")
return True
else:
print(" Extracting archive...")
else:
print(" Forcing extraction")

if filename.endswith("tar.gz"):
tfile = tarfile.open(filename, "r:gz")
tfile.extractall(destination)
dirname = tfile.getnames()[0]
if not cfile:
cfile = tarfile.open(filename, "r:gz")
cfile.extractall(destination)
elif filename.endswith("tar.xz"):
tfile = tarfile.open(filename, "r:xz")
tfile.extractall(destination)
dirname = tfile.getnames()[0]
if not cfile:
cfile = tarfile.open(filename, "r:xz")
cfile.extractall(destination)
elif filename.endswith("zip"):
zfile = zipfile.ZipFile(filename)
zfile.extractall(destination)
dirname = zfile.namelist()[0]
if not cfile:
cfile = zipfile.ZipFile(filename)
cfile.extractall(destination)
else:
raise NotImplementedError("Unsupported archive type")

# a little trick to rename tool directories so they don't contain version number
rename_to = re.match(r"^([a-z][^\-]*\-*)+", dirname).group(0).strip("-")
if rename_to == dirname and dirname.startswith("esp32-arduino-libs-"):
rename_to = "esp32-arduino-libs"
if rename_to != dirname:
print("Renaming {0} to {1} ...".format(dirname, rename_to))
if os.path.isdir(rename_to):
shutil.rmtree(rename_to)
shutil.move(dirname, rename_to)

return True


def download_file_with_progress(url, filename):
def download_file_with_progress(url, filename, start_time):
import ssl
import contextlib

Expand All @@ -124,16 +250,16 @@ def download_file_with_progress(url, filename):
with open(filename, "wb") as out_file:
out_file.write(block)
block_count += 1
report_progress(block_count, block_size, total_size)
report_progress(block_count, block_size, total_size, start_time)
while True:
block = fp.read(block_size)
if not block:
break
out_file.write(block)
block_count += 1
report_progress(block_count, block_size, total_size)
report_progress(block_count, block_size, total_size, start_time)
else:
raise Exception("nonexisting file or connection error")
raise Exception("Non-existing file or connection error")


def download_file(url, filename):
Expand All @@ -155,16 +281,21 @@ def download_file(url, filename):
break
out_file.write(block)
else:
raise Exception("nonexisting file or connection error")
raise Exception("Non-existing file or connection error")


def get_tool(tool):
def get_tool(tool, force_download, force_extract):
sys_name = platform.system()
archive_name = tool["archiveFileName"]
checksum = tool["checksum"][8:]
local_path = dist_dir + archive_name
url = tool["url"]
if not os.path.isfile(local_path):
print("Downloading " + archive_name + " ...")
start_time = time.time()
if not os.path.isfile(local_path) or force_download:
if verbose:
print("Downloading '" + archive_name + "' to '" + local_path + "'")
else:
print("Downloading '" + archive_name + "' ...")
sys.stdout.flush()
if "CYGWIN_NT" in sys_name:
import ssl
Expand All @@ -186,13 +317,18 @@ def get_tool(tool):
try:
urlretrieve(url, local_path, report_progress)
except: # noqa: E722
download_file_with_progress(url, local_path)
sys.stdout.write("\rDone \n")
download_file_with_progress(url, local_path, start_time)
sys.stdout.write(" - Done\n")
sys.stdout.flush()
else:
print("Tool {0} already downloaded".format(archive_name))
sys.stdout.flush()
unpack(local_path, ".")

if "esp32-arduino-libs" not in archive_name and sha256sum(local_path) != checksum:
print("Checksum mismatch for {0}".format(archive_name))
return False

return unpack(local_path, ".", force_extract)


def load_tools_list(filename, platform):
Expand Down Expand Up @@ -241,9 +377,44 @@ def identify_platform():


if __name__ == "__main__":
is_test = len(sys.argv) > 1 and sys.argv[1] == "-h"
parser = argparse.ArgumentParser(description="Download and extract tools")

parser.add_argument("-v", "--verbose", type=bool, default=False, required=False, help="Print verbose output")

parser.add_argument(
"-d", "--force_download", type=bool, default=False, required=False, help="Force download of tools"
)

parser.add_argument(
"-e", "--force_extract", type=bool, default=False, required=False, help="Force extraction of tools"
)

parser.add_argument(
"-f", "--force_all", type=bool, default=False, required=False, help="Force download and extraction of tools"
)

parser.add_argument("-t", "--test", type=bool, default=False, required=False, help=argparse.SUPPRESS)

args = parser.parse_args()

verbose = args.verbose
force_download = args.force_download
force_extract = args.force_extract
force_all = args.force_all
is_test = args.test

if is_test and (force_download or force_extract or force_all):
print("Cannot combine test (-t) and forced execution (-d | -e | -f)")
parser.print_help(sys.stderr)
sys.exit(1)

if is_test:
print("Test run!")

if force_all:
force_download = True
force_extract = True

identified_platform = identify_platform()
print("Platform: {0}".format(identified_platform))
tools_to_download = load_tools_list(
Expand All @@ -254,5 +425,11 @@ def identify_platform():
if is_test:
print("Would install: {0}".format(tool["archiveFileName"]))
else:
get_tool(tool)
if not get_tool(tool, force_download, force_extract):
if verbose:
print(f"Tool {tool['archiveFileName']} was corrupted. Re-downloading...\n")
if not get_tool(tool, True, force_extract):
print(f"Tool {tool['archiveFileName']} was corrupted, but re-downloading did not help!\n")
sys.exit(1)

print("Platform Tools Installed")

0 comments on commit 00fb799

Please sign in to comment.