Download progress bar and checking for botpack updates

rlbot_gui/bot_management/downloader.py

@@ -1,45 +1,152 @@
 import os
 import tempfile
-import urllib
+import urllib.request
 import zipfile
+import json
+import eel
+import time
 from pathlib import Path
 from shutil import rmtree
 
 
-def download_and_extract_zip(download_url: str, local_folder_path: Path, clobber: bool = False):
+def download_and_extract_zip(download_url: str, local_folder_path: Path,
+                             clobber: bool = False, progress_callback: callable = None,
+                             unzip_callback: callable = None):
     """
     :param clobber: If true, we will delete anything found in local_folder_path.
     :return:
     """
 
     with tempfile.TemporaryDirectory() as tmpdirname:
         downloadedZip = os.path.join(tmpdirname, 'downloaded.zip')
-        urllib.request.urlretrieve(download_url, downloadedZip)
+        urllib.request.urlretrieve(download_url, downloadedZip, progress_callback)
 
         if clobber and os.path.exists(local_folder_path):
             rmtree(local_folder_path)
 
+        if unzip_callback:
+            unzip_callback()
+
         with zipfile.ZipFile(downloadedZip, 'r') as zip_ref:
             zip_ref.extractall(local_folder_path)
 
 
-def download_gitlfs(repo_url: str, checkout_folder: Path, branch_name: str):
-    print('Starting download of a git repo... this might take a while.')
+def get_json_from_url(url):
+    response = urllib.request.urlopen(url)
+    return json.loads(response.read())
+
+
+def get_repo_size(repo_full_name):
+    """
+    Call GitHub API to get an estimate size of a GitHub repository.
+    :param repo_full_name: Full name of a repository. Example: 'RLBot/RLBotPack'
+    :return: Size of the repository in bytes, or 0 if the API call fails.
+    """
+    try:
+        data = get_json_from_url('https://api.github.com/repos/' + repo_full_name)
+        return int(data["size"]) * 1000
+    except:
+        return 0
+
+
+class BotpackDownloader:
+    """
+    Downloads the botpack while updating the progress bar and status text.
+    """
+
+    PROGRESSBAR_UPDATE_INTERVAL = 0.1  # How often to update the progress bar (seconds)
+    ZIP_PORTION = 0.6  # How much of total progress should the zip file take
+
+    def __init__(self):
+        self.status = ''
+        self.current_progress = 0
+        self.total_progress = 0
+
+        self.estimated_zip_size = 0
+        self.current_file_bytes_downloaded = 0
+        self.lfs_file_count = 0
+        self.lfs_files_completed = 0
+        self.one_lfs_file_portion = 0
+        self.last_progressbar_update_time = 0
+
+    def update_progressbar_and_status(self):
+        # it's not necessary to update on every callback, so update
+        # only when some amount of time has passed
+        now = time.time()
+        if now > self.last_progressbar_update_time + self.PROGRESSBAR_UPDATE_INTERVAL:
+            self.last_progressbar_update_time = now
+
+            total_progress_percent = self.total_progress * 100
+            current_progress_percent = int(self.current_progress * 100)
+            status = f'{self.status} ({current_progress_percent}%)'
+
+            eel.updateDownloadProgress(total_progress_percent, status)
+
+    def zip_download_callback(self, block_count, block_size, _):
+        self.current_file_bytes_downloaded += block_size
+        self.current_progress = min(self.current_file_bytes_downloaded / self.estimated_zip_size, 1.0)
+        self.total_progress = self.current_progress * self.ZIP_PORTION
+        self.update_progressbar_and_status()
+
+    def unzip_callback(self):
+        eel.updateDownloadProgress(int(self.ZIP_PORTION * 100), 'Extracting ZIP file')
+
+    def lfs_download_callback(self, block_count, block_size, total_size):
+        self.current_file_bytes_downloaded += block_size
+        self.current_progress = self.current_file_bytes_downloaded / total_size
+        prev_files_progress = self.ZIP_PORTION + self.one_lfs_file_portion * self.lfs_files_completed
+        self.total_progress = prev_files_progress + self.current_progress * self.one_lfs_file_portion
+        self.update_progressbar_and_status()
+
+    def download(self, repo_owner: str, repo_name: str, branch_name: str, checkout_folder: Path):
+        repo_full_name = repo_owner + '/' + repo_name
+        repo_url = 'https://github.com/' + repo_full_name
+        repo_extraction_name = f'/{repo_name}-{branch_name}/'
+
+        self.status = f'Downloading {repo_full_name}-{branch_name}'
+        print(self.status)
+        self.current_progress = 0
+        self.total_progress = 0
+
+        # Unfortunately we can't know the size of the zip file before downloading it,
+        # so we have to get the size from the GitHub API.
+        self.estimated_zip_size = get_repo_size(repo_full_name) * 0.9
+
+        # If we fail to get the repo size, set it to a fallback value,
+        # so the progress bar will show atleast some progress.
+        # Let's assume the zip file is around 50 MB.
+        if self.estimated_zip_size == 0:
+            self.estimated_zip_size = 50_000_000
+
+        # Download the most of the files eg. https://github.com/RLBot/RLBotPack/archive/master.zip
+        download_and_extract_zip(
+            download_url=repo_url + '/archive/' + branch_name + '.zip',
+            local_folder_path=checkout_folder, clobber=True,
+            progress_callback=self.zip_download_callback,
+            unzip_callback=self.unzip_callback)
+
+        # Get paths of all LFS files we need to download from the .gitattributes file.
+        lfs_file_paths = []
+        gitattributes_file = open(checkout_folder + repo_extraction_name + '.gitattributes')
+        for line in gitattributes_file:
+            if line.split(' ')[1].endswith('=lfs'):
+                lfs_file_paths.append(line.split(' ')[0])
+        gitattributes_file.close()
 
-    # Download the most of the files eg. https://github.com/RLBot/RLBotPack/archive/master.zip
-    download_and_extract_zip(
-        download_url=repo_url + '/archive/' + branch_name + '.zip',
-        local_folder_path=checkout_folder, clobber=True)
+        self.lfs_file_count = len(lfs_file_paths)
+        # How much of total progress does one LFS file take:
+        self.one_lfs_file_portion = (1.0 - self.ZIP_PORTION) / self.lfs_file_count
 
-    repo_extraction_name = '/' + repo_url.split('/')[-1] + '-' + branch_name + '/'
+        # Download all LFS files.
+        for lfs_file_path in lfs_file_paths:
+            self.current_progress = 0
+            self.current_file_bytes_downloaded = 0
+            self.status = 'Downloading ' + lfs_file_path
+            print(self.status)
 
-    file = open(checkout_folder + repo_extraction_name + '.gitattributes')
-    for line in file:
-        if line.split(' ')[1].endswith('=lfs'):
-            # Download gitlfs file
             urllib.request.urlretrieve(
-                repo_url + '/raw/' + branch_name + '/' + line.split(' ')[0],
-                checkout_folder + repo_extraction_name + line.split(' ')[0])
-    file.close()
+                repo_url + '/raw/' + branch_name + '/' + lfs_file_path,
+                checkout_folder + repo_extraction_name + lfs_file_path,
+                self.lfs_download_callback)
 
-    print('Done downloading git repo.')
+            self.lfs_files_completed += 1

rlbot_gui/gui.py

@@ -15,16 +15,20 @@
     existing_match_behavior_types
 
 from rlbot_gui.bot_management.bot_creation import bootstrap_python_bot, bootstrap_scratch_bot, convert_to_filename
-from rlbot_gui.bot_management.downloader import download_gitlfs
+from rlbot_gui.bot_management.downloader import BotpackDownloader, get_json_from_url
 from rlbot_gui.match_runner.match_runner import hot_reload_bots, shut_down, start_match_helper, do_infinite_loop_content
 
 DEFAULT_BOT_FOLDER = 'default_bot_folder'
 BOTPACK_FOLDER = 'RLBotPackDeletable'
 OLD_BOTPACK_FOLDER = 'RLBotPack'
+BOTPACK_REPO_OWNER = 'RLBot'
+BOTPACK_REPO_NAME = 'RLBotPack'
+BOTPACK_REPO_BRANCH = 'master'
 CREATED_BOTS_FOLDER = 'MyBots'
 BOT_FOLDER_SETTINGS_KEY = 'bot_folder_settings'
 MATCH_SETTINGS_KEY = 'match_settings'
 TEAM_SETTINGS_KEY = 'team_settings'
+COMMIT_ID_KEY = 'latest_botpack_commit_id'
 settings = QSettings('rlbotgui', 'preferences')
 
 bot_folder_settings = settings.value(BOT_FOLDER_SETTINGS_KEY, type=dict)
@@ -258,13 +262,33 @@ def install_package(package_string):
     return {'exitCode': exit_code, 'package': package_string}
 
 
+def get_last_botpack_commit_id():
+    url = f'https://api.github.com/repos/{BOTPACK_REPO_OWNER}/{BOTPACK_REPO_NAME}/branches/{BOTPACK_REPO_BRANCH}'
+    try:
+        return get_json_from_url(url)['commit']['sha']
+    except:
+        return 'unknown'
+
+
+@eel.expose
+def is_botpack_up_to_date():
+    local_commit_id = settings.value(COMMIT_ID_KEY, type=str)
+
+    if not local_commit_id:
+        return False
+
+    github_commit_id = get_last_botpack_commit_id()
+
+    if github_commit_id == 'unknown':
+        return True
+
+    return github_commit_id == local_commit_id
+
+
 @eel.expose
 def download_bot_pack():
     # The bot pack in now hosted at https://github.com/RLBot/RLBotPack
-    download_gitlfs(
-        repo_url="https://github.com/RLBot/RLBotPack",
-        checkout_folder=BOTPACK_FOLDER,
-        branch_name='master')
+    BotpackDownloader().download(BOTPACK_REPO_OWNER, BOTPACK_REPO_NAME, BOTPACK_REPO_BRANCH, BOTPACK_FOLDER)
 
     # Configure the folder settings.
     bot_folder_settings['folders'][os.path.abspath(BOTPACK_FOLDER)] = {'visible': True}
@@ -274,6 +298,7 @@ def download_bot_pack():
         bot_folder_settings['folders'][os.path.abspath(OLD_BOTPACK_FOLDER)] = {'visible': False}
 
     settings.setValue(BOT_FOLDER_SETTINGS_KEY, bot_folder_settings)
+    settings.setValue(COMMIT_ID_KEY, get_last_botpack_commit_id())
     settings.sync()
 
 

rlbot_gui/gui/js/main.js

@@ -65,7 +65,11 @@ const app = new Vue({
             folders: []
         },
         showFolderSettingsDialog: false,
-        showExtraOptions: false
+        showExtraOptions: false,
+        showDownloadProgressDialog: false,
+        downloadProgressPercent: 0,
+        downloadStatus: '',
+        showBotpackUpdateSnackbar: false
     },
     methods: {
         startMatch: function (event) {
@@ -109,7 +113,7 @@ const app = new Vue({
             this.matchSettings.instant_start = false;
             this.matchSettings.enable_lockstep = false;
             this.resetMutatorsToDefault();
-        
+
             this.updateBGImage(this.matchSettings.map);
         },
         updateBGImage: function(mapName) {
@@ -120,7 +124,10 @@ const app = new Vue({
             eel.install_package(this.packageString)(onInstallationComplete);
         },
         downloadBotPack: function() {
-            this.showProgressSpinner = true;
+            this.showBotpackUpdateSnackbar = false;
+            this.showDownloadProgressDialog = true;
+            this.downloadStatus = "Starting"
+            this.downloadProgressPercent = 0;
             eel.download_bot_pack()(botPackDownloaded);
         },
         showBotInExplorer: function (botPath) {
@@ -164,9 +171,16 @@ eel.get_language_support()((support) => {
     applyLanguageWarnings();
 });
 
+eel.is_botpack_up_to_date()(botpackUpdateChecked);
+
+function botpackUpdateChecked(isBotpackUpToDate) {
+    app.showBotpackUpdateSnackbar = !isBotpackUpToDate;
+}
+
 function botPackDownloaded(response) {
     app.snackbarContent = 'Downloaded Bot Pack!';
     app.showSnackbar = true;
+    app.showDownloadProgressDialog = false;
     eel.get_folder_settings()(folderSettingsReceived);
     eel.scan_for_bots()(botsReceived);
 }
@@ -242,6 +256,12 @@ function onInstallationComplete(result) {
     app.showProgressSpinner = false;
 }
 
+eel.expose(updateDownloadProgress);
+function updateDownloadProgress(progress, status) {
+    app.downloadStatus = status;
+    app.downloadProgressPercent = progress;
+}
+
 Vue.component('mutator-field', {
         props: ['label', 'options', 'value'],
         data: function() {

rlbot_gui/gui/main.html

@@ -260,6 +260,11 @@ <h3 class="md-title" style="flex: 1">RLBot</h3>
 				<span>{{snackbarContent}}</span>
 			</md-snackbar>
 
+			<md-snackbar md-position="center" :md-active="showBotpackUpdateSnackbar" md-persistent md-duration="10000">
+				<span>Bot Pack update available!</span>
+				<md-button class="md-accent" @click="downloadBotPack()">Download</md-button>
+			</md-snackbar>
+
 			<md-dialog v-if="activeBot && activeBot.info" :md-active.sync="showBotInfo">
 				<md-dialog-title>
 					{{activeBot.name}}
@@ -357,6 +362,22 @@ <h3 class="md-title" style="flex: 1">RLBot</h3>
 				</md-dialog-actions>
 			</md-dialog>
 
+			<md-dialog :md-active.sync="showDownloadProgressDialog"
+					   :md-close-on-esc="false"
+					   :md-click-outside-to-close="false">
+				<md-dialog-title>Downloading Bot Pack</md-dialog-title>
+
+				<md-dialog-content>
+					<div class="md-layout md-gutter" :class="`md-alignment-center-center`">
+						<md-icon class="md-size-4x">cloud_download</md-icon>
+					</div>
+					<md-progress-bar class="md-accent" md-mode="determinate" :md-value="downloadProgressPercent">
+					</md-progress-bar>
+					<p>{{ downloadStatus }}</p>
+				</md-dialog-content>
+
+			</md-dialog>
+
 		</div>
 	</div>
 

setup.py

@@ -1,6 +1,6 @@
 import setuptools
 
-__version__ = '0.0.29'
+__version__ = '0.0.30'
 
 with open("README.md", "r") as readme_file:
     long_description = readme_file.read()