diff --git a/src/vorta/assets/UI/sourcetab.ui b/src/vorta/assets/UI/sourcetab.ui index e17d13cd0..2e7d20838 100644 --- a/src/vorta/assets/UI/sourcetab.ui +++ b/src/vorta/assets/UI/sourcetab.ui @@ -24,7 +24,10 @@ - + + + true + 0 @@ -37,12 +40,50 @@ 0 + + QAbstractItemView::NoEditTriggers + QAbstractItemView::ExtendedSelection - - true + + QAbstractItemView::SelectRows + + + false + + false + + + false + + + false + + + false + + + + Path + + + + + Type + + + + + Size + + + + + File Count + + @@ -61,6 +102,13 @@ + + + + Remove + + + @@ -71,13 +119,26 @@ - - + + - Remove + Recalculate sizes + + + + Qt::Vertical + + + + 20 + 40 + + + + diff --git a/src/vorta/models.py b/src/vorta/models.py index 0d04b4845..ac6ef79ef 100644 --- a/src/vorta/models.py +++ b/src/vorta/models.py @@ -16,7 +16,7 @@ from vorta.i18n import trans_late from vorta.utils import slugify -SCHEMA_VERSION = 15 +SCHEMA_VERSION = 16 db = pw.Proxy() @@ -106,6 +106,9 @@ class Meta: class SourceFileModel(pw.Model): """A folder to be backed up, related to a Backup Configuration.""" dir = pw.CharField() + dir_size = pw.BigIntegerField() + dir_files_count = pw.BigIntegerField() + path_isdir = pw.BooleanField() profile = pw.ForeignKeyField(BackupProfileModel, default=1) added_at = pw.DateTimeField(default=datetime.utcnow) @@ -215,6 +218,11 @@ def get_misc_settings(): 'label': trans_late('settings', 'Open main window on startup') }, + { + 'key': 'get_srcpath_datasize', 'value': True, 'type': 'checkbox', + 'label': trans_late('settings', + 'Get statistics of file/folder when added') + }, { 'key': 'previous_profile_id', 'str_value': '1', 'type': 'internal', 'label': 'Previously selected profile' @@ -362,6 +370,17 @@ def init_db(con=None): 'dont_run_on_metered_networks', pw.BooleanField(default=True)) ) + if current_schema.version < 16: + _apply_schema_update( + current_schema, 16, + migrator.add_column(SourceFileModel._meta.table_name, + 'dir_size', pw.BigIntegerField(default=-1)), + migrator.add_column(SourceFileModel._meta.table_name, + 'dir_files_count', pw.BigIntegerField(default=-1)), + migrator.add_column(SourceFileModel._meta.table_name, + 'path_isdir', pw.BooleanField(default=False)) + ) + # Create missing settings and update labels. Leave setting values untouched. for setting in get_misc_settings(): s, created = SettingsModel.get_or_create(key=setting['key'], defaults=setting) diff --git a/src/vorta/utils.py b/src/vorta/utils.py index 78980c8db..c3ae92d57 100644 --- a/src/vorta/utils.py +++ b/src/vorta/utils.py @@ -17,6 +17,7 @@ from paramiko.ed25519key import Ed25519Key from paramiko.rsakey import RSAKey from PyQt5 import QtCore +from PyQt5.QtCore import QFileInfo, QThread, pyqtSignal from PyQt5.QtWidgets import QApplication, QFileDialog, QSystemTrayIcon from vorta.borg._compatibility import BorgCompatibility @@ -27,9 +28,48 @@ QApplication.setAttribute(QtCore.Qt.AA_UseHighDpiPixmaps, True) # use highdpi icons borg_compat = BorgCompatibility() +_network_status_monitor = None -_network_status_monitor = None +class FilePathInfoAsync(QThread): + signal = pyqtSignal(str, str, str) + + def __init__(self, path): + self.path = path + QThread.__init__(self) + self.exiting = False + + def run(self): + # logger.info("running thread to get path=%s...", self.path) + self.files_count = 0 + self.size, self.files_count = get_path_datasize(self.path) + self.signal.emit(self.path, str(self.size), str(self.files_count)) + + +def get_directory_size(dir_path): + ''' Get number of files only and total size in bytes from a path. + Based off https://stackoverflow.com/a/17936789 ''' + data_size = 0 + seen = set() + + for curr_path, _, file_names in os.walk(dir_path): + for file_name in file_names: + file_path = os.path.join(curr_path, file_name) + + # Ignore symbolic links, since borg doesn't follow them + if os.path.islink(file_path): + continue + + stat = os.stat(file_path) + + # Visit each file once + if stat.st_ino not in seen: + seen.add(stat.st_ino) + data_size += stat.st_size + + files_count = len(seen) + + return data_size, files_count def get_network_status_monitor(): @@ -40,6 +80,22 @@ def get_network_status_monitor(): return _network_status_monitor +def get_path_datasize(path): + file_info = QFileInfo(path) + data_size = 0 + + if file_info.isDir(): + data_size, files_count = get_directory_size(file_info.absoluteFilePath()) + # logger.info("path (folder) %s %u elements size now=%u (%s)", + # file_info.absoluteFilePath(), files_count, data_size, pretty_bytes(data_size)) + else: + # logger.info("path (file) %s size=%u", file_info.path(), file_info.size()) + data_size = file_info.size() + files_count = 1 + + return data_size, files_count + + def nested_dict(): """ Combination of two idioms to quickly build dicts from lists of keys: @@ -98,10 +154,24 @@ def get_private_keys(): return available_private_keys +def sort_sizes(size_list): + """ Sorts sizes with extensions. Assumes that size is already in largest unit possible """ + final_list = [] + for suffix in [" B", " KB", " MB", " GB", " TB"]: + sub_list = [float(size[:-len(suffix)]) + for size in size_list if size.endswith(suffix) and size[:-len(suffix)][-1].isnumeric()] + sub_list.sort() + final_list += [(str(size) + suffix) for size in sub_list] + # Skip additional loops + if len(final_list) == len(size_list): + break + return final_list + + def pretty_bytes(size): """from https://stackoverflow.com/questions/12523586/ python-format-size-application-converting-b-to-kb-mb-gb-tb/37423778""" - if type(size) != int: + if not isinstance(size, int): return '' power = 1000 # GiB is base 2**10, GB is base 10**3. n = 0 diff --git a/src/vorta/views/source_tab.py b/src/vorta/views/source_tab.py index 305931995..89096fc70 100644 --- a/src/vorta/views/source_tab.py +++ b/src/vorta/views/source_tab.py @@ -1,49 +1,161 @@ from PyQt5 import uic -from ..models import SourceFileModel, BackupProfileMixin -from ..utils import get_asset, choose_file_dialog -from PyQt5.QtWidgets import QApplication, QMessageBox +from ..models import SourceFileModel, BackupProfileMixin, SettingsModel +from ..utils import get_asset, choose_file_dialog, pretty_bytes, sort_sizes, FilePathInfoAsync +from PyQt5 import QtCore +from PyQt5.QtCore import QFileInfo +from PyQt5.QtWidgets import QApplication, QMessageBox, QTableWidgetItem, QHeaderView import os uifile = get_asset('UI/sourcetab.ui') SourceUI, SourceBase = uic.loadUiType(uifile) +class SourceColumn: + Path = 0 + Type = 1 + Size = 2 + FilesCount = 3 + + +class SizeItem(QTableWidgetItem): + def __lt__(self, other): + return sort_sizes([self.text(), other.text()]) == [self.text(), other.text()] + + +class FilesCount(QTableWidgetItem): + def __lt__(self, other): + # Verify that conversion is only performed on valid integers + # If one of the 2 elements is no number, put these elements at the end + # This is important if the text is "Calculating..." or "" + if self.text().isdigit() and other.text().isdigit(): + return int(self.text()) < int(other.text()) # Compare & return result + else: + if not self.text().isdigit(): + return 1 # Move one down if current item has no valid count + if not other.text().isdigit(): + return 0 + + class SourceTab(SourceBase, SourceUI, BackupProfileMixin): + updateThreads = [] + def __init__(self, parent=None): super().__init__(parent) self.setupUi(parent) + header = self.sourceFilesWidget.horizontalHeader() + + header.setVisible(True) + header.setSortIndicatorShown(1) + + header.setSectionResizeMode(SourceColumn.Path, QHeaderView.Stretch) + header.setSectionResizeMode(SourceColumn.Type, QHeaderView.ResizeToContents) + header.setSectionResizeMode(SourceColumn.Size, QHeaderView.ResizeToContents) + header.setSectionResizeMode(SourceColumn.FilesCount, QHeaderView.ResizeToContents) + + self.sourceFilesWidget.setSortingEnabled(True) self.sourceAddFolder.clicked.connect(lambda: self.source_add(want_folder=True)) self.sourceAddFile.clicked.connect(lambda: self.source_add(want_folder=False)) self.sourceRemove.clicked.connect(self.source_remove) + self.sourcesUpdate.clicked.connect(self.sources_update) self.paste.clicked.connect(self.paste_text) self.excludePatternsField.textChanged.connect(self.save_exclude_patterns) self.excludeIfPresentField.textChanged.connect(self.save_exclude_if_present) self.populate_from_profile() + def set_path_info(self, path, data_size, files_count): + items = self.sourceFilesWidget.findItems(path, QtCore.Qt.MatchExactly) + # Conversion int->str->int needed because QT limits int to 32-bit + data_size = int(data_size) + files_count = int(files_count) + + for item in items: + db_item = SourceFileModel.get(dir=path) + if QFileInfo(path).isDir(): + self.sourceFilesWidget.item(item.row(), SourceColumn.Type).setText(self.tr("Folder")) + self.sourceFilesWidget.item(item.row(), SourceColumn.FilesCount).setText(format(files_count)) + db_item.path_isdir = True + else: + self.sourceFilesWidget.item(item.row(), SourceColumn.Type).setText(self.tr("File")) + # No files count, if entry itself is a file + self.sourceFilesWidget.item(item.row(), SourceColumn.FilesCount).setText("") + db_item.path_isdir = False + self.sourceFilesWidget.item(item.row(), SourceColumn.Size).setText(pretty_bytes(data_size)) + + db_item.dir_size = data_size + db_item.dir_files_count = files_count + db_item.save() + # Remove thread from list when it's done + for thrd in self.updateThreads: + if thrd.objectName() == path: + self.updateThreads.remove(thrd) + + def update_path_info(self, index_row): + path = self.sourceFilesWidget.item(index_row, SourceColumn.Path).text() + self.sourceFilesWidget.item(index_row, SourceColumn.Type).setText(self.tr("Calculating...")) + self.sourceFilesWidget.item(index_row, SourceColumn.Size).setText(self.tr("Calculating...")) + self.sourceFilesWidget.item(index_row, SourceColumn.FilesCount).setText(self.tr("Calculating...")) + getDir = FilePathInfoAsync(path) + getDir.signal.connect(self.set_path_info) + getDir.setObjectName(path) + self.updateThreads.append(getDir) # this is ugly, is there a better way to keep the thread object? + getDir.start() + + def add_source_to_table(self, source, update_data): + index_row = self.sourceFilesWidget.rowCount() + self.sourceFilesWidget.insertRow(index_row) + # Insert all items on current row + self.sourceFilesWidget.setItem(index_row, SourceColumn.Path, QTableWidgetItem(source.dir)) + self.sourceFilesWidget.setItem(index_row, SourceColumn.Type, QTableWidgetItem("")) + self.sourceFilesWidget.setItem(index_row, SourceColumn.Size, SizeItem("")) + self.sourceFilesWidget.setItem(index_row, SourceColumn.FilesCount, FilesCount("")) + + if update_data: + self.update_path_info(index_row) + else: # Use cached data from DB + if source.dir_size > -1: + self.sourceFilesWidget.item(index_row, SourceColumn.Size).setText(pretty_bytes(source.dir_size)) + + if source.path_isdir: + self.sourceFilesWidget.item(index_row, SourceColumn.Type).setText(self.tr("Folder")) + self.sourceFilesWidget.item(index_row, + SourceColumn.FilesCount).setText(format(source.dir_files_count)) + else: + self.sourceFilesWidget.item(index_row, SourceColumn.Type).setText(self.tr("File")) + def populate_from_profile(self): profile = self.profile() self.excludePatternsField.textChanged.disconnect() self.excludeIfPresentField.textChanged.disconnect() - self.sourceFilesWidget.clear() + self.sourceFilesWidget.setRowCount(0) # Clear rows self.excludePatternsField.clear() self.excludeIfPresentField.clear() for source in SourceFileModel.select().where(SourceFileModel.profile == profile): - self.sourceFilesWidget.addItem(source.dir) + self.add_source_to_table(source, False) + # Initially, sort entries by path name in ascending order + self.sourceFilesWidget.model().sort(SourceColumn.Path, QtCore.Qt.AscendingOrder) self.excludePatternsField.appendPlainText(profile.exclude_patterns) self.excludeIfPresentField.appendPlainText(profile.exclude_if_present) self.excludePatternsField.textChanged.connect(self.save_exclude_patterns) self.excludeIfPresentField.textChanged.connect(self.save_exclude_if_present) + def sources_update(self): + for row in range(0, self.sourceFilesWidget.rowCount()): + self.update_path_info(row) # Update data for each entry + def source_add(self, want_folder): def receive(): dirs = dialog.selectedFiles() for dir in dirs: - new_source, created = SourceFileModel.get_or_create(dir=dir, profile=self.profile()) + new_source, created = SourceFileModel.get_or_create(dir=dir, + dir_size=-1, + dir_files_count=-1, + path_isdir=False, + profile=self.profile()) if created: - self.sourceFilesWidget.addItem(dir) + self.add_source_to_table(new_source, SettingsModel.get(key="get_srcpath_datasize").value) new_source.save() msg = self.tr("Choose directory to back up") if want_folder else self.tr("Choose file(s) to back up") @@ -51,14 +163,14 @@ def receive(): dialog.open(receive) def source_remove(self): - indexes = self.sourceFilesWidget.selectionModel().selectedIndexes() + indexes = self.sourceFilesWidget.selectionModel().selectedRows() # sort indexes, starting with lowest indexes.sort() - # remove each selected entry, starting with highest index (otherways, higher indexes become invalid) + # remove each selected row, starting with highest index (otherways, higher indexes become invalid) for index in reversed(indexes): - item = self.sourceFilesWidget.takeItem(index.row()) - db_item = SourceFileModel.get(dir=item.text()) + db_item = SourceFileModel.get(dir=self.sourceFilesWidget.item(index.row(), SourceColumn.Path).text()) db_item.delete_instance() + self.sourceFilesWidget.removeRow(index.row()) def save_exclude_patterns(self): profile = self.profile() diff --git a/tests/conftest.py b/tests/conftest.py index 475e74709..a163a8751 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -37,7 +37,7 @@ def init_db(qapp): test_archive1 = ArchiveModel(snapshot_id='99998', name='test-archive1', time=dt(2000, 1, 1, 0, 0), repo=1) test_archive1.save() - source_dir = SourceFileModel(dir='/tmp/another', repo=new_repo) + source_dir = SourceFileModel(dir='/tmp/another', repo=new_repo, dir_size=100, dir_files_count=18, path_isdir=True) source_dir.save() qapp.main_window = MainWindow(qapp) # Re-open main window to apply mock data in UI diff --git a/tests/test_source.py b/tests/test_source.py index b461dad09..0fb751a61 100644 --- a/tests/test_source.py +++ b/tests/test_source.py @@ -11,4 +11,4 @@ def test_add_folder(qapp, qtbot, tmpdir, monkeypatch, choose_file_dialog): tab = main.sourceTab tab.sourceAddFolder.click() - qtbot.waitUntil(lambda: tab.sourceFilesWidget.count() == 2, timeout=5000) + qtbot.waitUntil(lambda: tab.sourceFilesWidget.rowCount() == 2, timeout=5000)