diff --git a/requirements.d/Brewfile b/requirements.d/Brewfile index a3f4b08bc..6dbea82b0 100644 --- a/requirements.d/Brewfile +++ b/requirements.d/Brewfile @@ -4,6 +4,7 @@ brew 'create-dmg' brew 'qt' brew 'hub' +brew 'pre-commit' brew 'xmlstarlet' cask 'qt-creator' cask 'sparkle' diff --git a/setup.cfg b/setup.cfg index e0cdd9319..745536cf6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -105,7 +105,7 @@ extension-pkg-whitelist=PyQt6 load-plugins= [pylint.messages control] -disable= W0503,W0511,C0301,R0903,R0201,W0212,C0114,C0115,C0116,C0103,E0611,E1120,C0415,R0914,R0912,R0915 +disable= W0511,C0301,R0903,W0212,C0114,C0115,C0116,C0103,E0611,E1120,C0415,R0914,R0912,R0915 [pylint.format] max-line-length=120 diff --git a/src/vorta/store/models.py b/src/vorta/store/models.py index 400299fb2..d809814b4 100644 --- a/src/vorta/store/models.py +++ b/src/vorta/store/models.py @@ -12,7 +12,7 @@ import peewee as pw from playhouse import signals -from vorta.utils import slugify +from vorta.store.utils import slugify from vorta.views.utils import get_exclusion_presets DB = pw.Proxy() diff --git a/src/vorta/store/utils.py b/src/vorta/store/utils.py new file mode 100644 index 000000000..fc013309f --- /dev/null +++ b/src/vorta/store/utils.py @@ -0,0 +1,15 @@ +import re +import unicodedata + + +def slugify(value): + """ + Converts to lowercase, removes non-word characters (alphanumerics and + underscores) and converts spaces to hyphens. Also strips leading and + trailing whitespace. + + Copied from Django. + """ + value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii') + value = re.sub(r'[^\w\s-]', '', value).strip().lower() + return re.sub(r'[-\s]+', '-', value) diff --git a/src/vorta/utils.py b/src/vorta/utils.py index 033b486a5..9ea88c259 100644 --- a/src/vorta/utils.py +++ b/src/vorta/utils.py @@ -1,21 +1,21 @@ import argparse import errno -import fnmatch import getpass import math import os import re import socket import sys -import unicodedata from datetime import datetime as dt from functools import reduce from typing import Any, Callable, Iterable, List, Optional, Tuple, TypeVar -import psutil from PyQt6 import QtCore -from PyQt6.QtCore import QFileInfo, QThread, pyqtSignal -from PyQt6.QtWidgets import QApplication, QFileDialog, QSystemTrayIcon +from PyQt6.QtWidgets import ( + QApplication, + QFileDialog, + QSystemTrayIcon, +) from vorta.borg._compatibility import BorgCompatibility from vorta.log import logger @@ -26,130 +26,9 @@ DEFAULT_DIR_FLAG = object() METRIC_UNITS = ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'] NONMETRIC_UNITS = ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi'] - -borg_compat = BorgCompatibility() _network_status_monitor = None - -class FilePathInfoAsync(QThread): - signal = pyqtSignal(str, str, str) - - def __init__(self, path, exclude_patterns_str): - self.path = path - QThread.__init__(self) - self.exiting = False - self.exclude_patterns = [] - for _line in (exclude_patterns_str or '').splitlines(): - line = _line.strip() - if line != '': - self.exclude_patterns.append(line) - - def run(self): - # logger.info("running thread to get path=%s...", self.path) - self.size, self.files_count = get_path_datasize(self.path, self.exclude_patterns) - self.signal.emit(self.path, str(self.size), str(self.files_count)) - - -def normalize_path(path): - """normalize paths for MacOS (but do nothing on other platforms)""" - # HFS+ converts paths to a canonical form, so users shouldn't be required to enter an exact match. - # Windows and Unix filesystems allow different forms, so users always have to enter an exact match. - return unicodedata.normalize('NFD', path) if sys.platform == 'darwin' else path - - -# prepare patterns as borg does -# see `FnmatchPattern._prepare` at -# https://github.com/borgbackup/borg/blob/master//src/borg/patterns.py -def prepare_pattern(pattern): - """Prepare and process fnmatch patterns as borg does""" - if pattern.endswith(os.path.sep): - # trailing sep indicates that the contents should be excluded - # but not the directory it self. - pattern = os.path.normpath(pattern).rstrip(os.path.sep) - pattern += os.path.sep + '*' + os.path.sep - else: - pattern = os.path.normpath(pattern) + os.path.sep + '*' - - pattern = pattern.lstrip(os.path.sep) # sep at beginning is removed - return re.compile(fnmatch.translate(pattern)) - - -def match(pattern: re.Pattern, path: str): - """Check whether a path matches the given pattern.""" - path = path.lstrip(os.path.sep) + os.path.sep - return pattern.match(path) is not None - - -def get_directory_size(dir_path, exclude_patterns): - '''Get number of files only and total size in bytes from a path. - Based off https://stackoverflow.com/a/17936789''' - exclude_patterns = [prepare_pattern(p) for p in exclude_patterns] - - data_size_filtered = 0 - seen = set() - seen_filtered = set() - - for dir_path, subdirectories, file_names in os.walk(dir_path, topdown=True): - is_excluded = False - for pattern in exclude_patterns: - if match(pattern, dir_path): - is_excluded = True - break - - if is_excluded: - subdirectories.clear() # so that os.walk won't walk them - continue - - for file_name in file_names: - file_path = os.path.join(dir_path, file_name) - - # Ignore symbolic links, since borg doesn't follow them - if os.path.islink(file_path): - continue - - is_excluded = False - for pattern in exclude_patterns: - if match(pattern, file_path): - is_excluded = True - break - - try: - stat = os.stat(file_path) - if stat.st_ino not in seen: # Visit each file only once - # this won't add the size of a hardlinked file - seen.add(stat.st_ino) - if not is_excluded: - data_size_filtered += stat.st_size - seen_filtered.add(stat.st_ino) - except (FileNotFoundError, PermissionError): - continue - - files_count_filtered = len(seen_filtered) - - return data_size_filtered, files_count_filtered - - -def get_network_status_monitor(): - global _network_status_monitor - if _network_status_monitor is None: - _network_status_monitor = NetworkStatusMonitor.get_network_status_monitor() - logger.info( - 'Using %s NetworkStatusMonitor implementation.', - _network_status_monitor.__class__.__name__, - ) - return _network_status_monitor - - -def get_path_datasize(path, exclude_patterns): - file_info = QFileInfo(path) - - if file_info.isDir(): - data_size, files_count = get_directory_size(file_info.absoluteFilePath(), exclude_patterns) - else: - data_size = file_info.size() - files_count = 1 - - return data_size, files_count +borg_compat = BorgCompatibility() def nested_dict(): @@ -220,23 +99,6 @@ def get_private_keys() -> List[str]: return available_private_keys -def sort_sizes(size_list): - """Sorts sizes with extensions. Assumes that size is already in largest unit possible""" - final_list = [] - for suffix in [" B", " KB", " MB", " GB", " TB", " PB", " EB", " ZB", " YB"]: - sub_list = [ - float(size[: -len(suffix)]) - for size in size_list - if size.endswith(suffix) and size[: -len(suffix)][-1].isnumeric() - ] - sub_list.sort() - final_list += [(str(size) + suffix) for size in sub_list] - # Skip additional loops - if len(final_list) == len(size_list): - break - return final_list - - Number = TypeVar("Number", int, float) @@ -245,6 +107,17 @@ def clamp(n: Number, min_: Number, max_: Number) -> Number: return min(max_, max(n, min_)) +def get_network_status_monitor(): + global _network_status_monitor + if _network_status_monitor is None: + _network_status_monitor = NetworkStatusMonitor.get_network_status_monitor() + logger.info( + 'Using %s NetworkStatusMonitor implementation.', + _network_status_monitor.__class__.__name__, + ) + return _network_status_monitor + + def find_best_unit_for_sizes(sizes: Iterable[int], metric: bool = True, precision: int = 1) -> int: """ Selects the index of the biggest unit (see the lists in the pretty_bytes function) capable of @@ -303,38 +176,6 @@ def get_asset(path): return os.path.join(bundle_dir, path) -def get_sorted_wifis(profile): - """ - Get Wifi networks known to the OS (only current one on macOS) and - merge with networks from other profiles. Update last connected time. - """ - - from vorta.store.models import WifiSettingModel - - # Pull networks known to OS and all other backup profiles - system_wifis = get_network_status_monitor().get_known_wifis() - from_other_profiles = WifiSettingModel.select().where(WifiSettingModel.profile != profile.id).execute() - - for wifi in list(from_other_profiles) + system_wifis: - db_wifi, created = WifiSettingModel.get_or_create( - ssid=wifi.ssid, - profile=profile.id, - defaults={'last_connected': wifi.last_connected, 'allowed': True}, - ) - - # Update last connected time - if not created and db_wifi.last_connected != wifi.last_connected: - db_wifi.last_connected = wifi.last_connected - db_wifi.save() - - # Finally return list of networks and settings for that profile - return ( - WifiSettingModel.select() - .where(WifiSettingModel.profile == profile.id) - .order_by(-WifiSettingModel.last_connected) - ) - - def parse_args(): parser = argparse.ArgumentParser(description='Vorta Backup GUI for Borg.') parser.add_argument('--version', '-V', action='store_true', help="Show version and exit.") @@ -368,19 +209,6 @@ def parse_args(): return parser.parse_known_args()[0] -def slugify(value): - """ - Converts to lowercase, removes non-word characters (alphanumerics and - underscores) and converts spaces to hyphens. Also strips leading and - trailing whitespace. - - Copied from Django. - """ - value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii') - value = re.sub(r'[^\w\s-]', '', value).strip().lower() - return re.sub(r'[-\s]+', '-', value) - - def uses_dark_mode(): """ This function detects if we are running in dark mode (e.g. macOS dark mode). @@ -431,61 +259,6 @@ def format_archive_name(profile, archive_name_tpl): SHELL_PATTERN_ELEMENT = re.compile(r'([?\[\]*])') -def get_mount_points(repo_url): - mount_points = {} - repo_mounts = [] - for proc in psutil.process_iter(): - try: - name = proc.name() - if name == 'borg' or name.startswith('python'): - if 'mount' not in proc.cmdline(): - continue - - if borg_compat.check('V2'): - # command line syntax: - # `borg mount -r (-a )` - cmd = proc.cmdline() - if repo_url in cmd: - i = cmd.index(repo_url) - if len(cmd) > i + 1: - mount_point = cmd[i + 1] - - # Archive mount? - ao = '-a' in cmd - if ao or '--match-archives' in cmd: - i = cmd.index('-a' if ao else '--match-archives') - if len(cmd) >= i + 1 and not SHELL_PATTERN_ELEMENT.search(cmd[i + 1]): - mount_points[mount_point] = cmd[i + 1] - else: - repo_mounts.append(mount_point) - else: - for idx, parameter in enumerate(proc.cmdline()): - if parameter.startswith(repo_url): - # mount from this repo - - # The borg mount command specifies that the mount_point - # parameter comes after the archive name - if len(proc.cmdline()) > idx + 1: - mount_point = proc.cmdline()[idx + 1] - - # archive or full mount? - if parameter[len(repo_url) :].startswith('::'): - archive_name = parameter[len(repo_url) + 2 :] - mount_points[archive_name] = mount_point - break - else: - # repo mount point - repo_mounts.append(mount_point) - - except (psutil.ZombieProcess, psutil.AccessDenied, psutil.NoSuchProcess): - # Getting process details may fail (e.g. zombie process on macOS) - # or because the process is owned by another user. - # Also see https://github.com/giampaolo/psutil/issues/783 - continue - - return mount_points, repo_mounts - - def is_system_tray_available(): app = QApplication.instance() if app is None: diff --git a/src/vorta/views/archive_tab.py b/src/vorta/views/archive_tab.py index d2af5757b..5b43c58a0 100644 --- a/src/vorta/views/archive_tab.py +++ b/src/vorta/views/archive_tab.py @@ -39,14 +39,13 @@ find_best_unit_for_sizes, format_archive_name, get_asset, - get_mount_points, pretty_bytes, ) from vorta.views import diff_result, extract_dialog from vorta.views.diff_result import DiffResultDialog, DiffTree from vorta.views.extract_dialog import ExtractDialog, ExtractTree -from vorta.views.source_tab import SizeItem -from vorta.views.utils import get_colored_icon +from vorta.views.utils import SizeItem, get_colored_icon +from vorta.views.workers.mount_points_worker import MountPointsWorker uifile = get_asset('UI/archivetab.ui') ArchiveTabUI, ArchiveTabBase = uic.loadUiType(uifile) @@ -73,6 +72,7 @@ def __init__(self, parent=None, app=None): super().__init__(parent) self.setupUi(parent) self.mount_points = {} # mapping of archive name to mount point + self.workers = [] self.repo_mount_point: Optional[str] = None # mount point of whole repo self.menu = None self.app = app @@ -237,28 +237,43 @@ def _toggle_all_buttons(self, enabled=True): # Restore states self.on_selection_change() + def set_mount_points(self, mount_points, repo_mounts): + if len(repo_mounts) == 0: + return + + archives = [s for s in self.profile().repo.archives.select().order_by(ArchiveModel.time.desc())] + + # if no archive's name can be found in self.mount_points, then hide the mount point column + if not any(a.name in mount_points for a in archives): + return + else: + self.archiveTable.showColumn(3) + self.repo_mount_point = repo_mounts[0] + + for row, archive in enumerate(archives): + mount_point = self.mount_points.get(archive.name) + if mount_point is not None: + item = QTableWidgetItem(mount_point) + self.archiveTable.setItem(row, 3, item) + def populate_from_profile(self): """Populate archive list and prune settings from profile.""" + self.archiveTable.blockSignals(True) profile = self.profile() if profile.repo is not None: - # get mount points - self.mount_points, repo_mount_points = get_mount_points(profile.repo.url) - if repo_mount_points: - self.repo_mount_point = repo_mount_points[0] - if profile.repo.name: repo_name = f"{profile.repo.name} ({profile.repo.url})" else: repo_name = profile.repo.url self.toolBox.setItemText(0, self.tr('Archives for {}').format(repo_name)) - archives = [s for s in profile.repo.archives.select().order_by(ArchiveModel.time.desc())] + populateArchiveTableWorker = MountPointsWorker(profile.repo.url) + self.workers.append(populateArchiveTableWorker) # preserve worker reference + populateArchiveTableWorker.signal.connect(self.set_mount_points) + populateArchiveTableWorker.start() + self.archiveTable.hideColumn(3) - # if no archive's name can be found in self.mount_points, then hide the mount point column - if not any(a.name in self.mount_points for a in archives): - self.archiveTable.hideColumn(3) - else: - self.archiveTable.showColumn(3) + archives = list(profile.repo.archives.select().order_by(ArchiveModel.time.desc())) sorting = self.archiveTable.isSortingEnabled() self.archiveTable.setSortingEnabled(False) @@ -280,12 +295,6 @@ def populate_from_profile(self): formatted_duration = '' self.archiveTable.setItem(row, 2, QTableWidgetItem(formatted_duration)) - - mount_point = self.mount_points.get(archive.name) - if mount_point is not None: - item = QTableWidgetItem(mount_point) - self.archiveTable.setItem(row, 3, item) - self.archiveTable.setItem(row, 4, QTableWidgetItem(archive.name)) if archive.trigger == 'scheduled': @@ -304,8 +313,9 @@ def populate_from_profile(self): self.archiveTable.scrollToItem(item) self.archiveTable.selectionModel().clearSelection() - if self.remaining_refresh_archives == 0: - self._toggle_all_buttons(enabled=True) + + if self.remaining_refresh_archives == 0: + self._toggle_all_buttons(enabled=True) else: self.mount_points = {} self.archiveTable.setRowCount(0) @@ -316,13 +326,14 @@ def populate_from_profile(self): self.prunePrefixTemplate.setText(profile.prune_prefix) # Populate pruning options from database - profile = self.profile() for i in self.prune_intervals: getattr(self, f'prune_{i}').setValue(getattr(profile, f'prune_{i}')) getattr(self, f'prune_{i}').valueChanged.connect(self.save_prune_setting) self.prune_keep_within.setText(profile.prune_keep_within) self.prune_keep_within.editingFinished.connect(self.save_prune_setting) + self.archiveTable.blockSignals(False) + def on_selection_change(self, selected=None, deselected=None): """ React to a change of the selection of the archiveTableView. diff --git a/src/vorta/views/main_window.py b/src/vorta/views/main_window.py index f42e04c44..c2cf6c7e9 100644 --- a/src/vorta/views/main_window.py +++ b/src/vorta/views/main_window.py @@ -260,8 +260,7 @@ def profile_imported_event(profile): self.tr('Profile {} imported.').format(profile.name), ) self.repoTab.populate_from_profile() - self.scheduleTab.populate_logs() - self.scheduleTab.populate_wifi() + self.scheduleTab.populate_from_profile() self.miscTab.populate() self.populate_profile_selector() diff --git a/src/vorta/views/repo_tab.py b/src/vorta/views/repo_tab.py index e9f38dbad..a74f620d5 100644 --- a/src/vorta/views/repo_tab.py +++ b/src/vorta/views/repo_tab.py @@ -173,8 +173,6 @@ def init_repo_stats(self): self.repoEncryption.setText(na) self.repoEncryption.setToolTip(no_repo_selected) - self.repo_changed.emit() - def init_ssh(self): keys = get_private_keys() self.sshComboBox.clear() @@ -264,6 +262,7 @@ def repo_select_action(self): profile.repo = self.repoSelector.currentData() profile.save() self.init_repo_stats() + self.repo_changed.emit() def process_new_repo(self, result): if result['returncode'] == 0: @@ -276,10 +275,12 @@ def process_new_repo(self, result): self.repoSelector.setCurrentIndex(self.repoSelector.count() - 1) self.repo_added.emit() self.init_repo_stats() + self.repo_changed.emit() def repo_unlink_action(self): profile = self.profile() self.init_repo_stats() + self.repo_changed.emit() msg = QMessageBox() msg.setStandardButtons(QMessageBox.StandardButton.Ok) diff --git a/src/vorta/views/schedule_tab.py b/src/vorta/views/schedule_tab.py index dc97cfa47..f5c3b60b5 100644 --- a/src/vorta/views/schedule_tab.py +++ b/src/vorta/views/schedule_tab.py @@ -12,8 +12,9 @@ from vorta.i18n import get_locale from vorta.scheduler import ScheduleStatusType from vorta.store.models import BackupProfileMixin, EventLogModel, WifiSettingModel -from vorta.utils import get_asset, get_sorted_wifis +from vorta.utils import get_asset from vorta.views.utils import get_colored_icon +from vorta.views.workers.wifi_list_worker import WifiListWorker uifile = get_asset('UI/scheduletab.ui') ScheduleUI, ScheduleBase = uic.loadUiType(uifile) @@ -33,6 +34,7 @@ def __init__(self, parent=None): self.setupUi(parent) self.app: application.VortaApp = QApplication.instance() self.toolBox.setCurrentIndex(0) + self.workers = [] self.schedulerRadioMapping = { 'off': self.scheduleOffRadio, @@ -171,10 +173,26 @@ def populate_from_profile(self): else: self.createCmdLineEdit.setEnabled(False) - self.populate_wifi() + wifiListWorker = WifiListWorker(profile.id) + self.workers.append(wifiListWorker) # preserve reference + wifiListWorker.signal.connect(self.set_wifi_list) + wifiListWorker.start() + self.populate_logs() self.draw_next_scheduled_backup() + def set_wifi_list(self, wifi_list): + self.wifiListWidget.clear() + for wifi in wifi_list: + item = QListWidgetItem() + item.setText(wifi.ssid) + item.setFlags(item.flags() | QtCore.Qt.ItemFlag.ItemIsUserCheckable) + if wifi.allowed: + item.setCheckState(QtCore.Qt.CheckState.Checked) + else: + item.setCheckState(QtCore.Qt.CheckState.Unchecked) + self.wifiListWidget.addItem(item) + def draw_next_scheduled_backup(self): status = self.app.scheduler.next_job_for_profile(self.profile().id) if status.type in ( @@ -191,19 +209,6 @@ def draw_next_scheduled_backup(self): self.nextBackupDateTimeLabel.setText(text) self.nextBackupDateTimeLabel.repaint() - def populate_wifi(self): - self.wifiListWidget.clear() - for wifi in get_sorted_wifis(self.profile()): - item = QListWidgetItem() - item.setText(wifi.ssid) - item.setFlags(item.flags() | QtCore.Qt.ItemFlag.ItemIsUserCheckable) - if wifi.allowed: - item.setCheckState(QtCore.Qt.CheckState.Checked) - else: - item.setCheckState(QtCore.Qt.CheckState.Unchecked) - self.wifiListWidget.addItem(item) - self.wifiListWidget.itemChanged.connect(self.save_wifi_item) - def save_wifi_item(self, item): db_item = WifiSettingModel.get(ssid=item.text(), profile=self.profile().id) db_item.allowed = item.checkState() == Qt.CheckState.Checked diff --git a/src/vorta/views/source_tab.py b/src/vorta/views/source_tab.py index ed63be2a7..83da6a404 100644 --- a/src/vorta/views/source_tab.py +++ b/src/vorta/views/source_tab.py @@ -14,15 +14,10 @@ ) from vorta.store.models import BackupProfileMixin, SettingsModel, SourceFileModel -from vorta.utils import ( - FilePathInfoAsync, - choose_file_dialog, - get_asset, - pretty_bytes, - sort_sizes, -) +from vorta.utils import choose_file_dialog, get_asset, pretty_bytes from vorta.views.exclude_dialog import ExcludeDialog -from vorta.views.utils import get_colored_icon +from vorta.views.utils import SizeItem, get_colored_icon +from vorta.views.workers.file_path_info_worker import FilePathInfoAsync uifile = get_asset('UI/sourcetab.ui') SourceUI, SourceBase = uic.loadUiType(uifile) @@ -36,23 +31,6 @@ class SourceColumn: FilesCount = 2 -class SizeItem(QTableWidgetItem): - def __init__(self, s): - super().__init__(s) - self.setTextAlignment(Qt.AlignmentFlag.AlignVCenter + Qt.AlignmentFlag.AlignRight) - - def __lt__(self, other): - if other.text() == '': - return False - elif self.text() == '': - return True - else: - return sort_sizes([self.text(), other.text()]) == [ - self.text(), - other.text(), - ] - - class FilesCount(QTableWidgetItem): def __lt__(self, other): # Verify that conversion is only performed on valid integers diff --git a/src/vorta/views/utils.py b/src/vorta/views/utils.py index 5a9697a73..a4be4f76a 100644 --- a/src/vorta/views/utils.py +++ b/src/vorta/views/utils.py @@ -2,11 +2,47 @@ import os import sys +from PyQt6.QtCore import Qt from PyQt6.QtGui import QIcon, QImage, QPixmap +from PyQt6.QtWidgets import QTableWidgetItem from vorta.utils import get_asset, uses_dark_mode +class SizeItem(QTableWidgetItem): + def __init__(self, s): + super().__init__(s) + self.setTextAlignment(Qt.AlignmentFlag.AlignVCenter + Qt.AlignmentFlag.AlignRight) + + def __lt__(self, other): + if other.text() == '': + return False + elif self.text() == '': + return True + else: + return sort_sizes([self.text(), other.text()]) == [ + self.text(), + other.text(), + ] + + +def sort_sizes(size_list): + """Sorts sizes with extensions. Assumes that size is already in largest unit possible""" + final_list = [] + for suffix in [" B", " KB", " MB", " GB", " TB", " PB", " EB", " ZB", " YB"]: + sub_list = [ + float(size[: -len(suffix)]) + for size in size_list + if size.endswith(suffix) and size[: -len(suffix)][-1].isnumeric() + ] + sub_list.sort() + final_list += [(str(size) + suffix) for size in sub_list] + # Skip additional loops + if len(final_list) == len(size_list): + break + return final_list + + def get_colored_icon(icon_name, scaled_height=128, return_qpixmap=False): """ Return SVG icon in the correct color. diff --git a/src/vorta/views/workers/__init__.py b/src/vorta/views/workers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/vorta/views/workers/file_path_info_worker.py b/src/vorta/views/workers/file_path_info_worker.py new file mode 100644 index 000000000..a4950252a --- /dev/null +++ b/src/vorta/views/workers/file_path_info_worker.py @@ -0,0 +1,117 @@ +import fnmatch +import os +import re +import sys +import unicodedata + +from PyQt6.QtCore import QFileInfo, QThread, pyqtSignal + + +class FilePathInfoAsync(QThread): + signal = pyqtSignal(str, str, str) + + def __init__(self, path, exclude_patterns_str): + self.path = path + QThread.__init__(self) + self.exiting = False + self.exclude_patterns = [] + for _line in (exclude_patterns_str or '').splitlines(): + line = _line.strip() + if line != '': + self.exclude_patterns.append(line) + + def run(self): + # logger.info("running thread to get path=%s...", self.path) + self.size, self.files_count = get_path_datasize(self.path, self.exclude_patterns) + self.signal.emit(self.path, str(self.size), str(self.files_count)) + + +def normalize_path(path): + """normalize paths for MacOS (but do nothing on other platforms)""" + # HFS+ converts paths to a canonical form, so users shouldn't be required to enter an exact match. + # Windows and Unix filesystems allow different forms, so users always have to enter an exact match. + return unicodedata.normalize('NFD', path) if sys.platform == 'darwin' else path + + +def get_path_datasize(path, exclude_patterns): + file_info = QFileInfo(path) + + if file_info.isDir(): + data_size, files_count = get_directory_size(file_info.absoluteFilePath(), exclude_patterns) + else: + data_size = file_info.size() + files_count = 1 + + return data_size, files_count + + +# prepare patterns as borg does +# see `FnmatchPattern._prepare` at +# https://github.com/borgbackup/borg/blob/master//src/borg/patterns.py +def prepare_pattern(pattern): + """Prepare and process fnmatch patterns as borg does""" + if pattern.endswith(os.path.sep): + # trailing sep indicates that the contents should be excluded + # but not the directory it self. + pattern = os.path.normpath(pattern).rstrip(os.path.sep) + pattern += os.path.sep + '*' + os.path.sep + else: + pattern = os.path.normpath(pattern) + os.path.sep + '*' + + pattern = pattern.lstrip(os.path.sep) # sep at beginning is removed + return re.compile(fnmatch.translate(pattern)) + + +def match(pattern: re.Pattern, path: str): + """Check whether a path matches the given pattern.""" + path = path.lstrip(os.path.sep) + os.path.sep + return pattern.match(path) is not None + + +def get_directory_size(dir_path, exclude_patterns): + '''Get number of files only and total size in bytes from a path. + Based off https://stackoverflow.com/a/17936789''' + exclude_patterns = [prepare_pattern(p) for p in exclude_patterns] + + data_size_filtered = 0 + seen = set() + seen_filtered = set() + + for dir_path, subdirectories, file_names in os.walk(dir_path, topdown=True): + is_excluded = False + for pattern in exclude_patterns: + if match(pattern, dir_path): + is_excluded = True + break + + if is_excluded: + subdirectories.clear() # so that os.walk won't walk them + continue + + for file_name in file_names: + file_path = os.path.join(dir_path, file_name) + + # Ignore symbolic links, since borg doesn't follow them + if os.path.islink(file_path): + continue + + is_excluded = False + for pattern in exclude_patterns: + if match(pattern, file_path): + is_excluded = True + break + + try: + stat = os.stat(file_path) + if stat.st_ino not in seen: # Visit each file only once + # this won't add the size of a hardlinked file + seen.add(stat.st_ino) + if not is_excluded: + data_size_filtered += stat.st_size + seen_filtered.add(stat.st_ino) + except (FileNotFoundError, PermissionError): + continue + + files_count_filtered = len(seen_filtered) + + return data_size_filtered, files_count_filtered diff --git a/src/vorta/views/workers/mount_points_worker.py b/src/vorta/views/workers/mount_points_worker.py new file mode 100644 index 000000000..2df4f6c92 --- /dev/null +++ b/src/vorta/views/workers/mount_points_worker.py @@ -0,0 +1,69 @@ +import psutil +from PyQt6.QtCore import QThread, pyqtSignal + +from vorta.utils import SHELL_PATTERN_ELEMENT, borg_compat + +SIZE_DECIMAL_DIGITS = 1 + + +class MountPointsWorker(QThread): + + signal = pyqtSignal(dict, list) + + def __init__(self, repo_url): + QThread.__init__(self) + self.repo_url = repo_url + + def run(self): + mount_points = {} + repo_mounts = [] + for proc in psutil.process_iter(): + try: + name = proc.name() + if name == 'borg' or name.startswith('python'): + if 'mount' not in proc.cmdline(): + continue + + if borg_compat.check('V2'): + # command line syntax: + # `borg mount -r (-a )` + cmd = proc.cmdline() + if self.repo_url in cmd: + i = cmd.index(self.repo_url) + if len(cmd) > i + 1: + mount_point = cmd[i + 1] + + # Archive mount? + ao = '-a' in cmd + if ao or '--match-archives' in cmd: + i = cmd.index('-a' if ao else '--match-archives') + if len(cmd) >= i + 1 and not SHELL_PATTERN_ELEMENT.search(cmd[i + 1]): + mount_points[mount_point] = cmd[i + 1] + else: + repo_mounts.append(mount_point) + else: + for idx, parameter in enumerate(proc.cmdline()): + if parameter.startswith(self.repo_url): + # mount from this repo + + # The borg mount command specifies that the mount_point + # parameter comes after the archive name + if len(proc.cmdline()) > idx + 1: + mount_point = proc.cmdline()[idx + 1] + + # archive or full mount? + if parameter[len(self.repo_url) :].startswith('::'): + archive_name = parameter[len(self.repo_url) + 2 :] + mount_points[archive_name] = mount_point + break + else: + # repo mount point + repo_mounts.append(mount_point) + + except (psutil.ZombieProcess, psutil.AccessDenied, psutil.NoSuchProcess): + # Getting process details may fail (e.g. zombie process on macOS) + # or because the process is owned by another user. + # Also see https://github.com/giampaolo/psutil/issues/783 + continue + + self.signal.emit(mount_points, repo_mounts) diff --git a/src/vorta/views/workers/wifi_list_worker.py b/src/vorta/views/workers/wifi_list_worker.py new file mode 100644 index 000000000..cfcb6b3bc --- /dev/null +++ b/src/vorta/views/workers/wifi_list_worker.py @@ -0,0 +1,45 @@ +import logging + +from PyQt6.QtCore import QThread, pyqtSignal + +from vorta.store.models import WifiSettingModel +from vorta.utils import get_network_status_monitor + +logger = logging.getLogger(__name__) + + +class WifiListWorker(QThread): + signal = pyqtSignal(list) + + def __init__(self, profile_id): + QThread.__init__(self) + self.profile_id = profile_id + + def run(self): + """ + Get Wifi networks known to the OS (only current one on macOS) and + merge with networks from other profiles. Update last connected time. + """ + + # Pull networks known to OS and all other backup profiles + system_wifis = get_network_status_monitor().get_known_wifis() + from_other_profiles = WifiSettingModel.select().where(WifiSettingModel.profile != self.profile_id).execute() + + for wifi in list(from_other_profiles) + system_wifis: + db_wifi, created = WifiSettingModel.get_or_create( + ssid=wifi.ssid, + profile=self.profile_id, + defaults={'last_connected': wifi.last_connected, 'allowed': True}, + ) + + # Update last connected time + if not created and db_wifi.last_connected != wifi.last_connected: + db_wifi.last_connected = wifi.last_connected + db_wifi.save() + + # Finally return list of networks and settings for that profile + self.signal.emit( + WifiSettingModel.select() + .where(WifiSettingModel.profile == self.profile_id) + .order_by(-WifiSettingModel.last_connected) + ) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index e622a2118..39f2af378 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -72,8 +72,12 @@ def init_db(qapp, qtbot, tmpdir_factory): qapp.jobs_manager.cancel_all_jobs() qapp.backup_finished_event.disconnect() - qapp.scheduler.schedule_changed.disconnect() + for worker in qapp.main_window.archiveTab.workers + qapp.main_window.scheduleTab.workers: + worker.quit() + worker.wait() + worker.exit() qtbot.waitUntil(lambda: not qapp.jobs_manager.is_worker_running(), **pytest._wait_defaults) + qapp.scheduler.schedule_changed.disconnect() mock_db.close() diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index ea529c089..3180a859e 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1,16 +1,13 @@ -import sys import uuid import pytest from vorta.keyring.abc import VortaKeyring from vorta.utils import ( find_best_unit_for_sizes, - get_path_datasize, is_system_tray_available, - normalize_path, pretty_bytes, - sort_sizes, ) +from vorta.views.utils import sort_sizes def test_keyring(): @@ -110,53 +107,6 @@ def test_pretty_bytes_nonfixed_units(size, metric, expected_output): assert output == expected_output -def test_normalize_path(): - """ - Test that path is normalized for macOS, but does nothing for other platforms. - """ - input_path = '/Users/username/caf\u00e9/file.txt' - expected_output = '/Users/username/café/file.txt' - - actual_output = normalize_path(input_path) - - if sys.platform == 'darwin': - assert actual_output == expected_output - else: - assert actual_output == input_path - - -def test_get_path_datasize(tmpdir): - """ - Test that get_path_datasize() works correctly when passed excluded patterns. - """ - # Create a temporary directory for testing - test_dir = tmpdir.mkdir("test_dir") - test_file = test_dir.join("test_file.txt") - test_file.write("Hello, World!") - - # Create a subdirectory with a file to exclude - excluded_dir = test_dir.mkdir("excluded_dir") - excluded_file = excluded_dir.join("excluded_file.txt") - excluded_file.write("Excluded file, should not be checked.") - - exclude_patterns = [f"{excluded_dir}"] - - # Test when the path is a directory - data_size, files_count = get_path_datasize(str(test_dir), exclude_patterns) - assert data_size == len("Hello, World!") - assert files_count == 1 - - # Test when the path is a file - data_size, files_count = get_path_datasize(str(test_file), exclude_patterns) - assert data_size == len("Hello, World!") - assert files_count == 1 - - # Test when the path is a directory with an excluded file - data_size, files_count = get_path_datasize(str(excluded_dir), exclude_patterns) - assert data_size == 0 - assert files_count == 0 - - def test_is_system_tray_available(mocker): """ Sanity check to ensure proper behavior diff --git a/tests/unit/test_workers.py b/tests/unit/test_workers.py new file mode 100644 index 000000000..121d5e26f --- /dev/null +++ b/tests/unit/test_workers.py @@ -0,0 +1,53 @@ +import sys + +from vorta.views.workers.file_path_info_worker import ( + get_path_datasize, + normalize_path, +) + + +def test_normalize_path(): + """ + Test that path is normalized for macOS, but does nothing for other platforms. + """ + input_path = '/Users/username/caf\u00e9/file.txt' + expected_output = '/Users/username/café/file.txt' + + actual_output = normalize_path(input_path) + + if sys.platform == 'darwin': + assert actual_output == expected_output + else: + assert actual_output == input_path + + +def test_get_path_datasize(tmpdir): + """ + Test that get_path_datasize() works correctly when passed excluded patterns. + """ + # Create a temporary directory for testing + test_dir = tmpdir.mkdir("test_dir") + test_file = test_dir.join("test_file.txt") + test_file.write("Hello, World!") + + # Create a subdirectory with a file to exclude + excluded_dir = test_dir.mkdir("excluded_dir") + excluded_file = excluded_dir.join("excluded_file.txt") + excluded_file.write("Excluded file, should not be checked.") + + exclude_patterns = [f"{excluded_dir}"] + + # Test when the path is a directory + data_size, files_count = get_path_datasize(str(test_dir), exclude_patterns) + assert data_size == len("Hello, World!") + assert files_count == 1 + + # Test when the path is a file + data_size, files_count = get_path_datasize(str(test_file), exclude_patterns) + assert data_size == len("Hello, World!") + assert files_count == 1 + + # Test when the path is a directory with an excluded file + data_size, files_count = get_path_datasize(str(excluded_dir), exclude_patterns) + assert data_size == 0 + assert files_count == 0