Skip to content

Commit

Permalink
fix: initial tree entries parsing and entries skipping
Browse files Browse the repository at this point in the history
  • Loading branch information
F33RNI committed Jan 27, 2024
1 parent d266e7f commit 4b32a02
Show file tree
Hide file tree
Showing 2 changed files with 37 additions and 28 deletions.
52 changes: 30 additions & 22 deletions Backupper.py
Original file line number Diff line number Diff line change
Expand Up @@ -527,35 +527,38 @@ def _generate_tree(

# Add each entry
for path_abs, skip in entries.items():
if os.path.isdir(path_abs):
# Extract root directory
root_dir = os.path.dirname(path_abs) if root_relative_to_dirname else path_abs
# Ignore path
if ignore_filepaths_abs and path_abs in ignore_filepaths_abs:
continue

# Skipped entry
if skip:
skipped_entries_abs.append(path_abs)
continue

# Extract root directory
root_dir = os.path.dirname(path_abs) if root_relative_to_dirname else path_abs

# Get path type
path_type = (
PATH_IS_FILE
if os.path.isfile(path_abs)
else (PATH_IS_DIR if os.path.isdir(path_abs) else PATH_UNKNOWN)
)

# Add to the output queue
# (relative path, root dir path, PATH_..., is empty directory)
parsed_queue.put((os.path.relpath(path_abs, root_dir), root_dir, path_type, False))

# Parse directories
if path_type == PATH_IS_DIR:
# Extract parent directory
parent_dir = os.path.relpath(path_abs, root_dir) if root_relative_to_dirname else ""

# Add to the recursion queue (parent dir, abs root path)
if not skip:
directories_to_parse_queue.put((parent_dir, root_dir))

# Add to the output queue / list of skipped entries
if not ignore_filepaths_abs or path_abs not in ignore_filepaths_abs:
# Skipped entry
if skip:
skipped_entries_abs.append(path_abs)

# Non-skipped entry
else:
# Get path type
path_type = (
PATH_IS_FILE
if os.path.isfile(path_abs)
else (PATH_IS_DIR if os.path.isdir(path_abs) else PATH_UNKNOWN)
)

# (relative path, root dir path, PATH_..., is empty directory)
parsed_queue.put((os.path.relpath(path_abs, root_dir), root_dir, path_type, False))

# Create control Value for pause and cancel
control_value = multiprocessing.Value("i", PROCESS_CONTROL_WORK)

Expand Down Expand Up @@ -892,6 +895,11 @@ def _output_tree_queue_filler(
# Done
logging.info("Filler thread finished")

# Parse skipped entries
skipped_entries_abs_parts = [
os.path.normpath(skipped_entry_abs).split(os.path.sep) for skipped_entry_abs in skipped_entries_abs
]

# Create control Value for pause and cancel
control_value = multiprocessing.Value("i", PROCESS_CONTROL_WORK)

Expand Down Expand Up @@ -924,7 +932,7 @@ def _output_tree_queue_filler(
args=(
output_tree_queue,
input_tree,
skipped_entries_abs,
skipped_entries_abs_parts,
self._config_manager.get_config("delete_skipped"),
self._stats_deleted_ok_value,
self._stats_deleted_error_value,
Expand Down
13 changes: 7 additions & 6 deletions delete_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,12 @@ def is_path_relative_to(parent_path: List[str], child_path: List[str]) -> bool:
"""
child_part_index = 0
parent_part_last_match_index = -1
child_path_len = len(child_path)
for i, parent_path_part in enumerate(parent_path):
if parent_path_part == child_path[child_part_index]:
child_part_index += 1
parent_part_last_match_index = i
if child_part_index >= len(child_path):
if child_part_index >= child_path_len:
break
else:
child_part_index = 0
Expand All @@ -59,7 +60,7 @@ def is_path_relative_to(parent_path: List[str], child_path: List[str]) -> bool:
def delete_files(
output_tree_queue: multiprocessing.Queue,
input_tree: Dict,
skipped_entries_abs: List[str],
skipped_entries_abs_parts: List[List[str]],
delete_skipped: bool,
stats_deleted_ok_value: multiprocessing.Value,
stats_deleted_error_value: multiprocessing.Value,
Expand All @@ -71,7 +72,8 @@ def delete_files(
Args:
output_files_tree_queue (multiprocessing.Queue): output tree as Queue (tree_type, filepath_rel, root_empty)
input_tree (Dict): tree of all input files and directories
skipped_entries_abs (List[str]): list of normalized absolute paths to skip or delete (if delete_skipped)
skipped_entries_abs_parts (List[List[str]]): list of normalized absolute paths to skip or delete
(if delete_skipped) splitted by os.path.sep
delete_skipped (bool): True to also delete skipped files
stats_deleted_ok_value (multiprocessing.Value): counter of total successful delete calls
stats_deleted_error_value (multiprocessing.Value): counter of total unsuccessful delete calls
Expand Down Expand Up @@ -156,9 +158,8 @@ def delete_files(
# Try to find this path inside skipped entries
in_skipped = False
filepath_rel_parts = os.path.normpath(filepath_rel).split(os.path.sep)
for skipped_path_abs in skipped_entries_abs:
skipped_path_abs_parts = os.path.normpath(skipped_path_abs).split(os.path.sep)
if is_path_relative_to(skipped_path_abs_parts, filepath_rel_parts):
for skipped_entry_abs_parts in skipped_entries_abs_parts:
if is_path_relative_to(skipped_entry_abs_parts, filepath_rel_parts):
in_skipped = True
break

Expand Down

0 comments on commit 4b32a02

Please sign in to comment.