Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Fix for Issue #4: exclude directories during os.walk() #27

Merged
merged 3 commits into from

2 participants

@mdomsch
Owner

I borrowed liberally from the code that was doing exclusions, to apply those same exclusions during os.walk() time. This works for me.

@mludvig mludvig merged commit 9ea2780 into from
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
This page is out of date. Refresh to see the latest.
Showing with 65 additions and 4 deletions.
  1. +65 −4 S3/FileLists.py
View
69 S3/FileLists.py
@@ -14,6 +14,7 @@
import os
import glob
+import copy
__all__ = ["fetch_local_list", "fetch_remote_list", "compare_filelists", "filter_exclude_include"]
@@ -26,13 +27,15 @@ def _fswalk_follow_symlinks(path):
assert os.path.isdir(path) # only designed for directory argument
walkdirs = [path]
for dirpath, dirnames, filenames in os.walk(path):
+ handle_exclude_include_walk(dirpath, dirnames, [])
for dirname in dirnames:
current = os.path.join(dirpath, dirname)
if os.path.islink(current):
walkdirs.append(current)
for walkdir in walkdirs:
- for value in os.walk(walkdir):
- yield value
+ for dirpath, dirnames, filenames in os.walk(walkdir):
+ handle_exclude_include_walk(dirpath, dirnames, [])
+ yield (dirpath, dirnames, filenames)
def _fswalk(path, follow_symlinks):
'''
@@ -43,8 +46,10 @@ def _fswalk(path, follow_symlinks):
follow_symlinks (bool) indicates whether to descend into symbolically linked directories
'''
if follow_symlinks:
- return _fswalk_follow_symlinks(path)
- return os.walk(path)
+ yield _fswalk_follow_symlinks(path)
+ for dirpath, dirnames, filenames in os.walk(path):
+ handle_exclude_include_walk(dirpath, dirnames, filenames)
+ yield (dirpath, dirnames, filenames)
def filter_exclude_include(src_list):
info(u"Applying --exclude/--include")
@@ -75,6 +80,62 @@ def filter_exclude_include(src_list):
debug(u"PASS: %s" % (file))
return src_list, exclude_list
+def handle_exclude_include_walk(root, dirs, files):
+ cfg = Config()
+ copydirs = copy.copy(dirs)
+ copyfiles = copy.copy(files)
+
+ # exclude dir matches in the current directory
+ # this prevents us from recursing down trees we know we want to ignore
+ for x in copydirs:
+ d = os.path.join(root, x, '')
+ debug(u"CHECK: %s" % d)
+ excluded = False
+ for r in cfg.exclude:
+ if r.search(d):
+ excluded = True
+ debug(u"EXCL-MATCH: '%s'" % (cfg.debug_exclude[r]))
+ break
+ if excluded:
+ ## No need to check for --include if not excluded
+ for r in cfg.include:
+ if r.search(d):
+ excluded = False
+ debug(u"INCL-MATCH: '%s'" % (cfg.debug_include[r]))
+ break
+ if excluded:
+ ## Still excluded - ok, action it
+ debug(u"EXCLUDE: %s" % d)
+ dirs.remove(x)
+ continue
+ else:
+ debug(u"PASS: %s" % (d))
+
+ # exclude file matches in the current directory
+ for x in copyfiles:
+ file = os.path.join(root, x)
+ debug(u"CHECK: %s" % file)
+ excluded = False
+ for r in cfg.exclude:
+ if r.search(file):
+ excluded = True
+ debug(u"EXCL-MATCH: '%s'" % (cfg.debug_exclude[r]))
+ break
+ if excluded:
+ ## No need to check for --include if not excluded
+ for r in cfg.include:
+ if r.search(file):
+ excluded = False
+ debug(u"INCL-MATCH: '%s'" % (cfg.debug_include[r]))
+ break
+ if excluded:
+ ## Still excluded - ok, action it
+ debug(u"EXCLUDE: %s" % file)
+ files.remove(x)
+ continue
+ else:
+ debug(u"PASS: %s" % (file))
+
def fetch_local_list(args, recursive = None):
def _get_filelist_local(local_uri):
info(u"Compiling list of local files...")
Something went wrong with that request. Please try again.