Skip to content
This repository

Fix for Issue #4: exclude directories during os.walk() #27

Merged
merged 3 commits into from about 1 year ago

2 participants

Matt Domsch Michal Ludvig
Matt Domsch
Collaborator

I borrowed liberally from the code that was doing exclusions, to apply those same exclusions during os.walk() time. This works for me.

Michal Ludvig mludvig merged commit 9ea2780 into from February 18, 2013
Michal Ludvig mludvig closed this February 18, 2013
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
This page is out of date. Refresh to see the latest.

Showing 1 changed file with 65 additions and 4 deletions. Show diff stats Hide diff stats

  1. 69  S3/FileLists.py
69  S3/FileLists.py
@@ -14,6 +14,7 @@
14 14
 
15 15
 import os
16 16
 import glob
  17
+import copy
17 18
 
18 19
 __all__ = ["fetch_local_list", "fetch_remote_list", "compare_filelists", "filter_exclude_include"]
19 20
 
@@ -26,13 +27,15 @@ def _fswalk_follow_symlinks(path):
26 27
         assert os.path.isdir(path) # only designed for directory argument
27 28
         walkdirs = [path]
28 29
         for dirpath, dirnames, filenames in os.walk(path):
  30
+ 		handle_exclude_include_walk(dirpath, dirnames, [])
29 31
                 for dirname in dirnames:
30 32
                         current = os.path.join(dirpath, dirname)
31 33
                         if os.path.islink(current):
32 34
 				walkdirs.append(current)
33 35
         for walkdir in walkdirs:
34  
-                for value in os.walk(walkdir):
35  
-                        yield value
  36
+		for dirpath, dirnames, filenames in os.walk(walkdir):
  37
+			handle_exclude_include_walk(dirpath, dirnames, [])
  38
+                        yield (dirpath, dirnames, filenames)
36 39
 
37 40
 def _fswalk(path, follow_symlinks):
38 41
         '''
@@ -43,8 +46,10 @@ def _fswalk(path, follow_symlinks):
43 46
         follow_symlinks (bool) indicates whether to descend into symbolically linked directories
44 47
         '''
45 48
         if follow_symlinks:
46  
-                return _fswalk_follow_symlinks(path)
47  
-        return os.walk(path)
  49
+                yield _fswalk_follow_symlinks(path)
  50
+	for dirpath, dirnames, filenames in os.walk(path):
  51
+		handle_exclude_include_walk(dirpath, dirnames, filenames)
  52
+		yield (dirpath, dirnames, filenames)
48 53
 
49 54
 def filter_exclude_include(src_list):
50 55
     info(u"Applying --exclude/--include")
@@ -75,6 +80,62 @@ def filter_exclude_include(src_list):
75 80
             debug(u"PASS: %s" % (file))
76 81
     return src_list, exclude_list
77 82
 
  83
+def handle_exclude_include_walk(root, dirs, files):
  84
+    cfg = Config()
  85
+    copydirs = copy.copy(dirs)
  86
+    copyfiles = copy.copy(files)
  87
+
  88
+    # exclude dir matches in the current directory
  89
+    # this prevents us from recursing down trees we know we want to ignore
  90
+    for x in copydirs:
  91
+        d = os.path.join(root, x, '')
  92
+        debug(u"CHECK: %s" % d)
  93
+        excluded = False
  94
+        for r in cfg.exclude:
  95
+            if r.search(d):
  96
+                excluded = True
  97
+                debug(u"EXCL-MATCH: '%s'" % (cfg.debug_exclude[r]))
  98
+                break
  99
+        if excluded:
  100
+            ## No need to check for --include if not excluded
  101
+            for r in cfg.include:
  102
+                if r.search(d):
  103
+                    excluded = False
  104
+                    debug(u"INCL-MATCH: '%s'" % (cfg.debug_include[r]))
  105
+                    break
  106
+        if excluded:
  107
+            ## Still excluded - ok, action it
  108
+            debug(u"EXCLUDE: %s" % d)
  109
+	    dirs.remove(x)
  110
+            continue
  111
+        else:
  112
+            debug(u"PASS: %s" % (d))
  113
+
  114
+    # exclude file matches in the current directory
  115
+    for x in copyfiles:
  116
+        file = os.path.join(root, x)
  117
+        debug(u"CHECK: %s" % file)
  118
+        excluded = False
  119
+        for r in cfg.exclude:
  120
+            if r.search(file):
  121
+                excluded = True
  122
+                debug(u"EXCL-MATCH: '%s'" % (cfg.debug_exclude[r]))
  123
+                break
  124
+        if excluded:
  125
+            ## No need to check for --include if not excluded
  126
+            for r in cfg.include:
  127
+                if r.search(file):
  128
+                    excluded = False
  129
+                    debug(u"INCL-MATCH: '%s'" % (cfg.debug_include[r]))
  130
+                    break
  131
+        if excluded:
  132
+            ## Still excluded - ok, action it
  133
+            debug(u"EXCLUDE: %s" % file)
  134
+            files.remove(x)
  135
+            continue
  136
+        else:
  137
+            debug(u"PASS: %s" % (file))
  138
+
78 139
 def fetch_local_list(args, recursive = None):
79 140
     def _get_filelist_local(local_uri):
80 141
         info(u"Compiling list of local files...")
Commit_comment_tip

Tip: You can add notes to lines in a file. Hover to the left of a line to make a note

Something went wrong with that request. Please try again.