Merge pull request #2185 from DanCech/standard-finder

improve efficiency of standard finder

webapp/graphite/finders/standard.py

@@ -53,8 +53,7 @@ def find_nodes(self, query):
 
                 relative_path = absolute_path[len(root_dir):].lstrip('/')
                 metric_path = fs_to_metric(relative_path)
-                real_metric_path = get_real_metric_path(
-                    absolute_path, metric_path)
+                real_metric_path = get_real_metric_path(absolute_path, metric_path)
 
                 metric_path_parts = metric_path.split('.')
                 for field_index in find_escaped_pattern_fields(query.pattern):
@@ -70,29 +69,24 @@ def find_nodes(self, query):
                 if isdir(absolute_path):
                     yield BranchNode(metric_path)
 
-                elif isfile(absolute_path):
-                    if absolute_path.endswith(
-                            '.wsp') and WhisperReader.supported:
-                        reader = WhisperReader(absolute_path, real_metric_path)
-                        yield LeafNode(metric_path, reader)
-
-                    elif absolute_path.endswith('.wsp.gz') and GzippedWhisperReader.supported:
-                        reader = GzippedWhisperReader(
-                            absolute_path, real_metric_path)
-                        yield LeafNode(metric_path, reader)
-
-                    elif absolute_path.endswith('.rrd') and RRDReader.supported:
-                        if datasource_pattern is None:
-                            yield BranchNode(metric_path)
-
-                        else:
-                            for datasource_name in RRDReader.get_datasources(
-                                    absolute_path):
-                                if match_entries(
-                                        [datasource_name], datasource_pattern):
-                                    reader = RRDReader(
-                                        absolute_path, datasource_name)
-                                    yield LeafNode(metric_path + "." + datasource_name, reader)
+                elif absolute_path.endswith('.wsp') and WhisperReader.supported:
+                    reader = WhisperReader(absolute_path, real_metric_path)
+                    yield LeafNode(metric_path, reader)
+
+                elif absolute_path.endswith('.wsp.gz') and GzippedWhisperReader.supported:
+                    reader = GzippedWhisperReader(
+                        absolute_path, real_metric_path)
+                    yield LeafNode(metric_path, reader)
+
+                elif absolute_path.endswith('.rrd') and RRDReader.supported:
+                    if datasource_pattern is None:
+                        yield BranchNode(metric_path)
+
+                    else:
+                        for datasource_name in RRDReader.get_datasources(absolute_path):
+                            if match_entries([datasource_name], datasource_pattern):
+                                reader = RRDReader(absolute_path, datasource_name)
+                                yield LeafNode(metric_path + "." + datasource_name, reader)
 
     def _find_paths(self, current_dir, patterns):
         """Recursively generates absolute paths whose components underneath current_dir
@@ -102,42 +96,40 @@ def _find_paths(self, current_dir, patterns):
 
         has_wildcard = pattern.find(
             '{') > -1 or pattern.find('[') > -1 or pattern.find('*') > -1 or pattern.find('?') > -1
-        using_globstar = pattern == "**"
 
+        matching_subdirs = []
+        files = []
         if has_wildcard:  # this avoids os.listdir() for performance
+            subdirs = []
             try:
-                entries = [x.name for x in scandir(current_dir)]
+                for x in scandir(current_dir):
+                    if x.is_file():
+                        files.append(x.name)
+                    if x.is_dir():
+                        subdirs.append(x.name)
             except OSError as e:
                 log.exception(e)
-                entries = []
-        else:
-            entries = [pattern]
-
-        if using_globstar:
-            matching_subdirs = map(operator.itemgetter(0), walk(current_dir))
-        else:
-            subdirs = [
-                entry for entry in entries if isdir(
-                    join(
-                        current_dir,
-                        entry))]
-            matching_subdirs = match_entries(subdirs, pattern)
-
-        # if this is a terminal globstar, add a pattern for all files in
-        # subdirs
-        if using_globstar and not patterns:
-            patterns = ["*"]
+
+            if pattern == "**":
+                matching_subdirs = map(operator.itemgetter(0), walk(current_dir))
+
+                # if this is a terminal globstar, add a pattern for all files in subdirs
+                if not patterns:
+                    patterns = ["*"]
+            else:
+                matching_subdirs = match_entries(subdirs, pattern)
+        elif isdir(join(current_dir, pattern)):
+            matching_subdirs.append(pattern)
 
         # the last pattern may apply to RRD data sources
         if len(patterns) == 1 and RRDReader.supported:
             if not has_wildcard:
-                entries = [pattern + ".rrd"]
-            files = [
-                entry for entry in entries if isfile(
-                    join(
-                        current_dir,
-                        entry))]
-            rrd_files = match_entries(files, pattern + ".rrd")
+                entries = [
+                  pattern + ".rrd",
+                ]
+                rrd_files = [entry for entry in entries if isfile(join(current_dir, entry))]
+            else:
+                rrd_files = match_entries(files, pattern + ".rrd")
 
             if rrd_files:  # let's assume it does
                 datasource_pattern = patterns[0]
@@ -148,7 +140,6 @@ def _find_paths(self, current_dir, patterns):
 
         if patterns:  # we've still got more directories to traverse
             for subdir in matching_subdirs:
-
                 absolute_path = join(current_dir, subdir)
                 for match in self._find_paths(absolute_path, patterns):
                     yield match
@@ -158,13 +149,11 @@ def _find_paths(self, current_dir, patterns):
                 entries = [
                     pattern + '.wsp',
                     pattern + '.wsp.gz',
-                    pattern + '.rrd']
-            files = [
-                entry for entry in entries if isfile(
-                    join(
-                        current_dir,
-                        entry))]
-            matching_files = match_entries(files, pattern + '.*')
+                    pattern + '.rrd',
+                ]
+                matching_files = [entry for entry in entries if isfile(join(current_dir, entry))]
+            else:
+                matching_files = match_entries(files, pattern + '.*')
 
             for base_name in matching_files + matching_subdirs:
                 yield join(current_dir, base_name)