Pipes and other improvements

Makefile.am

@@ -1,7 +1,7 @@
 ACLOCAL_AMFLAGS = ${ACLOCAL_FLAGS}
 
 bin_PROGRAMS = ag
-ag_SOURCES = src/ignore.c src/log.c src/options.c src/print.c src/util.c src/main.c
+ag_SOURCES = src/ignore.c src/log.c src/options.c src/print.c src/search.c src/util.c src/main.c
 ag_LDADD = ${PCRE_LIBS}
 
 man_MANS = doc/ag.1

README.md

@@ -58,7 +58,6 @@ You can use this with [my fork](https://github.com/ggreer/AckMate) of the popula
 ### TODO ###
 * behave better when matching in files with really long lines
   * maybe say "match found at position X of line N" if line is > 10k chars
-* better support for inverted matching (it's stupidly slow right now)
 * support piping in stuff
 * allow searching files > 1GB in size
 * optimizations

src/main.c

@@ -1,250 +1,16 @@
-#include <dirent.h>
-#include <errno.h>
-#include <limits.h>
-#include <fcntl.h>
 #include <pcre.h>
 #include <stdarg.h>
 #include <stdio.h>
-#include <stdlib.h>
 #include <string.h>
-#include <sys/dir.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
 #include <sys/time.h>
-#include <sys/types.h>
-#include <unistd.h>
 
-#include "ignore.h"
 #include "log.h"
 #include "options.h"
-#include "print.h"
-#include "util.h"
+#include "search.h"
 
 #include "config.h"
 /* #define AG_DEBUG */
 
-/* TODO: make these configurable */
-const int MAX_SEARCH_DEPTH = 25;
-const int MAX_MATCHES_PER_FILE = 10000;
-
-ag_stats stats;
-
-size_t skip_lookup[256];
-
-/* TODO: append matches to some data structure instead of just printing them out
- * then there can be sweet summaries of matches/files scanned/time/etc
- */
-int search_dir(const pcre *re, const pcre_extra *re_extra, const char* path, const int depth) {
-    /* TODO: don't just die. also make max depth configurable */
-    if (depth > MAX_SEARCH_DEPTH) {
-        log_err("Search depth greater than %i, giving up.", depth);
-        exit(1);
-    }
-    struct dirent **dir_list = NULL;
-    struct dirent *dir = NULL;
-    int results = 0;
-
-    int fd = -1;
-    off_t f_len = 0;
-    char *buf = NULL;
-    int rv = 0;
-    char *dir_full_path = NULL;
-    size_t path_length = 0;
-    int i;
-
-    results = scandir(path, &dir_list, &ignorefile_filter, &alphasort);
-    if (results > 0) {
-        for (i = 0; i < results; i++) {
-            dir = dir_list[i];
-            path_length = (size_t)(strlen(path) + strlen(dir->d_name) + 2); /* 2 for slash and null char */
-            dir_full_path = malloc(path_length);
-            strlcpy(dir_full_path, path, path_length);
-            strlcat(dir_full_path, "/", path_length);
-            strlcat(dir_full_path, dir->d_name, path_length);
-            load_ignore_patterns(dir_full_path);
-            free(dir);
-            dir = NULL;
-            free(dir_full_path);
-            dir_full_path = NULL;
-        }
-    }
-    free(dir_list);
-    dir_list = NULL;
-
-    results = scandir(path, &dir_list, &filename_filter, &alphasort);
-    if (results == 0)
-    {
-        log_debug("No results found in directory %s", path);
-        free(dir_list);
-        dir_list = NULL;
-        return(0);
-    }
-    else if (results == -1) {
-        log_err("Error opening directory %s", path);
-        return(0);
-    }
-
-    match matches[MAX_MATCHES_PER_FILE];
-    int matches_len = 0;
-    int buf_len = 0;
-    int buf_offset = 0;
-    int offset_vector[MAX_MATCHES_PER_FILE * 3]; /* TODO */
-    int rc = 0;
-    struct stat statbuf;
-    int binary = 0;
-
-    for (i=0; i<results; i++) {
-        matches_len = 0;
-        buf_offset = 0;
-        binary = 0;
-        dir = dir_list[i];
-        /* TODO: this is copy-pasted from about 30 lines above */
-        path_length = (size_t)(strlen(path) + strlen(dir->d_name) + 2); /* 2 for slash and null char */
-        dir_full_path = malloc(path_length);
-        strlcpy(dir_full_path, path, path_length);
-        strlcat(dir_full_path, "/", path_length);
-        strlcat(dir_full_path, dir->d_name, path_length);
-
-        log_debug("dir %s type %i", dir_full_path, dir->d_type);
-        /* TODO: scan files in current dir before going deeper */
-        if (dir->d_type == DT_DIR) {
-            if (opts.recurse_dirs) {
-                log_debug("Searching dir %s", dir_full_path);
-                rv = search_dir(re, re_extra, dir_full_path, depth + 1);
-            }
-            goto cleanup;
-        }
-
-        if (opts.file_search_regex) {
-            rc = pcre_exec(opts.file_search_regex, NULL, dir_full_path, strlen(dir_full_path),
-                           buf_offset, 0, offset_vector, 3);
-            if (rc < 0) { /* no match */
-                log_debug("Skipping %s due to file_search_regex.", dir_full_path);
-                goto cleanup;
-            }
-        }
-
-        fd = open(dir_full_path, O_RDONLY);
-        if (fd < 0) {
-            log_err("Error opening file %s. Skipping...", dir_full_path);
-            goto cleanup;
-        }
-
-        rv = fstat(fd, &statbuf);
-        if (rv != 0) {
-            log_err("Error fstat()ing file %s. Skipping...", dir_full_path);
-            goto cleanup;
-        }
-
-        f_len = statbuf.st_size;
-
-        if (f_len == 0) {
-            log_debug("File %s is empty, skipping.", dir_full_path);
-            goto cleanup;
-        }
-
-        buf = mmap(0, f_len, PROT_READ, MAP_SHARED, fd, 0);
-        if (buf == MAP_FAILED) {
-            log_err("File %s failed to load: %s.", dir_full_path, strerror(errno));
-            goto cleanup;
-        }
-
-        buf_len = f_len;
-
-        if (is_binary((void*)buf, buf_len)) { /* Who needs duck typing when you have void cast? :) */
-            if (opts.search_binary_files) {
-                binary = 1;
-            }
-            else {
-                log_debug("File %s is binary. Skipping...", dir_full_path);
-                goto cleanup;
-            }
-        }
-
-        if (opts.literal) {
-            char *match_ptr = buf;
-            char *(*ag_strncmp_fp)(const char*, const char*, size_t, size_t, size_t[]) = &boyer_moore_strnstr;
-
-            if (opts.casing == CASE_INSENSITIVE) {
-                ag_strncmp_fp = &boyer_moore_strncasestr;
-            }
-            while (buf_offset < buf_len) {
-                match_ptr = ag_strncmp_fp(match_ptr, opts.query, buf_len - buf_offset, opts.query_len, skip_lookup);
-                if (match_ptr == NULL) {
-                    break;
-                }
-                matches[matches_len].start = match_ptr - buf;
-                matches[matches_len].end = matches[matches_len].start + opts.query_len;
-                buf_offset = matches[matches_len].end;
-                matches_len++;
-                match_ptr++;
-                /* Don't segfault. TODO: realloc this array */
-                if (matches_len >= MAX_MATCHES_PER_FILE) {
-                    log_err("Too many matches in %s. Skipping the rest of this file.", dir_full_path);
-                    break;
-                }
-            }
-        }
-        else {
-            /* In my profiling, most of the execution time is spent in this pcre_exec */
-            while (buf_offset < buf_len &&
-                 (rc = pcre_exec(re, re_extra, buf, buf_len, buf_offset, 0, offset_vector, 3)) >= 0) {
-                log_debug("Match found. File %s, offset %i bytes.", dir_full_path, offset_vector[0]);
-                buf_offset = offset_vector[1];
-                matches[matches_len].start = offset_vector[0];
-                matches[matches_len].end = offset_vector[1];
-                matches_len++;
-                /* Don't segfault. TODO: realloc this array */
-                if (matches_len >= MAX_MATCHES_PER_FILE) {
-                    log_err("Too many matches in %s. Skipping the rest of this file.", dir_full_path);
-                    break;
-                }
-            }
-        }
-
-        if (opts.stats) {
-            stats.total_bytes += buf_len;
-            stats.total_files++;
-            stats.total_matches += matches_len;
-        }
-
-        if (rc == -1) {
-            log_debug("No match in %s", dir_full_path);
-        }
-
-        if (matches_len > 0) {
-            if (opts.print_filename_only) {
-                print_path(dir_full_path);
-                putchar('\n');
-            }
-            else {
-                if (binary) {
-                    print_binary_file_matches(dir_full_path);
-                }
-                else {
-                    print_file_matches(dir_full_path, buf, buf_len, matches, matches_len);
-                }
-            }
-        }
-
-        cleanup:
-        if (fd != -1) {
-            munmap(buf, f_len);
-            close(fd);
-            fd = -1;
-        }
-
-        free(dir);
-        dir = NULL;
-        free(dir_full_path);
-        dir_full_path = NULL;
-    }
-
-    free(dir_list);
-    dir_list = NULL;
-    return(0);
-}
-
 int main(int argc, char **argv) {
     set_log_level(LOG_LEVEL_WARN);
 
@@ -303,7 +69,12 @@ int main(int argc, char **argv) {
         }
     }
 
-    search_dir(re, re_extra, path, 0);
+    if (opts.search_stdin) {
+        search_stdin(re, re_extra);
+    }
+    else {
+        search_dir(re, re_extra, path, 0);
+    }
 
     if (opts.stats) {
         gettimeofday(&(stats.time_end), NULL);

src/options.c

@@ -126,6 +126,7 @@ void parse_options(int argc, char **argv, char **query, char **path) {
 
     if (!isatty(fileno(stdin))) {
         opts.search_stdin = 1;
+        group = 0;
     }
 
     /* If we're not outputting to a terminal. change output to:
@@ -258,20 +259,8 @@ void parse_options(int argc, char **argv, char **query, char **path) {
         opts.print_break = 0;
     }
 
-    /* I can't figure out how to tell pcre_exec() to invert matches,
-       so build an inverse regex. Yes I know this sucks.
-     */
-    if (opts.invert_match) {
-        opts.query_len = strlen(argv[0]) + 11;
-        opts.query = malloc(opts.query_len);
-        strlcat(opts.query, "^((?!", opts.query_len);
-        strlcat(opts.query, argv[0], opts.query_len);
-        strlcat(opts.query, ").)*$", opts.query_len);
-    }
-    else {
-        opts.query = strdup(argv[0]);
-        opts.query_len = strlen(opts.query);
-    }
+    opts.query = strdup(argv[0]);
+    opts.query_len = strlen(opts.query);
 
     *query = opts.query;
 

src/print.c

@@ -36,6 +36,7 @@ void print_binary_file_matches(const char* path) {
 
 /* TODO: make print_matching_line() */
 
+/* TODO: doesn't work for matches across lines */
 void print_file_matches(const char* path, const char* buf, const int buf_len, const match matches[], const int matches_len) {
     int line = 1;
     int column = 0;
@@ -76,10 +77,19 @@ void print_file_matches(const char* path, const char* buf, const int buf_len, co
             }
 
             if (lines_since_last_match > 0) {
-                /* TODO: this is buggy as hell */
-                if (opts.before > 0 && lines_since_last_match > opts.after + opts.before) {
-                    /* We found the start of a match. print the previous line(s) */
-                    for (j = 0; j < opts.before; j++) {
+                if (opts.before > 0) {
+                    /* TODO: this is buggy as hell */
+                    /* TODO: yeah, it totally is */
+                    /* print the previous line(s) */
+                    int lines_to_print = lines_since_last_match - (opts.before + opts.after + 1);
+                    if (lines_to_print < 0) {
+                        lines_to_print = opts.before;
+                    }
+                    if (lines_to_print > opts.before) {
+                        lines_to_print = 0;
+                    }
+
+                    for (j = lines_to_print; j < opts.before; j++) {
                         prev_line = (last_prev_line + j) % opts.before;
                         if (context_prev_lines[prev_line] != NULL) {
                             if (opts.print_heading == 0) {