feat: Implement support for /**/ matching in file paths, and report I…

…O errors on first scan

Fixes #327
Fixes #285

docs/log-courier/Configuration.md

@@ -77,7 +77,7 @@ error.
 
 ## Location
 
-The service configuration for Log Courier in the packages versions (using the service wrappers from the [`contrib`](contrib) folder) will load the configuration from `/etc/log-courier/log-courier.yaml`. When running Log Courier manually a configuration file can be specified using the [`-config`](CommandLineArguments.md#-configpath) argument.
+The service configuration for Log Courier in the packaged versions (using the service wrappers from the [`contrib`](contrib) folder) will load the configuration from `/etc/log-courier/log-courier.yaml`. When running Log Courier manually a configuration file can be specified using the [`-config`](CommandLineArguments.md#-configpath) argument.
 
 ## JSON Comments
 
@@ -149,7 +149,7 @@ Several more configuration examples are available for perusal in the
 An example [JSON configuration](examples/example-json.conf) is also available
 that follows a single log file.
 
-The configuration is documented in full below.
+The configuration format is documented in full below.
 
 ## Field Types
 
@@ -192,10 +192,13 @@ character-range:
     lo '-' hi   matches character c for lo <= c <= hi
 ```
 
+In addition to the above, Log Courier also supports `/**/` to match arbitrary number of folders, through use of the [doublestar package](https://pkg.go.dev/github.com/bmatcuk/doublestar). Note that matched such as `/path**/` are not valid and will be instead interpreted as `/path*/`.
+
 - `/var/log/*.log`
 - `/var/log/program/log_????.log`
 - `/var/log/httpd/access.log`
 - `/var/log/httpd/access.log.[0-9]`
+- `/var/log/nginx/**/*.log`
 
 ## `admin`
 
@@ -262,6 +265,11 @@ Array of Fileglobs. Required
 At least one Fileglob must be specified and all matching files for all provided
 globs will be monitored.
 
+If IO errors are encountered whilst scanning for files, the first IO error will be logged
+as a warning by Log Courier and then subsequent scans will ignore IO errors to ensure that
+the logs are not flooded. Each time the configuration is reloaded, IO error detection will
+be reset, logging only the first IO error again.
+
 If the log file is rotated, Log Courier will detect this and automatically start
 harvesting the new file. It will also keep the old file open to catch any
 delayed writes that a still-reloading application has not yet written. You can

go.mod

@@ -3,6 +3,7 @@ module github.com/driskell/log-courier
 go 1.18
 
 require (
+	github.com/bmatcuk/doublestar/v4 v4.6.0
 	github.com/gizak/termui/v3 v3.1.0
 	github.com/google/cel-go v0.13.0
 	github.com/hashicorp/golang-lru v0.5.4

go.sum

@@ -1,5 +1,7 @@
 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves=
 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY=
+github.com/bmatcuk/doublestar/v4 v4.6.0 h1:HTuxyug8GyFbRkrffIpzNCSK4luc0TY3wzXvzIZhEXc=
+github.com/bmatcuk/doublestar/v4 v4.6.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

lc-lib/prospector/config.go

@@ -21,6 +21,7 @@ import (
 	"path/filepath"
 	"time"
 
+	"github.com/bmatcuk/doublestar/v4"
 	"github.com/driskell/log-courier/lc-lib/config"
 	"github.com/driskell/log-courier/lc-lib/harvester"
 )
@@ -142,10 +143,18 @@ func validateFileConfigs(p *config.Parser) (err error) {
 	c := p.Config().Section("files").(Config)
 	for k := range c {
 		if len(c[k].Paths) == 0 {
-			err = fmt.Errorf("No paths specified for /files[%d]/", k)
+			err = fmt.Errorf("no paths specified for /files[%d]/", k)
 			return
 		}
 
+		// Validate patterns
+		for l, path := range c[k].Paths {
+			if !doublestar.ValidatePattern(path) {
+				err = fmt.Errorf("pattern at /files[%d]/paths[%d] is invalid: %s", k, l, path)
+				return
+			}
+		}
+
 		// Init the harvester config
 		if err = c[k].Init(p, fmt.Sprintf("/files[%d]", k)); err != nil {
 			return

lc-lib/prospector/prospector.go

@@ -21,10 +21,10 @@ package prospector
 
 import (
 	"os"
-	"path/filepath"
 	"sync"
 	"time"
 
+	"github.com/bmatcuk/doublestar/v4"
 	"github.com/driskell/log-courier/lc-lib/admin"
 	"github.com/driskell/log-courier/lc-lib/config"
 	"github.com/driskell/log-courier/lc-lib/core"
@@ -50,6 +50,7 @@ type Prospector struct {
 	shutdownChan    <-chan struct{}
 	configChan      <-chan *config.Config
 	output          chan<- []*event.Event
+	didWarnIoError  bool
 }
 
 // NewProspector creates a new path crawler with the given configuration
@@ -198,6 +199,8 @@ DelayLoop:
 		case cfg := <-p.configChan:
 			p.genConfig = cfg.GeneralPart("prospector").(*General)
 			p.fileConfigs = cfg.Section("files").(Config)
+			// Reset flag to warn on IO errors again
+			p.didWarnIoError = false
 		}
 
 		now = time.Now()
@@ -212,7 +215,18 @@ DelayLoop:
 // scan crawls a path for file movements
 func (p *Prospector) scan(path string, cfg *FileConfig) {
 	// Evaluate the path as a wildcards/shell glob
-	matches, err := filepath.Glob(path)
+	var matches []string
+	var err error
+	if !p.didWarnIoError {
+		matches, err = doublestar.FilepathGlob(path, doublestar.WithFailOnIOErrors(), doublestar.WithFilesOnly())
+		if err != nil && err != doublestar.ErrBadPattern {
+			p.didWarnIoError = true
+			log.Warningf("WARNING: glob(%s) failed due to IO errors, further IO errors will be suppressed until reload: %s", path, err)
+			matches, err = doublestar.FilepathGlob(path, doublestar.WithFilesOnly())
+		}
+	} else {
+		matches, err = doublestar.FilepathGlob(path, doublestar.WithFilesOnly())
+	}
 	if err != nil {
 		log.Errorf("glob(%s) failed: %v", path, err)
 		return

testing/.gitignore

@@ -1,4 +1,4 @@
-/*-generated.log
+*-generated.log
 /persist/*/*
 !/persist/*/.gitkeep
 /cert/*

testing/README.md

@@ -22,6 +22,9 @@
       done
       exec 3>&-
     done
+    cp -f singleline-generated.log glob/1/1/
+    cp -f singleline-generated.log glob/2/1/
+    cp -f singleline-generated.log glob/2/2/
 
 ## Starting Log Carver
 

testing/log-courier-glob.yaml

@@ -0,0 +1,16 @@
+general:
+  log syslog: false
+  log stdout: true
+  persist directory: ./persist/lc1
+  log level: debug
+admin:
+  enabled: true
+  listen address: tcp:127.0.0.1:12340
+network:
+  servers:
+  - 127.0.0.1:12345
+  - 127.0.0.1:12346
+  transport: tcp
+files:
+- paths:
+  - 'glob/**/*.log'