Attempt to fix issue #9

falib/archiver.go

@@ -29,6 +29,7 @@ type Archiver struct {
 	excludePatterns    []string
 	output             *bufio.Writer
 	error              error
+	directoryQueueLock sync.Mutex
 }
 
 func NewArchiver(output io.Writer) *Archiver {
@@ -84,6 +85,8 @@ func (a *Archiver) Run() error {
 }
 
 func (a *Archiver) directoryScanner() {
+	outgoingDirectoryScanQueue := make([]string, 0)
+
 	for directoryPath := range a.directoryScanQueue {
 		if strings.HasPrefix(directoryPath, "/") {
 			a.error = ErrAbsoluteDirectoryPath
@@ -129,23 +132,32 @@ func (a *Archiver) directoryScanner() {
 
 			a.workInProgress.Add(1)
 			if fileInfo.IsDir() {
-				// Sending to directoryScanQueue can block if it's full; since
-				// we're also the goroutine responsible for reading from it,
-				// this could cause a deadlock.  We break that deadlock by
-				// performing the send in a goroutine, where it can block
-				// safely.  This does have the side-effect that
-				// directoryScanQueue's max size is pretty much ineffective...
-				// but that's better than a deadlock.
-				go func(filePath string) {
-					a.directoryScanQueue <- filePath
-				}(filePath)
+				outgoingDirectoryScanQueue = append(outgoingDirectoryScanQueue, filePath)
 			} else {
 				a.fileReadQueue <- filePath
 			}
 		}
 
 		directory.Close()
 		a.workInProgress.Done()
+
+		// Sending to directoryScanQueue can block if it's full; since
+		// we're also the goroutine responsible for reading from it,
+		// this could cause a deadlock.  We remove the possibility of deadlock
+		// by synchronizing access to directoryScanQueue and only appending items
+		// to it when there's room in the queue.  Otherwise, we keep the
+		// elements in local storage; this could cause higher memory usage,
+		// but that seems safer than deadlocks.  No need to increment
+		// workInProgress in this loop; that's already done when appending to
+		// outgoingDirectoryScanQueue.
+		if len(outgoingDirectoryScanQueue) > 0 {
+			a.directoryQueueLock.Lock()
+			for len(outgoingDirectoryScanQueue) > 0 && len(a.directoryScanQueue) < cap(a.directoryScanQueue) {
+				a.directoryScanQueue <- outgoingDirectoryScanQueue[0]
+				outgoingDirectoryScanQueue = outgoingDirectoryScanQueue[1:]
+			}
+			a.directoryQueueLock.Unlock()
+		}
 	}
 }