Improve the documentation of sincedb and file rotations

lib/logstash/inputs/file.rb

@@ -5,14 +5,62 @@
 require "pathname"
 require "socket" # for Socket.gethostname
 
-# Stream events from files.
+# Stream events from files, normally by tailing them in a manner
+# similar to `tail -0F` but optionally reading them from the
+# beginning.
 #
 # By default, each event is assumed to be one line. If you would like
 # to join multiple log lines into one event, you'll want to use the
-# multiline codec.
+# multiline codec or filter.
 #
-# Files are followed in a manner similar to `tail -0F`. File rotation
-# is detected and handled by this input.
+# The plugin aims to track changing files and emit new content as it's
+# appended to each file. It's not well-suited for reading a file from
+# beginning to end and storing all of it in a single event (not even
+# with the multiline codec or filter).
+#
+# ==== Tracking of current position in watched files
+#
+# The plugin keeps track of the current position in each file by
+# recording it in a separate file named sincedb. This makes it
+# possible to stop and restart Logstash and have it pick up where it
+# left off without missing the lines that were added to the file while
+# Logstash was stopped.
+#
+# By default, the sincedb file is placed in the home directory of the
+# user running Logstash with a filename based on the filename patterns
+# being watched (i.e. the `path` option). Thus, changing the filename
+# patterns will result in a new sincedb file being used and any
+# existing current position state will be lost. If you change your
+# patterns with any frequency it might make sense to explicitly choose
+# a sincedb path with the `sincedb_path` option.
+#
+# Sincedb files are text files with four columns:
+#
+# . The inode number (or equivalent).
+# . The major device number of the file system (or equivalent).
+# . The minor device number of the file system (or equivalent).
+# . The current byte offset within the file.
+#
+# On non-Windows systems you can obtain the inode number of a file
+# with e.g. `ls -li`.
+#
+# ==== File rotation
+#
+# File rotation is detected and handled by this input, regardless of
+# whether the file is rotated via a rename or a copy operation. To
+# support programs that write to the rotated file for some time after
+# the rotation has taken place, include both the original filename and
+# the rotated filename (e.g. /var/log/syslog and /var/log/syslog.1) in
+# the filename patterns to watch (the `path` option). Note that the
+# rotated filename will be treated as a new file so if
+# `start_position` is set to 'beginning' the rotated file will be
+# reprocessed.
+#
+# With the default value of `start_position` ('end') any messages
+# written to the end of the file between the last read operation prior
+# to the rotation and its reopening under the new name (an interval
+# determined by the `stat_interval` and `discover_interval` options)
+# will not get picked up.
 class LogStash::Inputs::File < LogStash::Inputs::Base
   config_name "file"
 
@@ -21,15 +69,15 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
   default :codec, "plain"
 
   # The path(s) to the file(s) to use as an input.
-  # You can use globs here, such as `/var/log/*.log`
+  # You can use filename patterns here, such as `/var/log/*.log`.
   # Paths must be absolute and cannot be relative.
   #
   # You may also configure multiple paths. See an example
   # on the <<array,Logstash configuration page>>.
   config :path, :validate => :array, :required => true
 
-  # Exclusions (matched against the filename, not full path). Globs
-  # are valid here, too. For example, if you have
+  # Exclusions (matched against the filename, not full path). Filename
+  # patterns are valid here, too. For example, if you have
   # [source,ruby]
   #     path => "/var/log/*"
   #
@@ -43,7 +91,8 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
   # but increase the time to detect new log lines.
   config :stat_interval, :validate => :number, :default => 1
 
-  # How often (in seconds) we expand globs to discover new files to watch.
+  # How often (in seconds) we expand the filename patterns in the
+  # `path` option to discover new files to watch.
   config :discover_interval, :validate => :number, :default => 15
 
   # Path of the sincedb database file (keeps track of the current
@@ -59,11 +108,13 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
   # Choose where Logstash starts initially reading files: at the beginning or
   # at the end. The default behavior treats files like live streams and thus
   # starts at the end. If you have old data you want to import, set this
-  # to 'beginning'
+  # to 'beginning'.
   #
-  # This option only modifies "first contact" situations where a file is new
-  # and not seen before. If a file has already been seen before, this option
-  # has no effect.
+  # This option only modifies "first contact" situations where a file
+  # is new and not seen before, i.e. files that don't have a current
+  # position recorded in a sincedb file read by Logstash. If a file
+  # has already been seen before, this option has no effect and the
+  # position recorded in the sincedb file will be used.
   config :start_position, :validate => [ "beginning", "end"], :default => "end"
 
   # set the new line delimiter, defaults to "\n"