feat(sources): Add new ifile source with improved implementation

.github/actions/spelling/allow.txt

@@ -235,6 +235,7 @@ Ideapad
 idn
 ifeq
 ifneq
+ifile
 imobile
 Infinix
 influxd

Cargo.toml

@@ -109,6 +109,7 @@ members = [
   "lib/fakedata",
   "lib/file-source",
   "lib/file-source-common",
+  "lib/ifile-source",
   "lib/k8s-e2e-tests",
   "lib/k8s-test-framework",
   "lib/loki-logproto",
@@ -596,6 +597,7 @@ sources-logs = [
   "sources-docker_logs",
   "sources-exec",
   "sources-file",
+  "sources-ifile",
   "sources-fluent",
   "sources-gcp_pubsub",
   "sources-heroku_logs",
@@ -649,7 +651,8 @@ sources-dnstap = ["sources-utils-net-tcp", "dep:base64", "dep:hickory-proto", "d
 sources-docker_logs = ["docker"]
 sources-eventstoredb_metrics = []
 sources-exec = []
-sources-file = ["vector-lib/file-source"]
+sources-file = ["vector-lib/file-source", "vector-lib/file-source-common"]
+sources-ifile = ["vector-lib/ifile-source", "vector-lib/file-source-common"]
 sources-file_descriptor = ["tokio-util/io"]
 sources-fluent = ["dep:base64", "sources-utils-net-tcp", "sources-utils-net-unix", "tokio-util/net", "dep:rmpv", "dep:rmp-serde", "dep:serde_bytes"]
 sources-gcp_pubsub = ["gcp", "dep:h2", "dep:prost", "dep:prost-types", "protobuf-build", "dep:tonic"]
@@ -662,7 +665,7 @@ sources-internal_metrics = []
 sources-static_metrics = []
 sources-journald = []
 sources-kafka = ["dep:rdkafka"]
-sources-kubernetes_logs = ["vector-lib/file-source", "kubernetes", "transforms-reduce"]
+sources-kubernetes_logs = ["vector-lib/file-source", "vector-lib/file-source-common", "kubernetes", "transforms-reduce"]
 sources-logstash = ["sources-utils-net-tcp", "tokio-util/net"]
 sources-mongodb_metrics = ["dep:mongodb"]
 sources-mqtt = ["dep:rumqttc"]

changelog.d/22803_ifile_source.feature.md

@@ -0,0 +1,17 @@
+A new `ifile` source has been added as an improved version of the `file` source. The `ifile` source is a complete rewrite that addresses several limitations of the original implementation while maintaining compatibility with existing configurations.
+
+Key improvements in the `ifile` source:
+
+- **Fully async implementation**: Uses async/await throughout for better performance and resource utilization
+- **Cross-platform filesystem notifications**: Uses the notify-rs library to detect file changes through OS-level notifications instead of polling
+- **Improved file discovery**: Detects new files within milliseconds using filesystem notifications instead of periodic globbing. Only globs once on startup for initial discovery, and therefore obsoletes and removes the `glob_minimum_cooldown_ms` option
+- **Resource efficiency**: Never keeps file handles open for idle files, only opening them when needed for reading
+- **Better shutdown behavior**: Properly handles shutdown signals and gracefully closes all resources
+- **Improved checkpointing**: Introduces a new `checkpoint_interval` configuration option
+- **Startup consistency**: Reads files at startup to detect changes that occurred while Vector was stopped
+- **Intelligent throttling**: Optimizes CPU usage and log verbosity with smart event handling
+- **Better error handling**: Properly handles file deletion events to prevent repeated error messages
+
+The original `file` source remains unchanged and fully supported. Users can migrate to the `ifile` source at their convenience to take advantage of these improvements.
+
+authors: tamer-hassan