forked from elastic/beats
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge remote-tracking branch 'upstream/master' into feature/2.0-skip-…
…ci-build-label-support * upstream/master: [CI] Change x-pack/auditbeat build events (comments, labels) (elastic#21463) [CI] changeset from elastic#20603 was not added to CI2.0 (elastic#21464) Add new log file reader for filestream input (elastic#21450) [CI] Send slack message with build status (elastic#21428) Remove duplicated sources url in dependencies report (elastic#21462) Add implementation of FSWatcher and FSScanner for filestream (elastic#21444) [Ingest Manager] Split index restrictions into type,dataset, namespace parts (elastic#21406) Update Filebeat module expected logs files (elastic#21454) Edit SQL module docs and fix broken doc structure (elastic#21233) [Ingest Manager] Send snapshot flag together with metadata (elastic#21285) Revert "[JJBB] Set shallow cloning to 10 (elastic#21409)" (elastic#21447) [JJBB] Use reference repo for fast checkouts (elastic#21410) Add initial skeleton of filestream input (elastic#21427) Initial spec file for apm-server (elastic#21225) [Ingest Manager] Upgrade Action: make source URI optional (elastic#21372) Add field limit check for AWS Cloudtrail flattened fields (elastic#21388) [Winlogbeat] Move winlogbeat javascript processor to libbeat (elastic#21402) ci: pipeline to generate the changelog (elastic#21426)
- Loading branch information
Showing
175 changed files
with
6,486 additions
and
2,241 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| --- | ||
| - job: | ||
| name: Beats/Release/beats-release-changelog | ||
| display-name: 'Prepare the Changelog for a Release' | ||
| description: 'Automate the steps to prepare the Changelog for a Release' | ||
| view: Beats | ||
| project-type: pipeline | ||
| pipeline-scm: | ||
| script-path: release_scripts/pipeline-release-changelog.groovy | ||
| scm: | ||
| - git: | ||
| url: git@github.com:elastic/ingest-dev.git | ||
| refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/* | ||
| wipe-workspace: 'True' | ||
| name: origin | ||
| shallow-clone: true | ||
| credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba | ||
| reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git | ||
| branches: | ||
| - master |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,147 @@ | ||
| // Licensed to Elasticsearch B.V. under one or more contributor | ||
| // license agreements. See the NOTICE file distributed with | ||
| // this work for additional information regarding copyright | ||
| // ownership. Elasticsearch B.V. licenses this file to you under | ||
| // the Apache License, Version 2.0 (the "License"); you may | ||
| // not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, | ||
| // software distributed under the License is distributed on an | ||
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| // KIND, either express or implied. See the License for the | ||
| // specific language governing permissions and limitations | ||
| // under the License. | ||
|
|
||
| package filestream | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "time" | ||
|
|
||
| "github.com/dustin/go-humanize" | ||
|
|
||
| "github.com/elastic/beats/v7/libbeat/common" | ||
| "github.com/elastic/beats/v7/libbeat/common/match" | ||
| "github.com/elastic/beats/v7/libbeat/reader/readfile" | ||
| ) | ||
|
|
||
| // Config stores the options of a file stream. | ||
| type config struct { | ||
| Paths []string `config:"paths"` | ||
| Close closerConfig `config:"close"` | ||
| FileWatcher *common.ConfigNamespace `config:"file_watcher"` | ||
| Reader readerConfig `config:"readers"` | ||
| FileIdentity *common.ConfigNamespace `config:"file_identity"` | ||
| CleanInactive time.Duration `config:"clean_inactive" validate:"min=0"` | ||
| CleanRemoved bool `config:"clean_removed"` | ||
| HarvesterLimit uint32 `config:"harvester_limit" validate:"min=0"` | ||
| IgnoreOlder time.Duration `config:"ignore_older"` | ||
| } | ||
|
|
||
| type closerConfig struct { | ||
| OnStateChange stateChangeCloserConfig `config:"on_state_change"` | ||
| Reader readerCloserConfig `config:"reader"` | ||
| } | ||
|
|
||
| type readerCloserConfig struct { | ||
| AfterInterval time.Duration | ||
| Inactive time.Duration | ||
| OnEOF bool | ||
| } | ||
|
|
||
| type stateChangeCloserConfig struct { | ||
| CheckInterval time.Duration | ||
| Removed bool | ||
| Renamed bool | ||
| } | ||
|
|
||
| // TODO should this be inline? | ||
| type readerConfig struct { | ||
| Backoff backoffConfig `config:"backoff"` | ||
| BufferSize int `config:"buffer_size"` | ||
| Encoding string `config:"encoding"` | ||
| ExcludeLines []match.Matcher `config:"exclude_lines"` | ||
| IncludeLines []match.Matcher `config:"include_lines"` | ||
| LineTerminator readfile.LineTerminator `config:"line_terminator"` | ||
| MaxBytes int `config:"message_max_bytes" validate:"min=0,nonzero"` | ||
| Tail bool `config:"seek_to_tail"` | ||
|
|
||
| Parsers []*common.ConfigNamespace `config:"parsers"` // TODO multiline, json, syslog? | ||
| } | ||
|
|
||
| type backoffConfig struct { | ||
| Init time.Duration `config:"init" validate:"nonzero"` | ||
| Max time.Duration `config:"max" validate:"nonzero"` | ||
| } | ||
|
|
||
| func defaultConfig() config { | ||
| return config{ | ||
| Paths: []string{}, | ||
| Close: defaultCloserConfig(), | ||
| Reader: defaultReaderConfig(), | ||
| CleanInactive: 0, | ||
| CleanRemoved: true, | ||
| HarvesterLimit: 0, | ||
| IgnoreOlder: 0, | ||
| } | ||
| } | ||
|
|
||
| func defaultCloserConfig() closerConfig { | ||
| return closerConfig{ | ||
| OnStateChange: stateChangeCloserConfig{ | ||
| CheckInterval: 5 * time.Second, | ||
| Removed: true, // TODO check clean_removed option | ||
| Renamed: false, | ||
| }, | ||
| Reader: readerCloserConfig{ | ||
| OnEOF: false, | ||
| Inactive: 0 * time.Second, | ||
| AfterInterval: 0 * time.Second, | ||
| }, | ||
| } | ||
| } | ||
|
|
||
| func defaultReaderConfig() readerConfig { | ||
| return readerConfig{ | ||
| Backoff: backoffConfig{ | ||
| Init: 1 * time.Second, | ||
| Max: 10 * time.Second, | ||
| }, | ||
| BufferSize: 16 * humanize.KiByte, | ||
| LineTerminator: readfile.AutoLineTerminator, | ||
| MaxBytes: 10 * humanize.MiByte, | ||
| Tail: false, | ||
| Parsers: nil, | ||
| } | ||
| } | ||
|
|
||
| func (c *config) Validate() error { | ||
| if len(c.Paths) == 0 { | ||
| return fmt.Errorf("no path is configured") | ||
| } | ||
| // TODO | ||
| //if c.CleanInactive != 0 && c.IgnoreOlder == 0 { | ||
| // return fmt.Errorf("ignore_older must be enabled when clean_inactive is used") | ||
| //} | ||
|
|
||
| // TODO | ||
| //if c.CleanInactive != 0 && c.CleanInactive <= c.IgnoreOlder+c.ScanFrequency { | ||
| // return fmt.Errorf("clean_inactive must be > ignore_older + scan_frequency to make sure only files which are not monitored anymore are removed") | ||
| //} | ||
|
|
||
| // TODO | ||
| //if c.JSON != nil && len(c.JSON.MessageKey) == 0 && | ||
| // c.Multiline != nil { | ||
| // return fmt.Errorf("When using the JSON decoder and multiline together, you need to specify a message_key value") | ||
| //} | ||
|
|
||
| //if c.JSON != nil && len(c.JSON.MessageKey) == 0 && | ||
| // (len(c.IncludeLines) > 0 || len(c.ExcludeLines) > 0) { | ||
| // return fmt.Errorf("When using the JSON decoder and line filtering together, you need to specify a message_key value") | ||
| //} | ||
|
|
||
| return nil | ||
| } |
Oops, something went wrong.