Merge branch 'main' into gosec-issues

.github/codecov.yaml

@@ -0,0 +1,20 @@
+codecov:
+  require_ci_to_pass: yes
+
+ignore:
+  - "internal/tools/*"
+
+coverage:
+  precision: 1
+  round: down
+  range: "70...100"
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 0.5%
+
+comment:
+  layout: "reach,diff,flags,tree"
+  behavior: default
+  require_changes: yes

.github/workflows/codecov.yml

@@ -0,0 +1,50 @@
+name: Run CodeCov
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+env:
+  # Path to where test results will be saved.
+  TEST_RESULTS: /tmp/test-results
+  # Default minimum version of Go to support.
+  DEFAULT_GO_VERSION: 1.15
+jobs:
+  test-coverage:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2.1.3
+      with:
+        go-version: ${{ env.DEFAULT_GO_VERSION }}
+    - name: Checkout Repo
+      uses: actions/checkout@v2
+    - name: Setup Environment
+      run: |
+        echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
+        echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
+    - name: Module cache
+      uses: actions/cache@v2.1.6
+      env:
+        cache-name: go-mod-cache
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('**/go.sum') }}
+    - name: Run coverage tests
+      run: |
+        make test-coverage
+        mkdir $TEST_RESULTS
+        cp coverage.out $TEST_RESULTS
+        cp coverage.txt $TEST_RESULTS
+        cp coverage.html $TEST_RESULTS
+    - name: Upload coverage report
+      uses: codecov/codecov-action@v1
+      with:
+        file: ./coverage.txt
+        fail_ci_if_error: true
+        verbose: true
+    - name: Store coverage test output
+      uses: actions/upload-artifact@v2
+      with:
+          name: opentelemetry-log-collection-test-output
+          path: ${{ env.TEST_RESULTS }}

.gitignore

@@ -8,3 +8,6 @@ artifacts/*
 .vscode/*
 gen/
 .idea/*
+coverage.out
+coverage.out.bak
+coverage.txt.bak

Makefile

@@ -4,10 +4,10 @@ GOARCH=$(shell go env GOARCH)
 GIT_SHA=$(shell git rev-parse --short HEAD)
 
 PROJECT_ROOT = $(shell pwd)
-ARTIFACTS = ${PROJECT_ROOT}/artifacts
 ALL_MODULES := $(shell find . -type f -name "go.mod" -exec dirname {} \; | sort )
 ALL_SRC := $(shell find . -name '*.go' -type f | sort)
 ADDLICENSE=addlicense
+ALL_COVERAGE_MOD_DIRS := $(shell find . -type f -name 'go.mod' -exec dirname {} \; | egrep -v '^./internal/tools' | sort)
 
 TOOLS_MOD_DIR := ./internal/tools
 .PHONY: install-tools
@@ -24,14 +24,27 @@ test: vet test-only
 test-only:
 	$(MAKE) for-all CMD="go test -race -coverprofile coverage.txt -coverpkg ./... ./..."
 
+.PHONY: test-coverage
+test-coverage: clean
+	@set -e; \
+	printf "" > coverage.txt; \
+	for dir in $(ALL_COVERAGE_MOD_DIRS); do \
+	  (cd "$${dir}" && \
+	    go list ./... \
+	    | grep -v third_party \
+	    | xargs go test -coverpkg=./... -covermode=atomic -coverprofile=coverage.out && \
+	  go tool cover -html=coverage.out -o coverage.html); \
+	  [ -f "$${dir}/coverage.out" ] && cat "$${dir}/coverage.out" >> coverage.txt; \
+	done; \
+	sed -i.bak -e '2,$$ { /^mode: /d; }' coverage.txt	
+
 .PHONY: bench
 bench:
-	$(MAKE) for-all CMD="go test -run=NONE -bench '.*' ./... -benchmem"
+	go test -benchmem -run=^$$ -bench ^* ./...
 
 .PHONY: clean
 clean:
-	rm -fr ./artifacts
-	$(MAKE) for-all CMD="rm -f coverage.txt coverage.html"
+	$(MAKE) for-all CMD="rm -f coverage.txt.* coverage.html coverage.out"
 
 .PHONY: tidy
 tidy:
@@ -96,4 +109,4 @@ for-all:
 	done
 
 .PHONY: ci-check
-ci-check: vet lint check-license secure
+ci-check: vet lint check-license secure

README.md

@@ -1,4 +1,7 @@
 # opentelemetry-log-collection
+
+[![codecov.io](https://codecov.io/gh/open-telemetry/opentelemetry-log-collection/coverage.svg?branch=main)](https://app.codecov.io/gh/open-telemetry/opentelemetry-log-collection?branch=main)
+
 ## Status
 
 This project was originally developed by [observIQ](https://observiq.com/) under the name [Stanza](https://github.com/observIQ/stanza). It has been contributed to the OpenTelemetry project in order to accelerate development of the collector's log collection capabilities.

operator/builtin/input/file/benchmark_test.go

@@ -0,0 +1,194 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package file
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/open-telemetry/opentelemetry-log-collection/operator"
+	"github.com/open-telemetry/opentelemetry-log-collection/testutil"
+)
+
+type fileInputBenchmark struct {
+	name   string
+	paths  []string
+	config func() *InputConfig
+}
+
+type benchFile struct {
+	*os.File
+	log func(int)
+}
+
+func simpleTextFile(file *os.File) *benchFile {
+	line := stringWithLength(49) + "\n"
+	return &benchFile{
+		File: file,
+		log:  func(_ int) { file.WriteString(line) },
+	}
+}
+
+func BenchmarkFileInput(b *testing.B) {
+	cases := []fileInputBenchmark{
+		{
+			name: "Single",
+			paths: []string{
+				"file0.log",
+			},
+			config: func() *InputConfig {
+				cfg := NewInputConfig("test_id")
+				cfg.Include = []string{
+					"file0.log",
+				}
+				return cfg
+			},
+		},
+		{
+			name: "Glob",
+			paths: []string{
+				"file0.log",
+				"file1.log",
+				"file2.log",
+				"file3.log",
+			},
+			config: func() *InputConfig {
+				cfg := NewInputConfig("test_id")
+				cfg.Include = []string{"file*.log"}
+				return cfg
+			},
+		},
+		{
+			name: "MultiGlob",
+			paths: []string{
+				"file0.log",
+				"file1.log",
+				"log0.log",
+				"log1.log",
+			},
+			config: func() *InputConfig {
+				cfg := NewInputConfig("test_id")
+				cfg.Include = []string{
+					"file*.log",
+					"log*.log",
+				}
+				return cfg
+			},
+		},
+		{
+			name: "MaxConcurrent",
+			paths: []string{
+				"file0.log",
+				"file1.log",
+				"file2.log",
+				"file3.log",
+			},
+			config: func() *InputConfig {
+				cfg := NewInputConfig("test_id")
+				cfg.Include = []string{
+					"file*.log",
+				}
+				cfg.MaxConcurrentFiles = 1
+				return cfg
+			},
+		},
+		{
+			name: "FngrPrntLarge",
+			paths: []string{
+				"file0.log",
+			},
+			config: func() *InputConfig {
+				cfg := NewInputConfig("test_id")
+				cfg.Include = []string{
+					"file*.log",
+				}
+				cfg.FingerprintSize = 10 * defaultFingerprintSize
+				return cfg
+			},
+		},
+		{
+			name: "FngrPrntSmall",
+			paths: []string{
+				"file0.log",
+			},
+			config: func() *InputConfig {
+				cfg := NewInputConfig("test_id")
+				cfg.Include = []string{
+					"file*.log",
+				}
+				cfg.FingerprintSize = defaultFingerprintSize / 10
+				return cfg
+			},
+		},
+	}
+
+	for _, bench := range cases {
+		b.Run(bench.name, func(b *testing.B) {
+			rootDir, err := ioutil.TempDir("", "")
+			require.NoError(b, err)
+
+			files := []*benchFile{}
+			for _, path := range bench.paths {
+				file := openFile(b, filepath.Join(rootDir, path))
+				files = append(files, simpleTextFile(file))
+			}
+
+			cfg := bench.config()
+			cfg.OutputIDs = []string{"fake"}
+			for i, inc := range cfg.Include {
+				cfg.Include[i] = filepath.Join(rootDir, inc)
+			}
+			cfg.StartAt = "beginning"
+
+			ops, err := cfg.Build(testutil.NewBuildContext(b))
+			require.NoError(b, err)
+			op := ops[0]
+
+			fakeOutput := testutil.NewFakeOutput(b)
+			err = op.SetOutputs([]operator.Operator{fakeOutput})
+			require.NoError(b, err)
+
+			// write half the lines before starting
+			mid := b.N / 2
+			for i := 0; i < mid; i++ {
+				for _, file := range files {
+					file.log(i)
+				}
+			}
+
+			b.ResetTimer()
+			err = op.Start(testutil.NewMockPersister("test"))
+			defer op.Stop()
+			require.NoError(b, err)
+
+			// write the remainder of lines while running
+			go func() {
+				for i := mid; i < b.N; i++ {
+					for _, file := range files {
+						file.log(i)
+					}
+				}
+			}()
+
+			for i := 0; i < b.N*len(files); i++ {
+				<-fakeOutput.Received
+			}
+		})
+	}
+}

operator/builtin/input/file/config.go

@@ -42,6 +42,7 @@ func NewInputConfig(operatorID string) *InputConfig {
 		IncludeFileName:    true,
 		IncludeFilePath:    false,
 		StartAt:            "end",
+		FingerprintSize:    defaultFingerprintSize,
 		MaxLogSize:         defaultMaxLogSize,
 		MaxConcurrentFiles: defaultMaxConcurrentFiles,
 		Encoding:           helper.NewEncodingConfig(),