Merge branch 'main' into wip-licenses-in-docker-image

.github/dependabot.yml

@@ -14,6 +14,10 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+  - package-ecosystem: "gomod"
+    directory: "/cmd/builder"
+    schedule:
+      interval: "weekly"
   - package-ecosystem: "gomod"
     directory: "/internal/tools"
     schedule:

.github/workflows/api-compatibility.yml

@@ -7,6 +7,8 @@
 name: "Inform Incompatible PRs"
 on:
   pull_request:
+    paths-ignore:
+      - 'cmd/builder/**'
     branches:
       - main
 

.github/workflows/build-and-test.yml

@@ -1,9 +1,9 @@
 name: build-and-test
 on:
   push:
-    branches: [ main ]
+    branches: [main]
     tags:
-      - 'v[0-9]+.[0-9]+.[0-9]+*'
+      - "v[0-9]+.[0-9]+.[0-9]+*"
   pull_request:
 
 jobs:
@@ -23,21 +23,15 @@ jobs:
       - name: Cache Go
         id: module-cache
         uses: actions/cache@v2
-        env:
-          cache-name: cache-go-modules
         with:
-          path: |
-            /home/runner/go/pkg/mod
-            /home/runner/.cache/go-build
+          path: /home/runner/go/pkg/mod
           key: go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }}
       - name: Install dependencies
         if: steps.module-cache.outputs.cache-hit != 'true'
         run: make gomoddownload
       - name: Cache Tools
         id: tool-cache
         uses: actions/cache@v2
-        env:
-          cache-name: cache-tool-binaries
         with:
           path: /home/runner/go/bin
           key: tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod', './cmd/mdatagen/go.mod', './cmd/mdatagen/*.go') }}
@@ -61,23 +55,17 @@ jobs:
       - name: Cache Go
         id: module-cache
         uses: actions/cache@v2
-        env:
-          cache-name: cache-go-modules
         with:
-          path: |
-            /home/runner/go/pkg/mod
-            /home/runner/.cache/go-build
+          path: /home/runner/go/pkg/mod
           key: go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }}
       - name: Cache Tools
         id: tool-cache
         uses: actions/cache@v2
-        env:
-          cache-name: cache-tool-binaries
         with:
           path: /home/runner/go/bin
           key: tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod', './cmd/mdatagen/go.mod', './cmd/mdatagen/*.go') }}
       - name: Lint
-        run: make -j5 checklicense misspell checkdoc goimpi golint
+        run: make -j5 checklicense misspell checkdoc goimpi golint goporto
       - name: Gen Metadata
         run: |
           make genmdata
@@ -90,6 +78,8 @@ jobs:
         run: |
           make gendependabot
           git diff -s --exit-code || (echo 'ALL_MODULES have been changed, please run "make gendependabot" and commit the changes in this PR.' && exit 1)
+      - name: Multimod verify
+        run: make multimod-verify
   unittest:
     runs-on: ubuntu-latest
     needs: [setup-environment]
@@ -107,21 +97,20 @@ jobs:
       - name: Cache Go
         id: module-cache
         uses: actions/cache@v2
-        env:
-          cache-name: cache-go-modules
         with:
-          path: |
-            /home/runner/go/pkg/mod
-            /home/runner/.cache/go-build
+          path: /home/runner/go/pkg/mod
           key: go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }}
       - name: Cache Tools
         id: tool-cache
         uses: actions/cache@v2
-        env:
-          cache-name: cache-tool-binaries
         with:
           path: /home/runner/go/bin
           key: tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod', './cmd/mdatagen/go.mod', './cmd/mdatagen/*.go') }}
+      - name: Cache Build
+        uses: actions/cache@v2
+        with:
+          path: /home/runner/.cache/go-build
+          key: go-build-unittest-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }}
       - name: Run Unit Tests
         run: make gotest
   test-coverage:
@@ -141,25 +130,24 @@ jobs:
       - name: Cache Go
         id: module-cache
         uses: actions/cache@v2
-        env:
-          cache-name: cache-go-modules
         with:
-          path: |
-            /home/runner/go/pkg/mod
-            /home/runner/.cache/go-build
+          path: /home/runner/go/pkg/mod
           key: go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }}
       - name: Cache Tools
         id: tool-cache
         uses: actions/cache@v2
-        env:
-          cache-name: cache-tool-binaries
         with:
           path: /home/runner/go/bin
           key: tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod', './cmd/mdatagen/go.mod', './cmd/mdatagen/*.go') }}
+      - name: Cache Build
+        uses: actions/cache@v2
+        with:
+          path: /home/runner/.cache/go-build
+          key: go-build-coverage-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }}
       - name: Run Unit Tests With Coverage
         run: make gotest-with-cover
       - name: Upload coverage report
-        uses: codecov/codecov-action@v2.0.3
+        uses: codecov/codecov-action@v2.1.0
         with:
           file: ./coverage.txt
           fail_ci_if_error: true
@@ -181,12 +169,8 @@ jobs:
       - name: Cache Go
         id: module-cache
         uses: actions/cache@v2
-        env:
-          cache-name: cache-go-modules
         with:
-          path: |
-            /home/runner/go/pkg/mod
-            /home/runner/.cache/go-build
+          path: /home/runner/go/pkg/mod
           key: go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }}
       - name: Build Collector for All Architectures
         run: make binaries-all-sys
@@ -206,4 +190,3 @@ jobs:
         with:
           name: collector-binaries-unstable
           path: ./bin-unstable.tar
-

.github/workflows/builder-integration-test.yaml

@@ -0,0 +1,28 @@
+name: Builder - Integration tests
+
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - 'cmd/builder/**'
+  pull_request:
+    branches: [ main ]
+    paths:
+      - 'cmd/builder/**'
+
+jobs:
+  integration-test:
+    name: Integration test
+    runs-on: ubuntu-20.04
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.17
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - name: Test
+      run: cd ./cmd/builder && ./test/test.sh

.github/workflows/check-links.yaml

@@ -2,7 +2,11 @@ name: check-links
 on:
   push:
     branches: [ main ]
+    paths-ignore:
+      - 'cmd/builder/**'
   pull_request:
+    paths-ignore:
+      - 'cmd/builder/**'
 
 jobs:
   check-links:

.gitignore

@@ -28,3 +28,6 @@ coverage.html
 
 # Third-Party license drops
 cmd/otelcol/third-party
+
+# OpenTelemetry proto files which are normally cleaned up
+model/internal/opentelemetry-proto

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "model/internal/opentelemetry-proto"]
-	path = model/internal/opentelemetry-proto
-	url = https://github.com/open-telemetry/opentelemetry-proto

CHANGELOG.md

@@ -4,15 +4,70 @@
 
 ## 🛑 Breaking changes 🛑
 
-- Move configcheck.ValidateConfigFromFactories as internal function in service package (#3876)
+- Remove `config.Receivers`, `config.Exporters`, `config.Processors`, and `config.Extensions`. Use map directly (#4344)
+- Remove `component.BaseProcessorFactory`, use `processorhelper.NewFactory` instead (#4175)
+- Force usage of `exporterhelper.NewFactory` to implement `component.ExporterFactory` (#4338)
+- Force usage of `receiverhelper.NewFactory` to implement `component.ReceiverFactory` (#4338)
+- Force usage of `extensionhelper.NewFactory` to implement `component.ExtensionFactory` (#4338)
+- Move `service/parserprovider` package to `config/configmapprovider` (#4206)
+- Renamed slice-valued `pdata` types and functions for consistency. (#4325)
+  - Rename `pdata.AnyValueArray` to `pdata.AttributeValueSlice`
+  - Rename `ArrayVal()` to `SliceVal()`
+  - Rename `SetArrayVal()` to `SetSliceVal()`
+- Remove `config.Pipeline.Name` (#4326)
+- Remove `config.Pipeline.InputDataType` (#4343)
+- otlpexporter: Do not retry on PermissionDenied and Unauthenticated (#4349)
+- Remove deprecated funcs `consumererror.As[Traces|Metrics|Logs]` (#4364)
+
+## v0.38.0 Beta
+
+## 🛑 Breaking changes 🛑
+
+- Removed `configauth.HTTPClientAuthenticator` and `configauth.GRPCClientAuthenticator` in favor of `configauth.ClientAuthenticator`. (#4255)
+- Rename `parserprovider.MapProvider` as `config.MapProvider`. (#4178)
+- Rename `parserprovider.Watchable` as `config.WatchableMapProvider`. (#4178)
+- Remove deprecated no-op flags to setup Collector's logging "--log-level", "--log-profile", "--log-format". (#4213)
+- Move `cmd/pdatagen` as internal package `model/internal/cmd/pdatagen`. (#4243)
+- Use directly the ComponentID in configauth. (#4238)
+- Refactor configauth, getters use the map instead of iteration. (#4234)
+- Change scraperhelper to follow the recommended append model for pdata. (#4202)
+
+## 💡 Enhancements 💡
+
+- Update proto to 0.11.0. (#4209)
+- Change pdata to use the newly added [Traces|Metrics|Logs]Data. (#4214)
+- Add ExponentialHistogram field to pdata. (#4219)
+- Make sure otlphttp exporter tests include TraceID and SpanID. (#4268)
+- Use multimod tool in release process. (#4229)
+- Change queue metrics to use opencensus metrics instead of stats, close to otel-go. (#4220)
+- Make receiver data delivery guarantees explicit (#4262)
+- Simplify unmarshal logic by adding more supported hooks. (#4237)
+- Add unmarshaler for otlpgrpc.[*]Request and otlpgrp.[*]Response (#4215)
+
+## v0.37.0 Beta
+
+## 🛑 Breaking changes 🛑
+
+- Move `configcheck.ValidateConfigFromFactories` as internal function in service package (#3876)
 - Rename `configparser.Parser` as `config.Map` (#4075)
 - Rename `component.DefaultBuildInfo()` to `component.NewDefaultBuildInfo()` (#4129)
-- Rename consumererror.Permanent to consumererror.NewPermanent (#4118)
+- Rename `consumererror.Permanent` to `consumererror.NewPermanent` (#4118)
 - Rename `config.NewID` to `config.NewComponentID` and `config.NewIDFromString` to `config.NewComponentIDFromString` (#4137)
+- Rename `config.NewIDWithName` to `config.NewComponentIDWithName` (#4151)
+- Move `extension/storage` to `extension/experimental/storage` (#4082)
+- Rename `obsreporttest.SetupRecordedMetricsTest()` to `obsreporttest.SetupTelemetry()` and `obsreporttest.TestTelemetrySettings` to `obsreporttest.TestTelemetry` (#4157)
 
 ## 💡 Enhancements 💡
 
 - Add Gen dependabot into CI (#4083)
+- Update OTLP to v0.10.0 (#4045).
+- Add Flags field to NumberDataPoint, HistogramDataPoint, SummaryDataPoint (#4081).
+- Add feature gate library (#4108)
+- Add version to the internal telemetry metrics (#4140)
+
+## 🧰 Bug fixes 🧰
+
+- Fix panic when not using `service.NewCommand` (#4139)
 
 ## v0.36.0 Beta
 

Makefile

@@ -71,6 +71,10 @@ gotest-with-cover:
 	@$(MAKE) for-all CMD="make test-with-cover"
 	$(GOCOVMERGE) $$(find . -name coverage.out) > coverage.txt
 
+.PHONY: goporto
+goporto:
+	@$(MAKE) for-all CMD="make porto"
+
 .PHONY: golint
 golint:
 	@$(MAKE) for-all CMD="make lint lint-unstable"
@@ -126,7 +130,6 @@ install-tools:
 	cd $(TOOLS_MOD_DIR) && go install github.com/golangci/golangci-lint/cmd/golangci-lint
 	cd $(TOOLS_MOD_DIR) && go install github.com/google/addlicense
 	cd $(TOOLS_MOD_DIR) && go install github.com/google/go-licenses
-	cd $(TOOLS_MOD_DIR) && go install github.com/mjibson/esc
 	cd $(TOOLS_MOD_DIR) && go install github.com/ory/go-acc
 	cd $(TOOLS_MOD_DIR) && go install github.com/pavius/impi/cmd/impi
 	cd $(TOOLS_MOD_DIR) && go install github.com/tcnksm/ghr
@@ -135,6 +138,8 @@ install-tools:
 	cd $(TOOLS_MOD_DIR) && go install go.opentelemetry.io/build-tools/semconvgen
 	cd $(TOOLS_MOD_DIR) && go install golang.org/x/exp/cmd/apidiff
 	cd $(TOOLS_MOD_DIR) && go install golang.org/x/tools/cmd/goimports
+	cd $(TOOLS_MOD_DIR) && go install github.com/jcchavezs/porto/cmd/porto
+	cd $(TOOLS_MOD_DIR) && go install go.opentelemetry.io/build-tools/multimod
 
 .PHONY: otelcol
 otelcol:
@@ -310,6 +315,9 @@ gendependabot: $(eval SHELL:=/bin/bash)
 # The source directory for OTLP ProtoBufs.
 OPENTELEMETRY_PROTO_SRC_DIR=model/internal/opentelemetry-proto
 
+# The SHA matching the current version of the proto to use
+OPENTELEMETRY_PROTO_VERSION=v0.11.0
+
 # Find all .proto files.
 OPENTELEMETRY_PROTO_FILES := $(subst $(OPENTELEMETRY_PROTO_SRC_DIR)/,,$(wildcard $(OPENTELEMETRY_PROTO_SRC_DIR)/opentelemetry/proto/*/v1/*.proto $(OPENTELEMETRY_PROTO_SRC_DIR)/opentelemetry/proto/collector/*/v1/*.proto))
 
@@ -326,13 +334,18 @@ DOCKER_PROTOBUF ?= otel/build-protobuf:0.4.1
 PROTOC := docker run --rm -u ${shell id -u} -v${PWD}:${PWD} -w${PWD}/$(PROTO_INTERMEDIATE_DIR) ${DOCKER_PROTOBUF} --proto_path=${PWD}
 PROTO_INCLUDES := -I/usr/include/github.com/gogo/protobuf -I./
 
+# Cleanup temporary directory
+genproto-cleanup:
+	rm -Rf ${OPENTELEMETRY_PROTO_SRC_DIR}
+
 # Generate OTLP Protobuf Go files. This will place generated files in PROTO_TARGET_GEN_DIR.
-genproto:
-	git submodule update --init
-	# Call a sub-make to ensure OPENTELEMETRY_PROTO_FILES is populated after the submodule
-	# files are present.
+genproto: genproto-cleanup
+	mkdir -p ${OPENTELEMETRY_PROTO_SRC_DIR}
+	curl -sSL https://api.github.com/repos/open-telemetry/opentelemetry-proto/tarball/${OPENTELEMETRY_PROTO_VERSION} | tar xz --strip 1 -C ${OPENTELEMETRY_PROTO_SRC_DIR}
+	# Call a sub-make to ensure OPENTELEMETRY_PROTO_FILES is populated
 	$(MAKE) genproto_sub
 	$(MAKE) fmt
+	$(MAKE) genproto-cleanup
 
 genproto_sub:
 	@echo Generating code for the following files:
@@ -363,7 +376,7 @@ genproto_sub:
 # Generate structs, functions and tests for pdata package. Must be used after any changes
 # to proto and after running `make genproto`
 genpdata:
-	go run cmd/pdatagen/main.go
+	go run model/internal/cmd/pdatagen/main.go
 	$(MAKE) fmt
 
 # Generate semantic convention constants. Requires a clone of the opentelemetry-specification repo
@@ -422,3 +435,12 @@ endif
 .PHONY: apidiff-compare
 apidiff-compare:
 	@$(foreach pkg,$(ALL_PKGS),$(call exec-command,./internal/buildscripts/compare-apidiff.sh -p $(pkg)))
+
+.PHONY: multimod-verify
+multimod-verify: install-tools
+	@echo "Validating versions.yaml"
+	multimod verify
+
+.PHONY: multimod-prerelease
+multimod-prerelease: install-tools
+	multimod prerelease -v ./versions.yaml -m collector-base

Makefile.Common

@@ -38,6 +38,10 @@ lint:
 lint-unstable:
 	$(LINT) run --allow-parallel-runners --build-tags enable_unstable
 
+.PHONY: porto
+porto:
+	porto -w --include-internal  ./
+
 .PHONY: impi
 impi:
 	@$(IMPI) --local go.opentelemetry.io/collector --scheme stdThirdPartyLocal ./...