enhancement(ci): refactor logic for int test file path changes detect…

…ion (vectordotdev#17725)

- The intent is to better be able to manage the file paths per
integration, as they are contained in a specific place for that
integration rather in one massive list.

.github/workflows/integration.yml

@@ -38,6 +38,8 @@ jobs:
     with:
       base_ref: ${{ github.event.pull_request.base.ref }}
       head_ref: ${{ github.event.pull_request.head.ref  }}
+      source: false
+      int_tests: true
     secrets: inherit
 
   # Calls the Integration Test workflow for each integration that was detected to have files changed that impact it.

scripts/integration/amqp/test.yaml

@@ -5,3 +5,13 @@ test_filter: '::amqp::'
 
 matrix:
   version: ['3.8']
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/amqp.rs"
+- "src/internal_events/amqp.rs"
+- "src/sinks/amqp/**"
+- "src/sources/amqp.rs"
+- "src/sources/util/**"
+- "src/sinks/util/**"

scripts/integration/appsignal/test.yaml

@@ -9,3 +9,9 @@ runner:
 
 matrix:
   version: [latest]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/sinks/appsignal/**"
+- "src/sinks/util/**"

scripts/integration/aws/test.yaml

@@ -17,3 +17,15 @@ env:
 
 matrix:
   version: [latest]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/aws_**"
+- "src/internal_events/aws_**"
+- "src/sources/aws_**"
+- "src/sources/util/**"
+- "src/sinks/aws_**"
+- "src/sinks/util/**"
+- "src/transforms/aws_**"
+- "scripts/integration/aws/**"

scripts/integration/axiom/test.yaml

@@ -12,3 +12,10 @@ runner:
 
 matrix:
   postgres: [13-alpine]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/sinks/axiom.rs"
+- "src/sinks/util/**"
+- "scripts/integration/axiom/**"

scripts/integration/azure/test.yaml

@@ -10,3 +10,10 @@ env:
 
 matrix:
   version: [3.14.0]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/sinks/azure_**"
+- "src/sinks/util/**"
+- "scripts/integration/azure/**"

scripts/integration/clickhouse/test.yaml

@@ -8,3 +8,10 @@ env:
 
 matrix:
   version: ['19']
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/sinks/clickhouse/**"
+- "src/sinks/util/**"
+- "scripts/integration/clickhouse/**"

scripts/integration/databend/test.yaml

@@ -11,3 +11,10 @@ runner:
 
 matrix:
   version: ['latest']
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/sinks/databend/**"
+- "src/sinks/util/**"
+- "scripts/integration/databend/**"

scripts/integration/datadog-agent/test.yaml

@@ -12,3 +12,11 @@ env:
 
 matrix:
   version: ['7']
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/common/datadog.rs"
+- "src/internal_events/datadog_*"
+- "src/sources/datadog_agent/**"
+- "scripts/integration/datadog-agent/**"

scripts/integration/datadog-logs/test.yaml

@@ -9,3 +9,12 @@ runner:
 
 matrix:
   version: [latest]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/common/datadog.rs"
+- "src/internal_events/datadog_*"
+- "src/sinks/datadog/logs/**"
+- "src/sinks/util/**"
+- "scripts/integration/datadog-logs/**"

scripts/integration/datadog-metrics/test.yaml

@@ -9,3 +9,12 @@ runner:
 
 matrix:
   version: [latest]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/common/datadog.rs"
+- "src/internal_events/datadog_*"
+- "src/sinks/datadog/metrics/**"
+- "src/sinks/util/**"
+- "scripts/integration/datadog-metrics/**"

scripts/integration/datadog-traces/test.yaml

@@ -13,3 +13,12 @@ env:
 
 matrix:
   version: [latest]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/common/datadog.rs"
+- "src/internal_events/datadog_*"
+- "src/sinks/datadog/**"
+- "src/sinks/util/**"
+- "scripts/integration/datadog-traces/**"

scripts/integration/dnstap/test.yaml

@@ -12,3 +12,11 @@ runner:
 
 matrix:
   version: ['latest']
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/internal_events/dnstap.rs"
+- "src/sources/dnstap/**"
+- "src/sources/util/**"
+- "scripts/integration/dnstap/**"

scripts/integration/docker-logs/test.yaml

@@ -8,3 +8,12 @@ runner:
 
 matrix:
   default: ["default"]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/docker.rs"
+- "src/internal_events/docker_logs.rs"
+- "src/sources/docker_logs/**"
+- "src/sources/util/**"
+- "scripts/integration/docker-logs/**"

scripts/integration/elasticsearch/test.yaml

@@ -12,3 +12,10 @@ env:
 
 matrix:
   version: [7.13.1]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/sinks/elasticsearch/**"
+- "src/sinks/util/**"
+- "scripts/integration/elasticsearch/**"

scripts/integration/eventstoredb/test.yaml

@@ -5,3 +5,11 @@ test_filter: '::eventstoredb_metrics::'
 
 matrix:
   version: [latest]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/internal_events/eventstoredb_metrics.rs"
+- "src/sources/eventstoredb_metrics/**"
+- "src/sources/util/**"
+- "scripts/integration/eventstoredb/**"

scripts/integration/fluent/test.yaml

@@ -10,3 +10,11 @@ runner:
 
 matrix:
   default: ["default"]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/internal_events/fluent.rs"
+- "src/sources/fluent/**"
+- "src/sources/util/**"
+- "scripts/integration/fluent/**"

scripts/integration/gcp/test.yaml

@@ -8,3 +8,14 @@ env:
 
 matrix:
   version: [latest]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/internal_events/gcp_pubsub.rs"
+- "src/sources/gcp_pubsub.rs"
+- "src/sources/util/**"
+- "src/sinks/gcp/**"
+- "src/sinks/util/**"
+- "src/gcp.rs"
+- "scripts/integration/gcp/**"

scripts/integration/http-client/test.yaml

@@ -10,3 +10,10 @@ env:
 
 matrix:
   version: ["v0.34.1"]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/sources/http_client/**"
+- "src/sources/util/**"
+- "scripts/integration/http-client/**"

scripts/integration/humio/test.yaml

@@ -9,3 +9,10 @@ runner:
 
 matrix:
   version: [1.13.1]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/sinks/humio/**"
+- "src/sinks/util/**"
+- "scripts/integration/humio/**"

scripts/integration/influxdb/test.yaml

@@ -10,3 +10,11 @@ env:
 
 matrix:
   version: ['1.8']
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/internal_events/influxdb.rs"
+- "src/sinks/influxdb/**"
+- "src/sinks/util/**"
+- "scripts/integration/influxdb/**"

scripts/integration/kafka/test.yaml

@@ -8,3 +8,14 @@ env:
 
 matrix:
   version: [latest]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/internal_events/kafka.rs"
+- "src/sinks/kafka/**"
+- "src/sinks/util/**"
+- "src/sources/kafka.rs"
+- "src/sources/util/**"
+- "src/kafka.rs"
+- "scripts/integration/kafka/**"

scripts/integration/logstash/test.yaml

@@ -9,3 +9,10 @@ env:
 
 matrix:
   version: [7.12.1]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/sources/logstash.rs"
+- "src/sources/util/**"
+- "scripts/integration/logstash/**"

scripts/integration/loki/test.yaml

@@ -8,3 +8,11 @@ env:
 
 matrix:
   version: [2.4.1]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/internal_events/loki.rs"
+- "src/sinks/loki/**"
+- "src/sinks/util/**"
+- "scripts/integration/loki/**"

scripts/integration/mongodb/test.yaml

@@ -9,3 +9,11 @@ env:
 
 matrix:
   version: [4.2.10]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/internal_events/mongodb_metrics.rs"
+- "src/sources/mongodb_metrics/**"
+- "src/sources/util/**"
+- "scripts/integration/mongodb/**"

scripts/integration/nats/test.yaml

@@ -14,3 +14,14 @@ env:
 
 matrix:
   version: [latest]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/internal_events/nats.rs"
+- "src/sources/nats.rs"
+- "src/sources/util/**"
+- "src/sinks/nats.rs"
+- "src/sinks/util/**"
+- "src/nats.rs"
+- "scripts/integration/nats/**"

scripts/integration/nginx/test.yaml

@@ -11,3 +11,11 @@ runner:
 
 matrix:
   version: [1.19.4]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/internal_events/nginx_metrics.rs"
+- "src/sources/nginx_metrics/**"
+- "src/sources/util/**"
+- "scripts/integration/nginx/**"

scripts/integration/opentelemetry/test.yaml

@@ -10,3 +10,10 @@ runner:
 
 matrix:
   version: [0.56.0]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/sources/opentelemetry/**"
+- "src/sources/util/**"
+- "scripts/integration/opentelemetry/**"

scripts/integration/postgres/test.yaml

@@ -13,3 +13,11 @@ runner:
 
 matrix:
   version: ['13.1']
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/internal_events/postgresql_metrics.rs"
+- "src/sources/postgresql_metrics.rs"
+- "src/sources/util/**"
+- "scripts/integration/postgres/**"

scripts/integration/prometheus/test.yaml

@@ -9,3 +9,13 @@ env:
 matrix:
   prometheus: ['v2.33.4']
   influxdb: ['1.8']
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/internal_events/prometheus.rs"
+- "src/sources/prometheus/**"
+- "src/sources/util/**"
+- "src/sinks/prometheus/**"
+- "src/sinks/util/**"
+- "scripts/integration/prometheus/**"

scripts/integration/pulsar/test.yaml

@@ -8,3 +8,11 @@ env:
 
 matrix:
   version: [latest]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/internal_events/pulsar.rs"
+- "src/sinks/pulsar/**"
+- "src/sinks/util/**"
+- "scripts/integration/pulsar/**"

scripts/integration/redis/test.yaml

@@ -8,3 +8,13 @@ env:
 
 matrix:
   version: [6-alpine]
+
+# changes to these files/paths will invoke the integration test in CI
+# expressions are evaluated using https://github.com/micromatch/picomatch
+paths:
+- "src/internal_events/redis.rs"
+- "src/sources/redis/**"
+- "src/sources/util/**"
+- "src/sinks/redis.rs"
+- "src/sinks/util/**"
+- "scripts/integration/redis/**"