ref: remove redundant property from topic files (#299)

it's already specified in the filename, we can also remove the test that asserts they are the same
getsentry · Jun 6, 2024 · 0b0aa09 · 0b0aa09
1 parent a8e93af
commit 0b0aa09
Show file tree

Hide file tree

Showing 63 changed files with 9 additions and 68 deletions.
diff --git a/README.md b/README.md
@@ -24,18 +24,22 @@ Example messages must be stripped of **all** customer related data. This also in
 
 Each topic is a yaml file in the topics directory. This topic name is a "logical" topic name as many services in Sentry support overriding the default name to a different physical topic name if desired. Topic names must be unique in Sentry: the same name cannot be used for different types of data.
 
-The yaml file of a topic has 2 keys:
+The yaml file of a topic has the following keys:
 
-1. `topic`. This is the logical topic name. It must match the filename.
+1. `schemas`. Schemas is an array. The following should be provided for each schema:
 
-2. `schemas`. Schemas is an array. The following should be provided for each schema:
    - `version`: Incrementing integer. Should start at 1.
    - `compatibility_mode`: `none` or `backward`.
    - `type`: Can be either `json` or `msgpack`. In both cases we use
      jsonschema to define the message schema.
    - `resource`: Should match the file name in the `schemas` directory
    - `examples`: Should match the file names in the `examples` directory
 
+2. `topic_configuration_config`. Configuration used to create the topic
+3. `services`. Which Sentry services produce to and consume from the topic.
+4. `description`.
+5. `pipeline`.
+
 ## Using the schema (in Python)
 
 ```python

diff --git a/python/tests/test_valid_topic_data.py b/python/tests/test_valid_topic_data.py
@@ -12,7 +12,6 @@
 _TOPIC_SCHEMA = fastjsonschema.compile(
     {
         "properties": {
-            "topic": {"type": "string"},
             "description": {"type": "string"},
             "pipeline": {"type": "string"},
             "services": {
@@ -54,7 +53,7 @@
             },
         },
         "aditionalProperties": False,
-        "required": ["topic", "description", "services"],
+        "required": ["description", "services"],
         "definitions": {
             "Repo": {
                 "enum": [
@@ -91,8 +90,7 @@ def test_all_topics() -> None:
             _TOPIC_SCHEMA(topic_data)
 
             # Check valid topic name
-            topic_name = topic_data["topic"]
-            assert topic_name == filename.stem
+            topic_name = filename.stem
             assert valid_chars.match(topic_name)
             assert len(topic_name) <= 255
 

diff --git a/rust/lib.rs b/rust/lib.rs
@@ -56,7 +56,6 @@ struct TopicSchema {
 
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 struct TopicData {
-    topic: String,
     schemas: Vec<TopicSchema>,
 }
 

diff --git a/topics/buffered-segments-dlq.yaml b/topics/buffered-segments-dlq.yaml
@@ -1,4 +1,3 @@
-topic: buffered-segments-dlq
 description: DLQ for buffered-segments
 services:
   producers:

diff --git a/topics/buffered-segments.yaml b/topics/buffered-segments.yaml
@@ -1,4 +1,3 @@
-topic: buffered-segments
 description: List of buffered spans for a segment
 services:
   producers:

diff --git a/topics/event-replacements.yaml b/topics/event-replacements.yaml
@@ -1,4 +1,3 @@
-topic: event-replacements
 description: Error replacements
 services:
   producers:

diff --git a/topics/events-subscription-results.yaml b/topics/events-subscription-results.yaml
@@ -1,4 +1,3 @@
-topic: events-subscription-results
 pipeline: errors
 description: Events Subscription Results
 services:

diff --git a/topics/events.yaml b/topics/events.yaml
@@ -1,4 +1,3 @@
-topic: events
 pipeline: errors
 description: Errors data for Snuba
 services:

diff --git a/topics/generic-events.yaml b/topics/generic-events.yaml
@@ -1,4 +1,3 @@
-topic: generic-events
 pipeline: generic-events
 description: Issue platform
 services:

diff --git a/topics/generic-metrics-subscription-results.yaml b/topics/generic-metrics-subscription-results.yaml
@@ -1,4 +1,3 @@
-topic: generic-metrics-subscription-results
 pipeline: generic-metrics
 description: Generic Metrics Subscription Results
 services:

diff --git a/topics/group-attributes.yaml b/topics/group-attributes.yaml
@@ -1,4 +1,3 @@
-topic: group-attributes
 description: A snapshot of Group/Issue attributes replicated from sentry
 services:
   producers:

diff --git a/topics/ingest-attachments-dlq.yaml b/topics/ingest-attachments-dlq.yaml
@@ -1,4 +1,3 @@
-topic: ingest-attachments-dlq
 pipeline: attachments
 description: DLQ for ingest-attachments
 services:

diff --git a/topics/ingest-attachments.yaml b/topics/ingest-attachments.yaml
@@ -1,4 +1,3 @@
-topic: ingest-attachments
 pipeline: attachments
 description: Attachments data from Relay
 services:

diff --git a/topics/ingest-events-dlq.yaml b/topics/ingest-events-dlq.yaml
@@ -1,4 +1,3 @@
-topic: ingest-events-dlq
 pipeline: errors
 description: DLQ for ingest-events
 services:

diff --git a/topics/ingest-events.yaml b/topics/ingest-events.yaml
@@ -1,4 +1,3 @@
-topic: ingest-events
 pipeline: errors
 description: Errors data from Relay
 services:

diff --git a/topics/ingest-feedback-events-dlq.yaml b/topics/ingest-feedback-events-dlq.yaml
@@ -1,4 +1,3 @@
-topic: ingest-feedback-events-dlq
 pipeline: user-feedback
 description: DLQ for ingest-feedback-events
 services:

diff --git a/topics/ingest-feedback-events.yaml b/topics/ingest-feedback-events.yaml
@@ -1,4 +1,3 @@
-topic: ingest-feedback-events
 pipeline: user-feedback
 description: User feedback events from SDK
 services:

diff --git a/topics/ingest-generic-metrics-dlq.yaml b/topics/ingest-generic-metrics-dlq.yaml
@@ -1,4 +1,3 @@
-topic: ingest-generic-metrics-dlq
 pipeline: release-health
 description: DLQ for ingest-performance-metrics
 services:

diff --git a/topics/ingest-metrics-dlq.yaml b/topics/ingest-metrics-dlq.yaml
@@ -1,4 +1,3 @@
-topic: ingest-metrics-dlq
 pipeline: release-health
 description: DLQ for ingest-metrics
 services:

diff --git a/topics/ingest-metrics.yaml b/topics/ingest-metrics.yaml
@@ -1,4 +1,3 @@
-topic: ingest-metrics
 pipeline: release-health
 description: release health metrics (before indexer)
 services:

diff --git a/topics/ingest-monitors.yaml b/topics/ingest-monitors.yaml
@@ -1,4 +1,3 @@
-topic: ingest-monitors
 pipeline: monitors
 description: cron monitor check-in ingest
 services:

diff --git a/topics/ingest-performance-metrics.yaml b/topics/ingest-performance-metrics.yaml
@@ -1,4 +1,3 @@
-topic: ingest-performance-metrics
 pipeline: generic-metrics
 description: performance metrics and soon also other usecases for generic-metrics (before indexer)
 services:

diff --git a/topics/ingest-replay-events.yaml b/topics/ingest-replay-events.yaml
@@ -1,4 +1,3 @@
-topic: ingest-replay-events
 pipeline: replays
 description: Replay events data for Snuba
 services:

diff --git a/topics/ingest-replay-recordings.yaml b/topics/ingest-replay-recordings.yaml
@@ -1,4 +1,3 @@
-topic: ingest-replay-recordings
 pipeline: replays
 description: Replay Recording payloads
 services:

diff --git a/topics/ingest-transactions-dlq.yaml b/topics/ingest-transactions-dlq.yaml
@@ -1,4 +1,3 @@
-topic: ingest-transactions-dlq
 pipeline: transactions
 description: DLQ for ingest-transactions
 services:

diff --git a/topics/metrics-subscription-results.yaml b/topics/metrics-subscription-results.yaml
@@ -1,4 +1,3 @@
-topic: metrics-subscription-results
 pipeline: release-health
 description: Metrics Subscription Results
 services:

diff --git a/topics/monitors-clock-tasks.yaml b/topics/monitors-clock-tasks.yaml
@@ -1,4 +1,3 @@
-topic: monitors-clock-tasks
 pipeline: monitors
 description: cron monitor clock based tasks
 services:

diff --git a/topics/monitors-clock-tick.yaml b/topics/monitors-clock-tick.yaml
@@ -1,4 +1,3 @@
-topic: monitors-clock-tick
 pipeline: monitors
 description: cron monitor clock ticks
 services:

diff --git a/topics/outcomes-billing.yaml b/topics/outcomes-billing.yaml
@@ -1,4 +1,3 @@
-topic: outcomes-billing
 pipeline: outcomes
 description: Outcomes for events sent to sentry
 services:

diff --git a/topics/outcomes.yaml b/topics/outcomes.yaml
@@ -1,4 +1,3 @@
-topic: outcomes
 pipeline: outcomes
 description: Outcomes for events sent to sentry
 services:

diff --git a/topics/processed-profiles.yaml b/topics/processed-profiles.yaml
@@ -1,4 +1,3 @@
-topic: processed-profiles
 pipeline: profiles
 description: Profile metadata extracted from profiles for Snuba
 services:

diff --git a/topics/profiles-call-tree.yaml b/topics/profiles-call-tree.yaml
@@ -1,4 +1,3 @@
-topic: profiles-call-tree
 pipeline: profiles
 description: Function data extracted from profiles for Snuba
 services:

diff --git a/topics/scheduled-subscriptions-events.yaml b/topics/scheduled-subscriptions-events.yaml
@@ -1,4 +1,3 @@
-topic: scheduled-subscriptions-events
 pipeline: errors
 description: Scheduled subscriptions for errors pipeline
 services:

diff --git a/topics/scheduled-subscriptions-generic-metrics-counters.yaml b/topics/scheduled-subscriptions-generic-metrics-counters.yaml
@@ -1,4 +1,3 @@
-topic: scheduled-subscriptions-generic-metrics-counters
 pipeline: generic-metrics
 description: Scheduled subscriptions
 services:

diff --git a/topics/scheduled-subscriptions-generic-metrics-distributions.yaml b/topics/scheduled-subscriptions-generic-metrics-distributions.yaml
@@ -1,4 +1,3 @@
-topic: scheduled-subscriptions-generic-metrics-distributions
 pipeline: generic-metrics
 description: Scheduled subscriptions
 services:

diff --git a/topics/scheduled-subscriptions-generic-metrics-gauges.yaml b/topics/scheduled-subscriptions-generic-metrics-gauges.yaml
@@ -1,4 +1,3 @@
-topic: scheduled-subscriptions-generic-metrics-gauges
 pipeline: generic-metrics
 description: Scheduled subscriptions
 services:

diff --git a/topics/scheduled-subscriptions-generic-metrics-sets.yaml b/topics/scheduled-subscriptions-generic-metrics-sets.yaml
@@ -1,4 +1,3 @@
-topic: scheduled-subscriptions-generic-metrics-sets
 pipeline: generic-metrics
 description: Scheduled subscriptions
 services:

diff --git a/topics/scheduled-subscriptions-metrics.yaml b/topics/scheduled-subscriptions-metrics.yaml
@@ -1,4 +1,3 @@
-topic: scheduled-subscriptions-metrics
 pipeline: release-health
 description: Scheduled subscriptions
 services:

diff --git a/topics/scheduled-subscriptions-transactions.yaml b/topics/scheduled-subscriptions-transactions.yaml
@@ -1,4 +1,3 @@
-topic: scheduled-subscriptions-transactions
 pipeline: transactions
 description: Scheduled subscriptions
 services:

diff --git a/topics/shared-resources-usage.yaml b/topics/shared-resources-usage.yaml
@@ -1,4 +1,3 @@
-topic: shared-resources-usage
 description: CoGS data extracted from various resources
 services:
   producers:

diff --git a/topics/snuba-commit-log.yaml b/topics/snuba-commit-log.yaml
@@ -1,4 +1,3 @@
-topic: snuba-commit-log
 pipeline: errors
 description: Commit log topic for errors pipeline
 services:

diff --git a/topics/snuba-dead-letter-generic-events.yaml b/topics/snuba-dead-letter-generic-events.yaml
@@ -1,4 +1,3 @@
-topic: snuba-dead-letter-generic-events
 pipeline: generic-events
 description: DLQ for generic-events
 services:

diff --git a/topics/snuba-dead-letter-generic-metrics.yaml b/topics/snuba-dead-letter-generic-metrics.yaml
@@ -1,4 +1,3 @@
-topic: snuba-dead-letter-generic-metrics
 pipeline: errors
 description: DLQ for snuba-generic-metrics
 services:

diff --git a/topics/snuba-dead-letter-group-attributes.yaml b/topics/snuba-dead-letter-group-attributes.yaml
@@ -1,4 +1,3 @@
-topic: snuba-dead-letter-group-attributes
 pipeline: errors
 description: DLQ for group-attributes
 services:

diff --git a/topics/snuba-dead-letter-metrics.yaml b/topics/snuba-dead-letter-metrics.yaml
@@ -1,4 +1,3 @@
-topic: snuba-dead-letter-metrics
 pipeline: errors
 description: DLQ for snuba-metrics
 services:

diff --git a/topics/snuba-dead-letter-querylog.yaml b/topics/snuba-dead-letter-querylog.yaml
@@ -1,4 +1,3 @@
-topic: snuba-dead-letter-querylog
 pipeline: errors
 description: DLQ for snuba-queries
 services:

diff --git a/topics/snuba-dead-letter-replays.yaml b/topics/snuba-dead-letter-replays.yaml
@@ -1,4 +1,3 @@
-topic: snuba-dead-letter-replays
 pipeline: errors
 description: DLQ for ingest-replay-events
 services:

diff --git a/topics/snuba-generic-events-commit-log.yaml b/topics/snuba-generic-events-commit-log.yaml
@@ -1,4 +1,3 @@
-topic: snuba-generic-events-commit-log
 pipeline: generic-events
 description: Commit log topic for generic events pipeline
 services:

diff --git a/topics/snuba-generic-metrics-counters-commit-log.yaml b/topics/snuba-generic-metrics-counters-commit-log.yaml
@@ -1,4 +1,3 @@
-topic: snuba-generic-metrics-counters-commit-log
 pipeline: generic-metrics
 description: Commit log topic for generic metrics counters pipeline
 services:

diff --git a/topics/snuba-generic-metrics-distributions-commit-log.yaml b/topics/snuba-generic-metrics-distributions-commit-log.yaml
@@ -1,4 +1,3 @@
-topic: snuba-generic-metrics-distributions-commit-log
 pipeline: generic-metrics
 description: Commit log topic for generic metrics distributions pipeline
 services:

diff --git a/topics/snuba-generic-metrics-gauges-commit-log.yaml b/topics/snuba-generic-metrics-gauges-commit-log.yaml
@@ -1,4 +1,3 @@
-topic: snuba-generic-metrics-gauges-commit-log
 pipeline: generic-metrics
 description: Commit log topic for generic metrics gauges pipeline
 services:

diff --git a/topics/snuba-generic-metrics-sets-commit-log.yaml b/topics/snuba-generic-metrics-sets-commit-log.yaml
@@ -1,4 +1,3 @@
-topic: snuba-generic-metrics-sets-commit-log
 pipeline: generic-metrics
 description: Commit log topic for generic metrics sets pipeline
 services:

diff --git a/topics/snuba-generic-metrics.yaml b/topics/snuba-generic-metrics.yaml
@@ -1,4 +1,3 @@
-topic: snuba-generic-metrics
 pipeline: generic-metrics
 description: Generic metrics for Snuba
 services:

diff --git a/topics/snuba-metrics-commit-log.yaml b/topics/snuba-metrics-commit-log.yaml
@@ -1,4 +1,3 @@
-topic: snuba-metrics-commit-log
 pipeline: release-health
 description: Commit log topic for release health pipeline
 services:

diff --git a/topics/snuba-metrics-summaries.yaml b/topics/snuba-metrics-summaries.yaml
@@ -1,4 +1,3 @@
-topic: snuba-metrics-summaries
 description: Metrics summaries for spans
 services:
   producers:

diff --git a/topics/snuba-metrics.yaml b/topics/snuba-metrics.yaml
@@ -1,4 +1,3 @@
-topic: snuba-metrics
 pipeline: release-health
 description: Release health metrics
 services:

diff --git a/topics/snuba-profile-chunks.yaml b/topics/snuba-profile-chunks.yaml
@@ -1,4 +1,3 @@
-topic: snuba-profile-chunks
 description: Profile chunks for continuous profiling
 services:
   producers:

diff --git a/topics/snuba-queries.yaml b/topics/snuba-queries.yaml
@@ -1,4 +1,3 @@
-topic: snuba-queries
 description: Snuba querylog
 services:
   consumers:

diff --git a/topics/snuba-spans.yaml b/topics/snuba-spans.yaml
@@ -1,4 +1,3 @@
-topic: snuba-spans
 description: Snuba spans only
 services:
   producers:

diff --git a/topics/snuba-transactions-commit-log.yaml b/topics/snuba-transactions-commit-log.yaml
@@ -1,4 +1,3 @@
-topic: snuba-transactions-commit-log
 pipeline: transactions
 description: Commit log topic for transactions pipeline
 services:

diff --git a/topics/transactions-subscription-results.yaml b/topics/transactions-subscription-results.yaml
@@ -1,4 +1,3 @@
-topic: transactions-subscription-results
 pipeline: transactions
 description: Transactions Subscription Results
 services:

diff --git a/topics/transactions.yaml b/topics/transactions.yaml
@@ -1,4 +1,3 @@
-topic: transactions
 pipeline: transactions
 description: Transactions data for Snuba
 services:

diff --git a/topics/uptime-results.yaml b/topics/uptime-results.yaml
@@ -1,4 +1,3 @@
-topic: uptime-results
 pipeline: uptime
 description: uptime check results
 services: