Skip to content

2.31.0.0-b252

@Deepti-yb Deepti-yb tagged this 12 Jul 14:54
Summary:
- Add support for exporting master logs to third-party sinks via the unified telemetry export API.
- Refactor so a new log/telemetry type touches only 2-3 files, not the whole task workflow:
  - `TelemetryConfig.section()/diff()/hasAnyConfig()` derive from it ---> new type = config model + one ExportType entry + its `OtelCollectorConfigGenerator` receiver.
  - `OtelCollectorUtil.getCurrentTelemetryConfig` resolves every section generically (table-first, per-section `userIntent` fallback, table-only types like master logs unchanged).
  - `ExportType.requiresDbRestart` drives the collector-only `NON_RESTART` fast path generically.
  - `MasterLogConfig/MasterLogLevel`, `UniverseServerLogsExporterConfig`, `MasterLogsTelemetrySpec` v2 schema, master filelog receiver (configurable min level + noise drop ratio), `ManageOtelCollector` `yb-master` systemd probe.

- Binary and systemD unit gets installed in both master and tserver nodes in case of dedicated master mode. Exports logs are from master node alone since tserver master log files are not populated

Test Plan:
```
$ cat otel-collector/config.yml
!!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat
exporters:
  datadog/master_logs_a1cf11bd-7443-4e96-9b28-94e820475eef: !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$DataDogExporter
    api: {key: <>, site: <>}
    retry_on_failure: {enabled: true, initial_interval: 30s, max_elapsed_time: 60m,
      max_interval: 10m}
extensions:
  file_storage/queue: !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$StorageExtension
    compaction: {directory: /mnt/d0/otel-collector/queue, on_rebound: true, on_start: true,
      rebound_needed_threshold_mib: 100, rebound_trigger_threshold_mib: 10}
    directory: /mnt/d0/otel-collector/queue
processors:
  attributes/master_logs_a1cf11bd-7443-4e96-9b28-94e820475eef: !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$AttributesProcessor
    actions:
    - {action: upsert, key: ddsource, value: yugabyte}
    - {action: upsert, key: service, value: yb-otel-collector}
    - {action: upsert, key: host, value: yb-admin-dkumar-1-n2}
    - {action: upsert, key: yugabyte.node_name, value: yb-admin-dkumar-1-n2}
    - {action: upsert, key: yugabyte.cloud, value: aws}
    - {action: upsert, key: yugabyte.universe_uuid, value: 85b62935-9139-4e25-9f6e-20639140e49d}
    - {action: upsert, key: yugabyte.node_type, value: PRIMARY}
    - {action: upsert, key: yugabyte.region, value: us-west-2}
    - {action: upsert, key: yugabyte.zone, value: us-west-2b}
    - {action: upsert, key: yugabyte.purpose, value: DATA_DOG_MASTER_LOG_EXPORT}
    - {action: upsert, from_attribute: log.file.name, key: yugabyte.log.file.name}
    - {action: delete, key: log.file.name}
    - {action: upsert, from_attribute: log_level, key: yugabyte.log_level}
    - {action: delete, key: log_level}
    - {action: upsert, from_attribute: statement_id, key: yugabyte.statement_id}
    - {action: delete, key: statement_id}
    - {action: upsert, from_attribute: substatement_id, key: yugabyte.substatement_id}
    - {action: delete, key: substatement_id}
    - {action: upsert, from_attribute: class, key: yugabyte.class}
    - {action: delete, key: class}
    - {action: upsert, from_attribute: command, key: yugabyte.command}
    - {action: delete, key: command}
    - {action: upsert, from_attribute: object_type, key: yugabyte.object_type}
    - {action: delete, key: object_type}
    - {action: upsert, from_attribute: object_name, key: yugabyte.object_name}
    - {action: delete, key: object_name}
    - {action: upsert, from_attribute: statement, key: yugabyte.statement}
    - {action: delete, key: statement}
    - {action: upsert, key: universe1, value: dkumar-cli-test}
  batch/master_logs_a1cf11bd-7443-4e96-9b28-94e820475eef: !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$BatchProcessor {
    send_batch_max_size: 1000, send_batch_size: 100, timeout: 10s}
  memory_limiter/master_logs_a1cf11bd-7443-4e96-9b28-94e820475eef: !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$MemoryLimiterProcessor {
    check_interval: 10s, limit_mib: 2048}
  transform/replace: !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$TransformProcessor
    log_statements:
    - context: log
      statements: ['replace_pattern(body, "^otho8Aut", "")']
receivers:
  filelog/master: !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$FileLogReceiver
    attributes: {yugabyte.server_log_type: master}
    exclude: [/home/yugabyte/master/logs/*.gz]
    include: [/home/yugabyte/master/logs/yb-master.*.INFO.*]
    multiline: {line_start_pattern: '^[IWEF]'}
    operators:
    - !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$FilterOperator {
      drop_ratio: 1.0, expr: body contains "Writing version edit" or body contains
        "Intent for transaction w/o metadata" or body contains "Colocated Tablet not
        found for table" or (body contains "READ_REPLICA" or (body contains "Skipping
        add replicas" and body contains "Currently remote bootstrapping")), type: filter}
    - !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$FilterOperator {
      drop_ratio: 0.99, expr: (body contains "Found uninitialized path" or body contains
        "Not allowing it to take more tablets" or body contains "Processing tablet"),
      type: filter}
    - !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$RegexOperator {
      if: 'body matches "^[IWEF]"', on_error: send, regex: '(?P<log_level>[IWEF])(?P<log_time>\d{4}
        \d{2}:\d{2}:\d{2}[.]\d{6})\s*(?P<thread_id>\d+) (?P<file_name>[^:]+):(?P<file_line>\d+)[]]
        (?P<message>.*)', type: regex_parser}
    - !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$RegexOperator {
      on_error: send, parse_from: 'attributes["log.file.name"]', regex: 'yb-master[.].*[.]log[.]INFO[.](?P<log_year>\d{4}).*',
      type: regex_parser}
    - !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$AddOperator {
      field: attributes.log_time_full, if: attributes.log_time != nil, type: add,
      value: EXPR(attributes.log_year + attributes.log_time)}
    - !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$TimeParserOperator {
      if: attributes.log_time != nil, layout: '%Y%m%d %H:%M:%S.%f', parse_from: attributes.log_time_full,
      type: time_parser}
    - !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$AddOperator {
      field: attributes.log_level, if: attributes.log_level == nil, type: add, value: I}
    - !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$SeverityParserOperator
      mapping: {info: I, warn: W, error: E, fatal: F}
      parse_from: attributes.log_level
      type: severity_parser
    - !!com.yugabyte.yw.common.audit.otel.OtelCollectorConfigFormat$MoveOperator {
      from: attributes.message, if: attributes.message != nil, to: body, type: move}
    poll_interval: 1s
    start_at: beginning
    storage: file_storage/queue
service:
  extensions: [file_storage/queue]
  pipelines:
    logs/master_logs_a1cf11bd-7443-4e96-9b28-94e820475eef:
      exporters: [datadog/master_logs_a1cf11bd-7443-4e96-9b28-94e820475eef]
      processors: [attributes/master_logs_a1cf11bd-7443-4e96-9b28-94e820475eef, batch/master_logs_a1cf11bd-7443-4e96-9b28-94e820475eef,
        memory_limiter/master_logs_a1cf11bd-7443-4e96-9b28-94e820475eef, transform/replace]
      receivers: [filelog/master]
  telemetry:
    logs:
      output_paths: [/home/yugabyte/otel-collector/logs/otel-collector.logs]
    metrics: {address: '0.0.0.0:8889'}
```

Reviewers: amalyshev, agupta, anabaria, abalakrishnan

Reviewed By: amalyshev

Subscribers: yugaware

Differential Revision: https://phorge.dev.yugabyte.com/D55016
Assets 2
Loading