From bc38f8f0d92dddcb5d14c284851e4a271c86bdae Mon Sep 17 00:00:00 2001 From: Bram Neijt Date: Fri, 25 Jul 2025 16:59:14 +0200 Subject: [PATCH] Add prune log streams --- .github/workflows/publish.yml | 11 +- Cargo.lock | 63 +++++++++ Cargo.toml | 2 +- build.sh | 2 +- lcl-dynamodb-export/README.md | 4 + lcl-prune-log-streams/Cargo.toml | 19 +++ lcl-prune-log-streams/README.md | 83 ++++++++++++ lcl-prune-log-streams/src/event_handler.rs | 142 +++++++++++++++++++++ lcl-prune-log-streams/src/main.rs | 11 ++ 9 files changed, 332 insertions(+), 5 deletions(-) create mode 100644 lcl-prune-log-streams/Cargo.toml create mode 100644 lcl-prune-log-streams/README.md create mode 100644 lcl-prune-log-streams/src/event_handler.rs create mode 100644 lcl-prune-log-streams/src/main.rs diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b4d4c77..7b10f24 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -10,6 +10,11 @@ jobs: publish: runs-on: ubuntu-latest environment: release + strategy: + matrix: + package: + - lcl-dynamodb-export + - lcl-prune-log-streams steps: - uses: actions/checkout@v3 - uses: docker/login-action@v3 @@ -20,7 +25,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: bneijt/lcl-dynamodb-export + images: bneijt/${{ matrix.package }} - run: rustup update stable && rustup default stable - name: build run: | @@ -31,8 +36,8 @@ jobs: uses: docker/build-push-action@v6 with: file: Dockerfile - context: "target/lambda/lcl-dynamodb-export" - build-args: "PACKAGE=lcl-dynamodb-export" + context: "target/lambda/${{ matrix.package }}" + build-args: "PACKAGE=${{ matrix.package }}" push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/Cargo.lock b/Cargo.lock index 71ab67a..3020aba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -166,6 +166,7 @@ dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async", + "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -204,6 +205,29 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-sdk-cloudwatchlogs" +version = "1.87.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce7c74b0c173dc6c382f745b0b3563be7e4a905c70f0b90a2f95d0fd63250b42" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", +] + [[package]] name = "aws-sdk-dynamodb" version = "1.74.0" @@ -304,6 +328,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3503af839bd8751d0bdc5a46b9cac93a003a353e635b0c12cf2376b5b53e41ea" dependencies = [ "aws-credential-types", + "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-runtime-api", "aws-smithy-types", @@ -330,12 +355,24 @@ dependencies = [ "tokio", ] +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c45d3dddac16c5c59d553ece225a88870cf81b7b813c9cc17b78cf4685eac7a" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + [[package]] name = "aws-smithy-http" version = "0.62.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99335bec6cdc50a346fda1437f9fefe33abf8c99060739a546a16457f2862ca9" dependencies = [ + "aws-smithy-eventstream", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", @@ -701,6 +738,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -1428,6 +1474,23 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "lcl-prune-log-streams" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "aws-config", + "aws-sdk-cloudwatchevents", + "aws-sdk-cloudwatchlogs", + "aws-sdk-dynamodb", + "aws_lambda_events", + "lambda_runtime", + "serde", + "tokio", + "tracing-subscriber", +] + [[package]] name = "libc" version = "0.2.172" diff --git a/Cargo.toml b/Cargo.toml index 5d33f0e..f6d8b94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,3 @@ [workspace] resolver = "2" -members = ["lcl-dynamodb-export"] +members = ["lcl-dynamodb-export", "lcl-prune-log-streams"] diff --git a/build.sh b/build.sh index a8656b5..8e56b80 100755 --- a/build.sh +++ b/build.sh @@ -1,6 +1,6 @@ #!/bin/bash set -e -PACKAGES="dynamodb-export" +PACKAGES="dynamodb-export prune-log-streams" for package in $PACKAGES; do echo "Building $package" diff --git a/lcl-dynamodb-export/README.md b/lcl-dynamodb-export/README.md index a4c21db..4b412ac 100644 --- a/lcl-dynamodb-export/README.md +++ b/lcl-dynamodb-export/README.md @@ -20,6 +20,10 @@ Read more about building your lambda function in [the Cargo Lambda documentation ## Testing +```bash +cargo lambda watch +``` + ```bash cargo lambda invoke --data-example eventbridge-schedule ``` diff --git a/lcl-prune-log-streams/Cargo.toml b/lcl-prune-log-streams/Cargo.toml new file mode 100644 index 0000000..8b424c1 --- /dev/null +++ b/lcl-prune-log-streams/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "lcl-prune-log-streams" +version = "0.1.0" +edition = "2024" + +[dependencies] +anyhow = "1.0.98" +async-trait = "0.1.88" +aws-config = "1.6.1" +aws-sdk-cloudwatchevents = "1.67.0" +aws-sdk-cloudwatchlogs = "1.67.0" +aws-sdk-dynamodb = "1.71.2" +aws_lambda_events = { version = "0.15.1", default-features = false, features = [ + "cloudwatch_events", +] } +lambda_runtime = "0.13.0" +serde = "1.0.219" +tokio = { version = "1", features = ["macros"] } +tracing-subscriber = "0.3.19" diff --git a/lcl-prune-log-streams/README.md b/lcl-prune-log-streams/README.md new file mode 100644 index 0000000..3d21912 --- /dev/null +++ b/lcl-prune-log-streams/README.md @@ -0,0 +1,83 @@ +# lcl-prune-log-streams + +## Introduction + +This AWS Lambda function prunes (deletes) empty and expired CloudWatch log streams from specified log groups, based on a scheduled EventBridge trigger. + +It is inspired by the AWS blog post: [Delete empty CloudWatch log streams](https://aws.amazon.com/blogs/mt/delete-empty-cloudwatch-log-streams/). + +### What it does + +- For each log group in a comma-separated list, it: + - Keeps the N most recent log streams (default: 5, configurable). + - Deletes log streams that are both: + - Empty (no stored bytes). + - Expired (last event and ingestion time are older than the log group's retention period). + +## Configuration + +Set the following environment variables: + +- `LCL_LOG_GROUPS`: Comma-separated list of CloudWatch log group names to prune. +- `LCL_KEEP_AT_LEAST` (optional): Minimum number of most recent log streams to always keep per log group (default: 5). + +Example: + +``` +LCL_LOG_GROUPS=/aws/lambda/my-func-prod,/aws/lambda/my-func-dev +LCL_KEEP_AT_LEAST=10 +``` + +## Building + +To build the project for production, run: + +```bash +cargo lambda build --release +``` + +Remove the `--release` flag to build for development. + +Read more about building your lambda function in [the Cargo Lambda documentation](https://www.cargo-lambda.info/commands/build.html). + +## Testing + +You can test locally with: + +```bash +cargo lambda watch +``` + +Or invoke with a sample EventBridge event: + +```bash +cargo lambda invoke --data-example eventbridge-schedule +``` + +Example event payload: + +```json +{ + "version": "0", + "id": "dbc1c73a-c51d-0c0e-ca61-ab9278974c57", + "account": "1234567890", + "time": "2023-05-23T11:38:46Z", + "region": "us-east-1", + "detail-type": "Scheduled Event", + "source": "aws.events", + "resources": [], + "detail": {} +} +``` + +## IAM Permissions + +The Lambda function requires the following AWS permissions: + +- `logs:DescribeLogGroups` +- `logs:DescribeLogStreams` +- `logs:DeleteLogStream` + +Grant these permissions to the Lambda execution role for the log groups you wish to prune. + +--- \ No newline at end of file diff --git a/lcl-prune-log-streams/src/event_handler.rs b/lcl-prune-log-streams/src/event_handler.rs new file mode 100644 index 0000000..440ba44 --- /dev/null +++ b/lcl-prune-log-streams/src/event_handler.rs @@ -0,0 +1,142 @@ +use anyhow::{Context, Result}; +use aws_config::BehaviorVersion; +use aws_lambda_events::event::cloudwatch_events::CloudWatchEvent; +use aws_sdk_cloudwatchlogs::Client as LogsClient; +use lambda_runtime::{Error, LambdaEvent, tracing}; + +use std::env; +use std::time::{SystemTime, UNIX_EPOCH}; + +pub(crate) async fn function_handler(_: LambdaEvent) -> Result { + tracing::info!("start"); + tracing::info!("start prune log streams"); + + let config = aws_config::load_defaults(BehaviorVersion::latest()).await; + let logs_client = LogsClient::new(&config); + + let log_groups_env = require_env("LCL_LOG_GROUPS")?; + let log_groups: Vec<&str> = log_groups_env + .split(',') + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .collect(); + + // Read keep-at-least value from env, default to 5 + let keep_at_least: usize = match env::var("LCL_KEEP_AT_LEAST") { + Ok(val) => val.parse().unwrap_or(5), + Err(_) => 5, + }; + + for log_group in log_groups { + tracing::info!("Processing log group: {}", log_group); + + // Get log group details (to fetch retention period) + let describe_resp = logs_client + .describe_log_groups() + .log_group_name_prefix(log_group) + .send() + .await + .context("Failed to describe log groups")?; + + let group = describe_resp.log_groups.as_ref().and_then(|groups| { + groups + .iter() + .find(|g| g.log_group_name.as_deref() == Some(log_group)) + }); + + let retention_days = match group.and_then(|g| g.retention_in_days) { + Some(days) => days, + None => { + tracing::warn!( + "No retention period set for log group {}, skipping", + log_group + ); + continue; + } + }; + + let retention_secs = retention_days as u64 * 24 * 60 * 60; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + + // Gather all log streams for this group, sorted by last event time descending + let mut all_streams = Vec::new(); + let mut next_token = None; + loop { + let mut req = logs_client + .describe_log_streams() + .log_group_name(log_group) + .order_by(aws_sdk_cloudwatchlogs::types::OrderBy::LastEventTime) + .descending(true) + .limit(50); + + if let Some(token) = &next_token { + req = req.next_token(token); + } + + let streams_resp = req.send().await.context("Failed to describe log streams")?; + let mut streams = streams_resp.log_streams.unwrap_or_default(); + all_streams.append(&mut streams); + + next_token = streams_resp.next_token; + if next_token.is_none() { + break; + } + } + + // Sort streams by last event time descending (most recent first) + all_streams.sort_by(|a, b| { + let a_time = a.last_event_timestamp.unwrap_or(0); + let b_time = b.last_event_timestamp.unwrap_or(0); + b_time.cmp(&a_time) + }); + + // Only consider deleting streams after the N most recent ones + for (idx, stream) in all_streams.iter().enumerate() { + let stream_name = match &stream.log_stream_name { + Some(name) => name, + None => continue, + }; + + if idx < keep_at_least { + tracing::info!("Keeping recent stream '{}'", stream_name); + continue; + } + + let last_event_ts = stream.last_event_timestamp.unwrap_or(0) as u64 / 1000; + let last_ingestion_ts = stream.last_ingestion_time.unwrap_or(0) as u64 / 1000; + + let is_expired = + now > last_event_ts + retention_secs && now > last_ingestion_ts + retention_secs; + + // Check if stream is empty (no stored bytes or no events) + let stored_bytes = stream.stored_bytes.unwrap_or(0); + let is_empty = stored_bytes == 0; + + if is_expired && is_empty { + tracing::info!( + "Deleting expired and empty log stream: {} in group {}", + stream_name, + log_group + ); + let _ = logs_client + .delete_log_stream() + .log_group_name(log_group) + .log_stream_name(stream_name) + .send() + .await; + } + } + } + + Ok("Pruned log streams".to_string()) +} + +fn require_env(environment_variable_name: &str) -> anyhow::Result { + env::var(environment_variable_name).context(format!( + "Missing required environment variable '{}'", + environment_variable_name + )) +} diff --git a/lcl-prune-log-streams/src/main.rs b/lcl-prune-log-streams/src/main.rs new file mode 100644 index 0000000..4548bd3 --- /dev/null +++ b/lcl-prune-log-streams/src/main.rs @@ -0,0 +1,11 @@ +use lambda_runtime::{Error, run, service_fn}; +use tracing_subscriber; + +mod event_handler; +use event_handler::function_handler; + +#[tokio::main] +async fn main() -> Result<(), Error> { + tracing_subscriber::fmt().json().init(); + run(service_fn(function_handler)).await +}