Prune log streams

.github/workflows/publish.yml

@@ -10,6 +10,11 @@ jobs:
   publish:
     runs-on: ubuntu-latest
     environment: release
+    strategy:
+      matrix:
+        package:
+          - lcl-dynamodb-export
+          - lcl-prune-log-streams
     steps:
       - uses: actions/checkout@v3
       - uses: docker/login-action@v3
@@ -20,7 +25,7 @@ jobs:
         id: meta
         uses: docker/metadata-action@v5
         with:
-          images: bneijt/lcl-dynamodb-export
+          images: bneijt/${{ matrix.package }}
       - run: rustup update stable && rustup default stable
       - name: build
         run: |
@@ -31,8 +36,8 @@ jobs:
         uses: docker/build-push-action@v6
         with:
           file: Dockerfile
-          context: "target/lambda/lcl-dynamodb-export"
-          build-args: "PACKAGE=lcl-dynamodb-export"
+          context: "target/lambda/${{ matrix.package }}"
+          build-args: "PACKAGE=${{ matrix.package }}"
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

Cargo.toml

@@ -1,3 +1,3 @@
 [workspace]
 resolver = "2"
-members = ["lcl-dynamodb-export"]
+members = ["lcl-dynamodb-export", "lcl-prune-log-streams"]

build.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 set -e
-PACKAGES="dynamodb-export"
+PACKAGES="dynamodb-export prune-log-streams"
 
 for package in $PACKAGES; do
     echo "Building $package"

lcl-dynamodb-export/README.md

@@ -20,6 +20,10 @@ Read more about building your lambda function in [the Cargo Lambda documentation
 
 ## Testing
 
+```bash
+cargo lambda watch
+```
+
 ```bash
 cargo lambda invoke --data-example eventbridge-schedule
 ```

lcl-prune-log-streams/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "lcl-prune-log-streams"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+anyhow = "1.0.98"
+async-trait = "0.1.88"
+aws-config = "1.6.1"
+aws-sdk-cloudwatchevents = "1.67.0"
+aws-sdk-cloudwatchlogs = "1.67.0"
+aws-sdk-dynamodb = "1.71.2"
+aws_lambda_events = { version = "0.15.1", default-features = false, features = [
+    "cloudwatch_events",
+] }
+lambda_runtime = "0.13.0"
+serde = "1.0.219"
+tokio = { version = "1", features = ["macros"] }
+tracing-subscriber = "0.3.19"

lcl-prune-log-streams/README.md

@@ -0,0 +1,83 @@
+# lcl-prune-log-streams
+
+## Introduction
+
+This AWS Lambda function prunes (deletes) empty and expired CloudWatch log streams from specified log groups, based on a scheduled EventBridge trigger.
+
+It is inspired by the AWS blog post: [Delete empty CloudWatch log streams](https://aws.amazon.com/blogs/mt/delete-empty-cloudwatch-log-streams/).
+
+### What it does
+
+- For each log group in a comma-separated list, it:
+  - Keeps the N most recent log streams (default: 5, configurable).
+  - Deletes log streams that are both:
+    - Empty (no stored bytes).
+    - Expired (last event and ingestion time are older than the log group's retention period).
+
+## Configuration
+
+Set the following environment variables:
+
+- `LCL_LOG_GROUPS`: Comma-separated list of CloudWatch log group names to prune.
+- `LCL_KEEP_AT_LEAST` (optional): Minimum number of most recent log streams to always keep per log group (default: 5).
+
+Example:
+
+```
+LCL_LOG_GROUPS=/aws/lambda/my-func-prod,/aws/lambda/my-func-dev
+LCL_KEEP_AT_LEAST=10
+```
+
+## Building
+
+To build the project for production, run:
+
+```bash
+cargo lambda build --release
+```
+
+Remove the `--release` flag to build for development.
+
+Read more about building your lambda function in [the Cargo Lambda documentation](https://www.cargo-lambda.info/commands/build.html).
+
+## Testing
+
+You can test locally with:
+
+```bash
+cargo lambda watch
+```
+
+Or invoke with a sample EventBridge event:
+
+```bash
+cargo lambda invoke --data-example eventbridge-schedule
+```
+
+Example event payload:
+
+```json
+{
+  "version": "0",
+  "id": "dbc1c73a-c51d-0c0e-ca61-ab9278974c57",
+  "account": "1234567890",
+  "time": "2023-05-23T11:38:46Z",
+  "region": "us-east-1",
+  "detail-type": "Scheduled Event",
+  "source": "aws.events",
+  "resources": [],
+  "detail": {}
+}
+```
+
+## IAM Permissions
+
+The Lambda function requires the following AWS permissions:
+
+- `logs:DescribeLogGroups`
+- `logs:DescribeLogStreams`
+- `logs:DeleteLogStream`
+
+Grant these permissions to the Lambda execution role for the log groups you wish to prune.
+
+---