Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
63 commits
Select commit Hold shift + click to select a range
03d6e11
Add README.md
cbartz Sep 16, 2025
8757398
Add go.mod
cbartz Sep 16, 2025
394f7cb
add queue package
cbartz Sep 16, 2025
3e49300
add webhook package
cbartz Sep 16, 2025
72f4cfd
add webhook-gateway initial code
cbartz Sep 16, 2025
b391079
checkin github actions config
cbartz Sep 16, 2025
b25ae1a
fix data races
cbartz Sep 16, 2025
2dd96f7
cleanup
cbartz Sep 16, 2025
e1e538c
checkin go.sum file
cbartz Sep 16, 2025
dd05d4a
update copyright
cbartz Sep 16, 2025
14e19e0
use self-hosted runner for tests
cbartz Sep 16, 2025
a0422f8
checkin CODEOWNERS
cbartz Sep 16, 2025
da0e0cc
use env vars in main
cbartz Sep 16, 2025
0dfb29e
use single label approach in ci
cbartz Sep 16, 2025
085715b
use struct instead of init functions
cbartz Sep 17, 2025
8c8b722
use internal server error
cbartz Sep 17, 2025
1e24f01
use hmac.Equal
cbartz Sep 17, 2025
79ac5a2
simplify interface : Queue -> Producer
cbartz Sep 17, 2025
289f52f
introduce AmqpProducer
cbartz Sep 17, 2025
76f472f
pass context to push
cbartz Sep 17, 2025
9407315
use fmt.Errorf
cbartz Sep 17, 2025
ee7e1b2
update action versions
cbartz Sep 17, 2025
098f7cc
add first version of integration test
cbartz Sep 17, 2025
5bba565
add server start retry loop to integration test
cbartz Sep 17, 2025
31f617c
update integration test
cbartz Sep 17, 2025
4c9e95b
set timeout to wait for message
cbartz Sep 17, 2025
c2e33d2
refactor integration test
cbartz Sep 17, 2025
4981d6f
add cover information
cbartz Sep 18, 2025
e94d123
Add CONTRIBUTING.md
cbartz Sep 18, 2025
46e0c38
Add pull request template
cbartz Sep 18, 2025
161a014
add connection retry
cbartz Sep 18, 2025
c6ee239
add setupConnection
cbartz Sep 18, 2025
06f2e80
update README.md
cbartz Sep 18, 2025
149207f
add concurrency cancel-in-progress
cbartz Sep 18, 2025
50b5116
Apply suggestions from code review
cbartz Sep 19, 2025
446804c
Update CONTRIBUTING.md
cbartz Sep 19, 2025
51cdf34
Apply suggestions from code review
cbartz Sep 19, 2025
4a0e659
Apply suggestions from code review
cbartz Sep 19, 2025
284d6db
Update internal/queue/queue.go
cbartz Sep 19, 2025
684e517
checkin queue_alt implementation
cbartz Sep 19, 2025
60addaf
add queue declare and connection reset logic
cbartz Sep 22, 2025
82278ed
use queue_alt
cbartz Sep 22, 2025
915e7b1
abstract away DeferredConfirm
cbartz Sep 22, 2025
17fe34e
add err cases
cbartz Sep 22, 2025
3ae9214
use new queue implementation
cbartz Sep 22, 2025
24dac09
fix unit tests
cbartz Sep 22, 2025
4f16825
add err cases
cbartz Sep 23, 2025
32a9a68
refactor Push to only contain high level fcts
cbartz Sep 23, 2025
4861c0e
add types file
cbartz Sep 23, 2025
27966d4
rename queue file to producer
cbartz Sep 23, 2025
d99c6ff
add locks
cbartz Sep 23, 2025
d3fb383
log errors
cbartz Sep 23, 2025
a522753
remove sleep in rabbitmq setup
cbartz Sep 23, 2025
1268a77
add sleep
cbartz Sep 23, 2025
7fde551
Revert "add sleep"
cbartz Sep 23, 2025
a0de71f
Revert "remove sleep in rabbitmq setup"
cbartz Sep 23, 2025
5e74280
use curl for health check
cbartz Sep 23, 2025
35a6c49
apply suggestions from code review
cbartz Sep 23, 2025
a599faa
Apply suggestions from code review
cbartz Sep 24, 2025
dfa5726
fix ctx code
cbartz Sep 24, 2025
374562d
fix Producer
cbartz Sep 24, 2025
25f35e5
make types private
cbartz Sep 24, 2025
7967ffb
add comment about manual unlock without defer
cbartz Sep 24, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions .github/.jira_sync_config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# See https://github.com/canonical/gh-jira-sync-bot for config
settings:
jira_project_key: "ISD"

status_mapping:
opened: Untriaged
closed: done
not_planned: rejected

add_gh_comment: true

epic_key: ISD-3981

label_mapping:
bug: Bug
enhancement: Story
13 changes: 13 additions & 0 deletions .github/pull_request_template.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
### Overview

<!-- A high level overview of the change -->

### Rationale

<!-- The reason the change is needed -->

### Checklist

- [ ] The PR is tagged with appropriate label (`urgent`, `trivial`, `senior-review-required`, `documentation`).

<!-- Explanation for any unchecked items above -->
31 changes: 31 additions & 0 deletions .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
name: Tests

on:
pull_request:
workflow_call:

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
test:
name: Run Unit and Lint Tests
runs-on: [self-hosted-linux-amd64-noble-edge]

steps:
- uses: actions/checkout@v5

- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: 1.24

- name: Ensure No Formatting Changes
run: |
go fmt ./...
git diff --exit-code

- name: Build and Test
run: |
go test -v -cover -race ./...
Comment thread
cbartz marked this conversation as resolved.
38 changes: 38 additions & 0 deletions .github/workflows/webhook_gateway_tests.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
name: Webhook Gateway Integration Tests
on:
pull_request:
paths:
- 'webhook-gateway/**'
- 'internal/**'
workflow_call:

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true


jobs:
integration-test:
runs-on: [self-hosted-linux-amd64-noble-edge]
name: Run Integration Tests
steps:
- uses: actions/checkout@v5

- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: 1.24
- name: Setup rabbitmq in an OCI container
run: |
docker run -d --name rabbitmq -p 5672:5672 -p 15672:15672 rabbitmq:3-management
Comment thread
cbartz marked this conversation as resolved.
until curl -fs http://localhost:15672;
do echo "waiting for rabbit mq"
sleep 2
done
- name: Run integration test
env:
RABBITMQ_CONNECT_STRING: amqp://guest:guest@localhost:5672/
APP_PORT: 8080
WEBHOOK_SECRET: fake-secret
run: |
go test -cover -v ./webhook-gateway -integration
1 change: 1 addition & 0 deletions CODEOWNERS
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* @cbartz @yhaliaw @javierdelapuente @yanksyoon @weiiwang01 @florentianayuwono
108 changes: 108 additions & 0 deletions CONTRIBUTING.md
Comment thread
cbartz marked this conversation as resolved.
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
# Contribute

## Overview

This document explains the processes and practices recommended for contributing enhancements to the codebase.

* Generally, before developing enhancements to this code base, you should consider [opening an issue](https://github.com/canonical/github-runner-operator/issues) explaining your use case.
* If you would like to chat with us about your use-cases or proposed implementation, you can reach us at [Canonical Charm Development Matrix public channel](https://matrix.to/#/#charmhub-charmdev:ubuntu.com) or [Discourse](https://discourse.charmhub.io/).
* All enhancements require review before being merged. Code review typically examines
* code quality
* test coverage

## Code of conduct

When contributing, you must abide by the
[Ubuntu Code of Conduct](https://ubuntu.com/community/ethos/code-of-conduct).

## Submissions

If you want to address an issue or a bug in this project,
notify in advance the people involved to avoid confusion;
also, reference the issue or bug number when you submit the changes.

- [Fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-forks)
our [GitHub repository](https://github.com/canonical/github-runner-operators)
and add the changes to your fork, properly structuring your commits,
providing detailed commit messages and signing your commits.
- Make sure the updated project builds and runs without warnings or errors;
this includes linting, documentation, code and tests.
- Submit the changes as a
[pull request (PR)](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork).

Your changes will be reviewed in due time; if approved, they will be eventually merged.

### Describing pull requests

To be properly considered, reviewed and merged,
your pull request must provide the following details:

- **Title**: Summarize the change in a short, descriptive title.

- **Overview**: Describe the problem that your pull request solves.
Mention any new features, bug fixes or refactoring.

- **Rationale**: Explain why the change is needed.


- **Checklist**: Complete the following items:

- The PR is tagged with appropriate label (`urgent`, `trivial`, `senior-review-required`, `documentation`).

### Signing commits

To improve contribution tracking,
we use the [Canonical contributor license agreement](https://assets.ubuntu.com/v1/ff2478d1-Canonical-HA-CLA-ANY-I_v1.2.pdf)
(CLA) as a legal sign-off, and we require all commits to have verified signatures.

### Canonical contributor agreement

Canonical welcomes contributions to this repository. Please check out our [contributor agreement](https://ubuntu.com/legal/contributors) if you’re interested in contributing to the solution.

#### Verified signatures on commits

All commits in a pull request must have cryptographic (verified) signatures.
To add signatures on your commits, follow the
[GitHub documentation](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits).


## Develop

For any problems with this charm, please [report bugs here](https://github.com/canonical/github-runner-operator/issues).

The code can be downloaded as follows:

```shell
git clone https://github.com/canonical/github-runner-operators.git
```

The code structure is as follows

- `internal/`: Internal libraries for the applications
- `webhook-gateway`: The webhook gateway application code


### Test

This project uses standard Go testing tools for unit tests and integration tests.
You can have a look at the GitHub actions workflows in `.github/workflows/` to see how the tests are run in CI.

Run unit tests using:

```shell
go test -race -v ./...
```

Run `webhook-gateway` integration tests using:

```shell
APP_PORT=8080 WEBHOOK_SECRET=fake RABBITMQ_CONNECT_STRING="amqp://guest:guest@localhost:5672/" go test -cover -v ./webhook-gateway -integration
```

It assumes you have access to a RabbitMQ server running reachable at $RABBITMQ_CONNECT_STRING.
You can use `docker` to run a RabbitMQ server locally:

```shell
docker run -d --rm --name rabbitmq -p 5672:5672 -p 15672:15672 rabbitmq:4-management
```

9 changes: 9 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# GitHub runner operators


![WIP](https://img.shields.io/badge/status-WIP-yellow)

A monorepo containing charms to operate Self-Hosted GitHub Action Runners.

At the moment, it contains initial code for the `webhook-gateway`
application, that receives and forwards GitHub webhooks to an AMQP queue.
14 changes: 14 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
module github.com/canonical/mayfly

go 1.24.6

require (
github.com/rabbitmq/amqp091-go v1.10.0
github.com/stretchr/testify v1.11.1
)

require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
14 changes: 14 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
131 changes: 131 additions & 0 deletions internal/queue/producer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
/*
* Copyright 2025 Canonical Ltd.
* See LICENSE file for licensing details.
*/

package queue

import (
"context"
"fmt"

amqp "github.com/rabbitmq/amqp091-go"
)

func (p *AmqpProducer) Push(ctx context.Context, headers map[string]interface{}, msg []byte) error {
p.mu.Lock() // Lock to prevent concurrent access to connection/channel object
err := p.resetConnectionOrChannelIfNecessary()
if err != nil {
p.mu.Unlock()
return err
}

msgConfirmation, err := p.publishMsg(msg, headers)
if err != nil {
p.mu.Unlock()
return err
}
p.mu.Unlock() // Unlock to not unblock other Push calls while waiting for confirmation
err = waitForMsgConfirmation(ctx, msgConfirmation)

return err
}

func (p *AmqpProducer) resetConnectionOrChannelIfNecessary() error {
if p.amqpConnection == nil || p.amqpConnection.IsClosed() {
err := p.resetConnection()
if err != nil {
return err
}
}

if p.amqpChannel == nil || p.amqpChannel.IsClosed() {
err := p.resetChannel()
if err != nil {
return err
}
}
return nil
}

func waitForMsgConfirmation(ctx context.Context, confirmation confirmation) error {
select {
case <-ctx.Done():
return ctx.Err()

case <-confirmation.Done():
if !confirmation.Acked() {
return fmt.Errorf("confirmation not acknowledged")
}
}
return nil
}

func (p *AmqpProducer) publishMsg(msg []byte, headers map[string]interface{}) (confirmation, error) {
confirmation, err := p.amqpChannel.PublishWithDeferredConfirm(
"", // exchange
p.queueName, // routing key
false, // mandatory
false, // immediate
amqp.Publishing{
ContentType: "application/json",
Body: msg,
Headers: headers,
},
)

if err != nil {
return nil, fmt.Errorf("failed to publish message: %w", err)
}
return confirmation, nil
}

func (p *AmqpProducer) resetConnection() error {
conn, err := p.connectFunc(p.uri)

if err != nil {
return fmt.Errorf("failed to connect to AMQP server: %w", err)
}
p.amqpConnection = conn
return nil
}

func (p *AmqpProducer) resetChannel() error {
c, err := p.amqpConnection.Channel()

if err != nil {
return fmt.Errorf("failed to open channel: %w", err)
}
p.amqpChannel = c

err = c.Confirm(false)
if err != nil {
return fmt.Errorf("failed to put channel in confirm mode: %w", err)
}

_, err = c.QueueDeclare(
p.queueName, // queueName
true, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
if err != nil {
return fmt.Errorf("failed to declare queue: %w", err)
}
return nil
}

func NewAmqpProducer(uri string, queueName string) *AmqpProducer {
return &AmqpProducer{
uri: uri,
queueName: queueName,
connectFunc: amqpConnect,
}
}

func amqpConnect(uri string) (amqpConnection, error) {
amqpConnection, err := amqp.Dial(uri)
return &amqpConnectionWrapper{Connection: amqpConnection}, err
}
Loading
Loading