devnet instructions and more API requests

.gitignore

@@ -24,3 +24,4 @@
 # Live configs
 testdata/*-config.json
 testdata/*-secrets.json
+/database.dump

Makefile

@@ -4,6 +4,11 @@
 
 VERSION := $(shell git describe --tags --always --dirty="-dev")
 
+# A few colors
+RED:=\033[0;31m
+GREEN:=\033[0;32m
+NC:=\033[0m
+
 ##@ Help
 
 .PHONY: help
@@ -27,6 +32,16 @@ build: ## Build the HTTP server
 
 ##@ Test & Development
 
+.PHONY: lt
+lt: lint test ## Run linters and tests (always do this!)
+
+.PHONY: fmt
+fmt: ## Format the code (use this often)
+	gofmt -s -w .
+	gci write .
+	gofumpt -w -extra .
+	go mod tidy
+
 .PHONY: test
 test: ## Run tests
 	go test ./...
@@ -43,20 +58,10 @@ lint: ## Run linters
 	staticcheck ./...
 	golangci-lint run
 
-.PHONY: fmt
-fmt: ## Format the code
-	gofmt -s -w .
-	gci write .
-	gofumpt -w -extra .
-	go mod tidy
-
 .PHONY: gofumpt
 gofumpt: ## Run gofumpt
 	gofumpt -l -w -extra .
 
-.PHONY: lt
-lt: lint test ## Run linters and tests
-
 .PHONY: cover
 cover: ## Run tests with coverage
 	go test -coverprofile=/tmp/go-sim-lb.cover.tmp ./...
@@ -72,8 +77,30 @@ cover-html: ## Run tests with coverage and open the HTML report
 .PHONY: docker-httpserver
 docker-httpserver: ## Build the HTTP server Docker image
 	DOCKER_BUILDKIT=1 docker build \
+		--file docker/httpserver/Dockerfile \
 		--platform linux/amd64 \
 		--build-arg VERSION=${VERSION} \
-		--file httpserver.dockerfile \
-		--tag your-project \
+		--tag builder-hub \
 	.
+
+.PHONY: db-dump
+db-dump: ## Dump the database contents to file 'database.dump'
+	pg_dump -h localhost -U postgres --column-inserts --data-only postgres -f database.dump
+	@printf "Database dumped to file: $(GREEN)database.dump$(NC) ✅\n"
+
+.PHONY: dev-db-setup
+db-dev-setup: ## Create the basic database entries for testing and development
+	@printf "$(GREEN)Create the allow-all measurements $(NC)\n"
+	curl --request POST --url http://localhost:8081/api/admin/v1/measurements --data '{"measurement_id": "test1","attestation_type": "test","measurements": {}}'
+
+	@printf "$(GREEN)Enable the measurements $(NC)\n"
+	curl --request POST --url http://localhost:8081/api/admin/v1/measurements/activation/test1 --data '{"enabled": true}'
+
+	@printf "$(GREEN)Create the builder $(NC)\n"
+	curl --request POST --url http://localhost:8081/api/admin/v1/builders --data '{"name": "test_builder","ip_address": "1.2.3.4"}'
+
+	@printf "$(GREEN)Create the builder configuration $(NC)\n"
+	curl --request POST --url http://localhost:8081/api/admin/v1/builders/configuration/test_builder --data '{"dns_name": "foobar-v1.a.b.c","rbuilder": {"extra_data": "FooBar"}}'
+
+	@printf "$(GREEN)Enable the builder $(NC)\n"
+	curl --request POST --url http://localhost:8081/api/admin/v1/builders/activation/test_builder --data '{"enabled": true}'

README.md

@@ -13,14 +13,17 @@ BuilderHub has these responsibilities:
 2. Provisioning of secrets and configuration
 3. Peer discovery
 
-System context diagram:
+---
 
 ![Architecture](https://buildernet.org/assets/ideal-img/flashbots-infra-dataflow.7377b1f.3909.png)
 
 ---
 
 ## Getting started
 
+
+### Manual setup
+
 **Start the database and the server:**
 
 ```bash
@@ -34,21 +37,11 @@ for file in schema/*.sql; do psql "postgres://postgres:postgres@localhost:5432/p
 go run cmd/httpserver/main.go
 ```
 
-**Start everything in Docker from our published images:**
-
-```bash
-# Switch into the 'docker' directory
-cd docker
-
-# Update and start the services
-docker-compose pull
-docker-compose up
+### Using Docker
 
-# Make an example request
-curl localhost:8888
-```
+See instructions on using Docker to run the full stack at [`docs/devenv-setup.md`](./docs/devenv-setup.md)
 
-**Query a few endpoints:**
+### Example requests
 
 ```bash
 # Public endpoints
@@ -60,16 +53,12 @@ curl localhost:8080/api/l1-builder/v1/configuration
 curl -X POST localhost:8080/api/l1-builder/v1/register_credentials/rbuilder
 ```
 
-Run database tests>
-
-```bash
-RUN_DB_TESTS=1 make test
-```
+### Testing
 
-**Stop the database:**
+Run test suite with database tests included:
 
 ```bash
-docker rm -f postgres-test
+RUN_DB_TESTS=1 make test
 ```
 
 ---

docker/docker-compose.yaml

@@ -1,13 +1,15 @@
 services:
   db:
     image: flashbots/builder-hub-db
+    ports:
+      - 5432:5432
     environment:
       PGUSER: postgres
       POSTGRES_DB: postgres
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: postgres
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_PASSWORD}"]
+      test: ["CMD-SHELL", "pg_isready"]
       interval: 5s
       retries: 5
       start_period: 2s
@@ -27,6 +29,7 @@ services:
       - 8082:8082
       - 8090:8090
     environment:
+      MOCK_SECRETS: true
       POSTGRES_DSN: "postgres://postgres:postgres@db:5432/postgres?sslmode=disable"
       LISTEN_ADDR: "0.0.0.0:8080"
       ADMIN_ADDR: "0.0.0.0:8081"

docs/api-docs/README.md

@@ -1,4 +1,5 @@
 You can load the Bruno collection with https://www.usebruno.com
 
 The docs include these collections:
-- [Admin API](./admin-api/)
+- [Admin API](./admin-api/)
+- [Instance API](./instance-api/)

docs/api-docs/instance-api/BuilderHub Instance API/Get measurements.bru

@@ -0,0 +1,11 @@
+meta {
+  name: Get measurements
+  type: http
+  seq: 2
+}
+
+get {
+  url: http://localhost:8888/api/l1-builder/v1/measurements
+  body: none
+  auth: none
+}

docs/api-docs/instance-api/BuilderHub Instance API/Get peers.bru

@@ -0,0 +1,11 @@
+meta {
+  name: Get peers
+  type: http
+  seq: 3
+}
+
+get {
+  url: http://localhost:8888/api/l1-builder/v1/builders
+  body: none
+  auth: none
+}

docs/api-docs/instance-api/BuilderHub Instance API/Register credentials.bru

@@ -0,0 +1,17 @@
+meta {
+  name: Register credentials
+  type: http
+  seq: 4
+}
+
+post {
+  url: http://localhost:8888/api/l1-builder/v1/register_credentials/rbuilder
+  body: json
+  auth: none
+}
+
+body:json {
+  {
+    "ecdsa_pubkey_address": "0x321f3426eEc20DE1910af1CD595c4DD83BEA0BA5"
+  }
+}

docs/api-docs/instance-api/BuilderHub Instance API/bruno.json

@@ -0,0 +1,9 @@
+{
+  "version": "1",
+  "name": "BuilderHub Instance API",
+  "type": "collection",
+  "ignore": [
+    "node_modules",
+    ".git"
+  ]
+}

docs/api-docs/instance-api/Get measurements.bru

@@ -0,0 +1,11 @@
+meta {
+  name: Get measurements
+  type: http
+  seq: 11
+}
+
+get {
+  url: http://localhost:8081/api/admin/v1/builders/configuration/{builder}/full
+  body: none
+  auth: none
+}

docs/devenv-setup.md

@@ -0,0 +1,124 @@
+This is a quick guide on setting setting up and configuring a dev BuilderHub instance.
+
+---
+
+## Getting started
+
+First, use docker-compose to start the three containers:
+
+| Service     | Ports & Notes                                                                                                          |
+| ----------- | ---------------------------------------------------------------------------------------------------------------------- |
+| builder-hub | **8080** (Instance API), **8081** (Admin API), **8082** (Internal API)                                                 |
+| mock-proxy  | **8888** (forwards to builder-hub Instance API on port 8080, adds dummy auth headers that in prod cvm-proxy would add) |
+| postgres    | **5432** (with applied migrations)                                                                                     |
+
+See also [`docker-compose.yaml`](../docker/docker-compose.yaml) for more details.
+
+```bash
+# Switch into the 'docker' directory
+cd docker
+
+# Update and start the services
+docker-compose pull
+docker-compose up
+```
+
+---
+
+## Public API
+
+There are some public API points, most notably the `get-measurements` endpoint, which is accessible without authentication:
+
+```bash
+curl http://localhost:8080/api/l1-builder/v1/measurements | jq
+```
+
+---
+
+## Admin API
+
+For initial setup, use the [Admin API](https://github.com/flashbots/builder-hub?tab=readme-ov-file#admin-endpoints) (on port 8081) to:
+1. Create and enable allowed measurements
+2. Create a builder instance
+3. Create a builder configuration
+4. Enable the builder instance
+
+```bash
+# 1a. Create a new measurements entry (empty allowed measurements will allow all client measurements)
+curl -v \
+  --request POST \
+  --url http://localhost:8081/api/admin/v1/measurements \
+  --data '{
+  "measurement_id": "test1",
+  "attestation_type": "test",
+  "measurements": {}
+}'
+
+# 1b. Enable the new measurements
+curl -v \
+  --request POST \
+  --url http://localhost:8081/api/admin/v1/measurements/activation/test1 \
+  --data '{
+  "enabled": true
+}'
+
+# 2. Create a new builder instance (with IP address 1.2.3.4, which is fixed in the mock-proxy)
+curl -v \
+  --request POST \
+  --url http://localhost:8081/api/admin/v1/builders \
+  --data '{
+  "name": "test_builder",
+  "ip_address": "1.2.3.4"
+}'
+
+# 3. Create (and enable) a new builder configuration
+curl -v \
+  --request POST \
+  --url http://localhost:8081/api/admin/v1/builders/configuration/test_builder \
+  --data '{
+  "dns_name": "foobar-v1.a.b.c",
+  "rbuilder": {
+    "extra_data": "FooBar"
+  }
+}'
+
+# 4. Enable the new builder instance
+curl -v \
+  --request POST \
+  --url http://localhost:8081/api/admin/v1/builders/activation/test_builder \
+  --data '{
+  "enabled": true
+}'
+```
+
+---
+
+## Instance API (authenticated)
+
+Now you can use the (authenticated) Instance API, like any production Yocto TDX instance would:
+1. Get own instance configuration
+2. Get the list of peers
+3. Register credentials
+
+The API is authenticated through headers that are attached by the proxy, namely measurements, attestation type and IP address headers.
+The mock-proxy on port 8888 [adds these headers](https://github.com/flashbots/builder-hub/blob/main/docker/mock-proxy/nginx-default.conf),
+so you can use the API without any additional setup (or cvm-proxy instances).
+
+```bash
+# 1. Get the instance configuration
+curl http://localhost:8888/api/l1-builder/v1/configuration | jq
+
+# 2. Get the list of peers
+curl http://localhost:8888/api/l1-builder/v1/builders | jq
+
+# 3. Register credentials for 'rbuilder' service
+curl -v \
+  --request POST \
+  --url http://localhost:8888/api/l1-builder/v1/register_credentials/rbuilder \
+  --data '{
+  "ecdsa_pubkey_address": "0x321f3426eEc20DE1910af1CD595c4DD83BEA0BA5"
+}'
+
+# If you now call get-peers again, it will contain the newly registered address:
+curl http://localhost:8888/api/l1-builder/v1/builders | jq
+```