doc: add haproxy example

example/README.md → examples/backfill/README.md

@@ -1,4 +1,4 @@
-# honeytail example
+# honeytail backfill example
 
 There is an example log file (`honeytail-example.log`) to see how this would show up in Honeycomb.
 We are using the `--backfill` option to get the existing logs.

examples/haproxy/README.md

@@ -0,0 +1,79 @@
+## honeytail HAProxy example
+
+Starting with Honeycomb instrumentation "at the edge" (i.e., with your reverse
+proxies or load balancers) can allow you to quickly get valuable data into
+Honeycomb. You can begin gaining visibility into your systems quickly this way,
+and gradually work your way inward (consequently gaining more power) by adding
+native code instrumentation later.
+
+This example demonstrates this concept by using haproxy as a reverse proxy to the
+[simple server](../simple-server), and
+ingesting the haproxy access logs as Honeycomb events.
+
+## Run Natively
+
+Run the Simple Server example linked above.
+
+Use the provided `haproxy.cfg` for your local haproxy, and `rsyslog.conf` 
+for your local rsyslog. You may need to update the following hosts in `haproxy.cfg`:
+- log to `localhost` instead of `rsyslog`
+- point backend app at `localhost` instead of `api`
+
+Then:
+
+```
+$ honeytail --debug \
+    --parser=nginx \
+    --dataset=examples.honeytail.haproxy \
+    --writekey=$HONEYCOMB_WRITE_KEY \
+    --nginx.conf=hny-haproxy.conf \
+    --nginx.format=haproxy \
+    --file=/var/log/honeytail/access.log
+```
+
+## Run in Docker
+
+```shell
+docker-compose build && docker-compose up -d
+curl localhost
+> I'm here!
+curl localhost/hello/meow
+> Hello meow!
+```
+
+## Event Fields
+
+| **Name**                   | **Description**                                                    | **Example Value**            |
+|----------------------------|--------------------------------------------------------------------|------------------------------|
+| `act_conn`                 | # of active connections                                            | `1`                          |
+| `backend`                  | Proxy backend name                                                 | `app`                        |
+| `backend_queue`            | # requests processed ahead of current request on the backend queue | `0`                          |
+| `backend_server`           | Proxy backend server name                                          | `simpleServer`               |
+| `be_conn`                  | # active backend connections                                       | `0`                          |
+| `bytes_read`               | # bytes sent to the client                                         | `341`                        |
+| `client_ip`                | IP address of the client which initiated the connection to haproxy | `172.21.0.1`                 |
+| `client_port`              | TCP port of the client which initiated the connection to haproxy   | `59240`                      |
+| `fe_conn`                  | # active frontend connections                                      | `1`                          |
+| `frontend`                 | Proxy frontend name                                                | `main`                       |
+| `hostname`                 | Proxy hostname                                                     | `localhost`                  |
+| `pid`                      | Proxy pid                                                          | `7`                          |
+| `process`                  | Proxy process                                                      | `haproxy`                    |
+| `request`                  | Complete HTTP request line                                         | `GET /hello/meow HTTP/1.1`   |
+| `request_headers`          | Captured request headers                                           | `localhost|curl/7.79.1` |
+| `request_method`           | HTTP request method                                                | `GET`                        |
+| `request_path`             | URL of the request                                                 | `/hello/meow`                |
+| `request_pathshape`        | "Shape" of the request path                                        | `/hello/meow`                |
+| `request_protocol_version` | HTTP version                                                       | `HTTP/1.1`                   |
+| `request_shape`            | "Shape" of the request                                             | `/hello/meow`                |
+| `request_uri`              | URI for the request                                                | `/hello/meow`                |
+| `response_headers`         | Captured response headers                                          | `|text/plain; charset=` |
+| `retries`                  | # connection retries                                               | `0`                          |
+| `srv_conn`                 | # active server connections                                        | `0`                          |
+| `srv_queue`                | # requests processed ahead of current request on the server queue  | `0`                          |
+| `status_code`              | HTTP status code returned                                          | `200`                        |
+| `termination_state`        | Session state at disconnection                                     | `----`                       |
+| `time_backend_conn`        | Total time (ms) waiting for connection to final server             | `0`                          |
+| `time_backend_resp`        | Total time (ms) waiting for a full request from the backend        | `5`                          |
+| `time_client_connect`      | Total time (ms) waiting for a full request from the client         | `0`                          |
+| `time_queued`              | Total time (ms) waiting in proxy queues                            | `0`                          |
+| `time_total`               | Total time (ms) the request remained active in haproxy             | `5`                          |

examples/haproxy/docker-compose.yml

@@ -0,0 +1,55 @@
+version: '3'
+
+services:
+  api:
+    build: ../simple-server
+    networks:
+    - main
+    restart: on-failure
+
+  reverse_proxy:
+    image: haproxy:alpine
+    ports:
+    - "80:80"
+    networks:
+    - main
+    volumes:
+    - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
+    depends_on:
+    - "api"
+    - "rsyslog"
+
+  rsyslog:
+    build:
+      context: .
+      dockerfile: rsyslog.Dockerfile
+    ports:
+    - "10514:10514/udp"
+    networks:
+    - main
+    volumes:
+    - ./rsyslog.conf:/config/rsyslog.conf
+    - honeytail-logs:/var/log/honeytail
+    environment:
+      RSYSLOG_CONF: "/config/rsyslog.conf"
+
+  honeytail:
+    build: ../../.
+    volumes:
+    - ./hny-haproxy.conf:/etc/hny-conf/hny-haproxy.conf
+    - honeytail-logs:/var/log/honeytail
+    - honeytail-tmp:/tmp # Used to maintain state of what's already been processed
+    command: honeytail --debug --parser=nginx --dataset=examples.honeytail.haproxy --writekey=$HONEYCOMB_WRITE_KEY --nginx.conf=/etc/hny-conf/hny-haproxy.conf --nginx.format=haproxy --file=/var/log/honeytail/access.log
+    environment:
+      HONEYCOMB_WRITE_KEY:
+    depends_on:
+    - "reverse_proxy"
+    restart: on-failure
+
+volumes:
+  honeytail-logs:
+  honeytail-tmp:
+
+networks:
+  main:
+    driver: bridge

examples/haproxy/haproxy.cfg

@@ -0,0 +1,27 @@
+global
+    log rsyslog:10514 local0
+
+defaults
+    log     global
+    option  httplog
+    mode    http
+    timeout client  30s
+    timeout server  30s
+    timeout connect 30s
+
+frontend main
+    bind *:80
+
+    http-request capture req.hdr(Host)          len 10
+    http-request capture req.hdr(User-Agent)    len 100
+
+    declare capture response len 20
+    http-response capture res.hdr(Server) id 0
+
+    declare capture response len 20
+    http-response capture res.hdr(Content-Type) id 1
+
+    default_backend app
+
+backend app
+    server simpleServer api:8080

examples/haproxy/hny-haproxy.conf

@@ -0,0 +1,18 @@
+http {
+    log_format haproxy '....-..-.....:..:.....:.. $hostname $process[$pid]: '
+        '$client_ip:$client_port [$time_local] $frontend $backend/$backend_server '
+        '$time_client_connect/$time_queued/$time_backend_conn/$time_backend_resp/$time_total '
+        '$status_code $bytes_read $request_cookie $response_cookie $termination_state '
+        '$act_conn/$fe_conn/$be_conn/$srv_conn/$retries $srv_queue/$backend_queue '
+        '{$request_headers} {$response_headers} '
+        '"$request"';
+}
+
+# sample log line
+# 2022-09-04T21:34:04+00:00 haproxy_reverse_proxy_1.haproxy_main haproxy[8]:
+# 172.24.0.1:62106 [04/Sep/2022:21:34:04.666] main app/simpleServer
+# 0/0/0/0/0
+# 200 132 - - ----
+# 1/1/0/0/0 0/0
+# {localhost|curl/7.79.1} {|text/plain; charset=}
+# "GET /hello/meow HTTP/1.1"

examples/haproxy/rsyslog.Dockerfile

@@ -0,0 +1,6 @@
+FROM rsyslog/syslog_appliance_alpine
+
+# Create this so honeytail won't crash due to it not existing the first time
+# it's brought up.
+RUN mkdir -p /var/log/honeytail
+RUN touch /var/log/honeytail/access.log

examples/haproxy/rsyslog.conf

@@ -0,0 +1,11 @@
+global(processInternalMessages="on")
+
+module(load="imudp" TimeRequery="500")
+
+input(type="imudp" port="10514")
+
+include(file="/etc/rsyslog.conf.d/log_to_files.conf" config.enabled=`echo $ENABLE_LOGFILES`)
+
+local0.* action(type="omfile" file="/var/log/honeytail/access.log")
+
+include(text=`echo $CNF_CALL_LOG_TO_LOGFILES`)

examples/simple-server/Dockerfile

@@ -0,0 +1,10 @@
+FROM golang:alpine AS build
+WORKDIR /src
+ENV CGO_ENABLED=0
+COPY . .
+RUN go build -o /out/simple-server ./main.go
+
+FROM scratch AS bin
+WORKDIR /app
+COPY --from=build /out/simple-server /app/
+CMD ["/app/simple-server"]

examples/simple-server/README.md

@@ -0,0 +1 @@
+# Simple Server used for examples

examples/simple-server/go.mod

@@ -0,0 +1,5 @@
+module github.com/honeycombio/honeytail/examples/simple-server
+
+go 1.17
+
+require github.com/gorilla/mux v1.8.0

examples/simple-server/go.sum

@@ -0,0 +1,2 @@
+github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=

examples/simple-server/main.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+
+	"github.com/gorilla/mux"
+)
+
+func main() {
+	router := mux.NewRouter()
+
+	router.HandleFunc("/", mainHandler)
+	router.HandleFunc("/hello/{person}", helloHandler)
+
+	fmt.Println("Listening on localhost:8080...")
+	log.Fatal(http.ListenAndServe(":8080", router))
+}
+
+func mainHandler(w http.ResponseWriter, _ *http.Request) {
+	_, _ = w.Write([]byte("I'm here!\n"))
+}
+
+func helloHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	person := vars["person"]
+	_, _ = w.Write([]byte(fmt.Sprintf("Hello %s!\n", person)))
+}