GW Tester: Add Prometheus support

examples/gw-tester/README.md

@@ -9,6 +9,8 @@ A pseudo eNB and MME as a tester for S/P-GW.
 It is a burden to use actual UE/eNB/MME just to test S/P-GW, isn't it?  
 GW Tester emulates the minimal required behavior of surrounding nodes to perform a quick and simple testing on S/P-GW.
 
+A blog post by the author is available [here](https://wmnsk.com/posts/20200116_gw-tester/) for those who interested :)
+
 ## How it works
 
 ### Authentication
@@ -97,14 +99,15 @@ In general, config consists of the network information of local/remote node, and
 
 These values are used to identify eNB. Some of them are just to set inside the packets, and not validated.
 
-| config     | type of value | description                                  |
-|------------|---------------|----------------------------------------------|
-| `mcc`      | string        | MCC of eNB                                   |
-| `mnc`      | string        | MNC of eNB                                   |
-| `rat_type` | uint8         | RAT Type (`6` for E-UTRAN)                   |
-| `tai`      | uint16        | TAI of eNB                                   |
-| `eci`      | uint32        | ECI of eNB                                   |
-| `mme_addr` | string        | IP/Port of MME to dial, for S1-MME interface |
+| config      | type of value | description                                  |
+|-------------|---------------|----------------------------------------------|
+| `mcc`       | string        | MCC of eNB                                   |
+| `mnc`       | string        | MNC of eNB                                   |
+| `rat_type`  | uint8         | RAT Type (`6` for E-UTRAN)                   |
+| `tai`       | uint16        | TAI of eNB                                   |
+| `eci`       | uint32        | ECI of eNB                                   |
+| `mme_addr`  | string        | IP/Port of MME to dial, for S1-MME interface |
+| `prom_addr` | string        | IP/Port of MME to serve Prometheus           |
 
 #### Local Addressess
 
@@ -137,11 +140,12 @@ These values are used to identify eNB. Some of them are just to set inside the p
 
 These values are used to identify MME. Some of them are just to set inside the packets, and not validated.
 
-| config | type of value | description                        |
-|--------|---------------|------------------------------------|
-| `mcc`  | string        | MCC of MME                         |
-| `mnc`  | string        | MNC of MME                         |
-| `apn`  | string        | APN assigned to all the subscriber |
+| config      | type of value | description                        |
+|-------------|---------------|------------------------------------|
+| `mcc`       | string        | MCC of MME                         |
+| `mnc`       | string        | MNC of MME                         |
+| `apn`       | string        | APN assigned to all the subscriber |
+| `prom_addr` | string        | IP/Port of MME to serve Prometheus |
 
 #### Local Addressess
 
@@ -167,12 +171,13 @@ IP addresses required to know/tell S-GW. This is normally done by DNS lookup wit
 
 `local_addresses` are the IP addresses/ports to be bound on local machine.
 
-| config     | type of value | description                      |
-|------------|---------------|----------------------------------|
-| `s11_addr` | string        | local IP/Port for S11 interface  |
-| `s1u_addr` | string        | local IP/Port for S1-U interface |
-| `s5c_addr` | string        | local IP/Port for S5-C interface |
-| `s5u_addr` | string        | local IP/Port for S5-U interface |
+| config      | type of value | description                        |
+|-------------|---------------|------------------------------------|
+| `s11_addr`  | string        | local IP/Port for S11 interface    |
+| `s1u_addr`  | string        | local IP/Port for S1-U interface   |
+| `s5c_addr`  | string        | local IP/Port for S5-C interface   |
+| `s5u_addr`  | string        | local IP/Port for S5-U interface   |
+| `prom_addr` | string        | IP/Port of MME to serve Prometheus |
 
 ### P-GW
 
@@ -182,6 +187,7 @@ IP addresses required to know/tell S-GW. This is normally done by DNS lookup wit
 |----------------|---------------|------------------------------------------------------------------------|
 | `sgi_if_name`  | string        | name of network interface on SGi side. Used to downlink route traffic. |
 | `route_subnet` | string        | IP subnet of UEs that should be routed properly.                       |
+| `prom_addr`    | string        | IP/Port of MME to serve Prometheus                                     |
 
 #### Local Addressess
 
@@ -199,6 +205,14 @@ IP addresses required to know/tell S-GW. This is normally done by DNS lookup wit
 
 The programs can handle `SIGHUP` to reload config without deleting sessions. Update YAML file and send `SIGHUP` to the process.
 
-### (WIP) Instrumentation
+### Instrumentation
+
+GW Tester nodes expose some metrics for Prometheus if `prom_addr` is given in each config. You can see the sample response from each node in [this Gist](https://gist.github.com/wmnsk/72f6d2d2450452090cd6351ffe63f660).  
+I'm planning to add some more metrics like "success rate of HTTP probe" etc.
 
-_(Metrics exposure for Prometheus is work in progress... stay tuned!)_
+| Metrics           | Name                                  | Description                                   |
+|-------------------|---------------------------------------|-----------------------------------------------|
+| Active sessions   | `<node-name>_active_sessions`         | number of session established currently       |
+| Active bearers    | `<node-name>_active_bearers`          | number of GTP-U tunnels established currently |
+| Messages sent     | `<node-name>_messages_sent_total`     | number of messages sent by messagge type      |
+| Messages received | `<node-name>_messages_received_total` | number of messages received by messagge type  |

examples/gw-tester/enb/config.go

@@ -18,7 +18,8 @@ type Config struct {
 		S1UAddr string `yaml:"s1u_addr"`
 	} `yaml:"local_addresses"`
 
-	MMEAddr string `yaml:"mme_addr"`
+	MMEAddr  string `yaml:"mme_addr"`
+	PromAddr string `yaml:"prom_addr"`
 
 	MCC         string        `yaml:"mcc"`
 	MNC         string        `yaml:"mnc"`

examples/gw-tester/enb/enb.go

@@ -16,6 +16,7 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/vishvananda/netlink"
 	"google.golang.org/grpc"
 
@@ -27,7 +28,7 @@ type enb struct {
 	mu sync.Mutex
 
 	// S1-MME
-	cAddr       net.Addr
+	mmeAddr     net.Addr
 	cConn       *grpc.ClientConn
 	s1mmeClient s1mme.AttacherClient
 
@@ -45,6 +46,9 @@ type enb struct {
 	addedRoutes []*netlink.Route
 	addedRules  []*netlink.Rule
 
+	promAddr string
+	mc       *metricsCollector
+
 	errCh chan error
 }
 
@@ -70,21 +74,30 @@ func newENB(cfg *Config) (*enb, error) {
 		return nil, err
 	}
 
-	e.cAddr, err = net.ResolveTCPAddr("tcp", cfg.MMEAddr)
+	e.mmeAddr, err = net.ResolveTCPAddr("tcp", cfg.MMEAddr)
 	if err != nil {
 		return nil, err
 	}
 
+	if cfg.PromAddr != "" {
+		// validate if the address is valid or not.
+		if _, err = net.ResolveTCPAddr("tcp", cfg.PromAddr); err != nil {
+			return nil, err
+		}
+		e.promAddr = cfg.PromAddr
+	}
+
 	return e, nil
 }
 
 func (e *enb) run(ctx context.Context) error {
 	// TODO: bind local address(cfg.LocalAddrs.S1CIP) with WithDialer option?
-	conn, err := grpc.Dial(e.cAddr.String(), grpc.WithInsecure())
+	conn, err := grpc.Dial(e.mmeAddr.String(), grpc.WithInsecure())
 	if err != nil {
 		return err
 	}
 	e.s1mmeClient = s1mme.NewAttacherClient(conn)
+	log.Printf("Established S1-MME connection with %s", e.mmeAddr)
 
 	e.uConn = v1.NewUPlaneConn(e.uAddr)
 	if err := e.uConn.EnableKernelGTP("gtp-enb", v1.RoleSGSN); err != nil {
@@ -97,6 +110,22 @@ func (e *enb) run(ctx context.Context) error {
 		}
 		log.Println("uConn.ListenAndServe exitted")
 	}()
+	log.Printf("Started serving S1-U on %s", e.uAddr)
+
+	// start serving Prometheus, if address is given
+	if e.promAddr != "" {
+		if err := e.runMetricsCollector(); err != nil {
+			return err
+		}
+
+		http.Handle("/metrics", promhttp.Handler())
+		go func() {
+			if err := http.ListenAndServe(e.promAddr, nil); err != nil {
+				log.Println(err)
+			}
+		}()
+		log.Printf("Started serving Prometheus on %s", e.promAddr)
+	}
 
 	for _, sub := range e.candidateSubs {
 		select {
@@ -111,10 +140,9 @@ func (e *enb) run(ctx context.Context) error {
 		}
 	}
 
+	// wait for new subscribers to be attached
 	e.attachCh = make(chan *Subscriber)
 	defer close(e.attachCh)
-
-	// wait for new subscribers to be attached
 	for {
 		select {
 		case <-ctx.Done():
@@ -225,6 +253,10 @@ func (e *enb) attach(ctx context.Context, sub *Subscriber) error {
 	if err != nil {
 		return err
 	}
+	if e.mc != nil {
+		e.mc.messagesSent.WithLabelValues(e.mmeAddr.String(), "Attach Request").Inc()
+		e.mc.messagesReceived.WithLabelValues(e.mmeAddr.String(), "Attach Response").Inc()
+	}
 
 	switch rsp.Cause {
 	case s1mme.Cause_SUCCESS:

examples/gw-tester/enb/enb.yml

@@ -7,6 +7,7 @@ local_addresses:
   s1c_ip: "127.0.1.11"
   s1u_addr: "127.0.0.11:2152"
 mme_addr: "127.0.1.12:36412"
+prom_addr: "127.0.10.1:58080"
 subscribers:
   - imsi: "001010000000001"
     msisdn: "814000000001"

examples/gw-tester/enb/go.mod

@@ -4,6 +4,7 @@ go 1.13
 
 require (
 	github.com/pkg/errors v0.8.1
+	github.com/prometheus/client_golang v1.3.0
 	github.com/vishvananda/netlink v1.0.0
 	github.com/wmnsk/go-gtp v0.7.1
 	google.golang.org/grpc v1.26.0