Instrument makepkg builder

cmd/foundry/furnace.go

@@ -19,22 +19,22 @@ func registerFurnace(cmd *cobra.Command, g *run.Group, logger log.Logger, metric
 	furnaceCmd := &cobra.Command{
 		Use:   "furnace",
 		Short: "Run the Furnace component",
-		Run: func(cmd *cobra.Command, args []string) {
-			setupFurnace(config, g, logger, metrics)
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return setupFurnace(config, g, logger, metrics)
 		},
 	}
 	cmd.AddCommand(furnaceCmd)
 	config.registerFlags(furnaceCmd)
 }
 
-func setupFurnace(config *furnaceConfig, g *run.Group, logger log.Logger, _ *prometheus.Registry) {
+func setupFurnace(config *furnaceConfig, g *run.Group, logger log.Logger, reg prometheus.Registerer) error {
 	fc := furnace.NewFurnace(config.maxConcurrency, config.queueLimit, logger)
 
 	{
 		conn, err := net.Listen("tcp", config.grpc.bindAddress)
 		if err != nil {
-			// TODO(prmsrswt): this is a non-recoverable error, handle it like one.
-			level.Error(logger).Log("msg", err.Error())
+			level.Error(logger).Log("msg", "listening on gRPC bind address", "err", err)
+			return err
 		}
 		s := grpc.NewServer()
 		furnace.RegisterFurnaceServer(s, &fc)
@@ -48,14 +48,16 @@ func setupFurnace(config *furnaceConfig, g *run.Group, logger log.Logger, _ *pro
 	}
 
 	{
+		// TODO(prmsrswt): Make this directory configurable.
 		makepkgBuilder := builder.NewMakepkgBuilder("/tmp/foundry/furnace")
 		g.Add(func() error {
-			fc.Start(makepkgBuilder)
+			fc.Start(builder.BuilderWithMetrics(makepkgBuilder, reg))
 			return nil
 		}, func(_ error) {
 			fc.Stop()
 		})
 	}
+	return nil
 }
 
 type furnaceConfig struct {

cmd/foundry/main.go

@@ -47,7 +47,7 @@ func main() {
 		})
 	}
 	if err := rootCmd.Execute(); err != nil {
-		level.Error(logger).Log("msg", err.Error())
+		level.Error(logger).Log("err", err)
 		return
 	}
 	// Short circuit in case help command is called.

pkg/furnace/builder/builder.go

@@ -5,15 +5,31 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"time"
 
 	"github.com/go-git/go-git/v5"
 	"github.com/mikkeloscar/aur"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
 type Builder interface {
 	Build(aur.Pkg) error
 }
 
+// NoOpBuilder implements Builder interface. It's just a stub
+// used for testing.
+type NoOpBuilder struct {
+	ShouldFail bool
+}
+
+func (nb *NoOpBuilder) Build(aur.Pkg) error {
+	if nb.ShouldFail {
+		return fmt.Errorf("build failed")
+	}
+	return nil
+}
+
 // MakepkgBuilder implements the Builder interface. It builds the
 // packages using the `makepkg` script.
 type MakepkgBuilder struct {
@@ -67,3 +83,51 @@ func gitClone(dir, url string) error {
 	_, err = git.PlainClone(dir, false, &git.CloneOptions{URL: url, Depth: 1})
 	return err
 }
+
+type instrumentedBuilder struct {
+	builder Builder
+
+	buildOps          prometheus.Counter
+	buildOpsCompleted prometheus.Counter
+	buildOpsFailure   prometheus.Counter
+
+	buildDuration prometheus.Histogram
+}
+
+func BuilderWithMetrics(b Builder, reg prometheus.Registerer) *instrumentedBuilder {
+	return &instrumentedBuilder{
+		builder: b,
+		buildOps: promauto.With(reg).NewCounter(prometheus.CounterOpts{
+			Name: "foundry_furnace_builder_builds_started_total",
+			Help: "Total number of attempted builds by the builder.",
+		}),
+		buildOpsCompleted: promauto.With(reg).NewCounter(prometheus.CounterOpts{
+			Name: "foundry_furnace_builder_builds_completed_total",
+			Help: "Total number of builds completed by the builder irrespective of success or failure.",
+		}),
+		buildOpsFailure: promauto.With(reg).NewCounter(prometheus.CounterOpts{
+			Name: "foundry_furnace_builder_build_failures_total",
+			Help: "Total number of builds attempted by the builder that failed.",
+		}),
+		buildDuration: promauto.With(reg).NewHistogram(prometheus.HistogramOpts{
+			Name:    "foundry_furnace_builder_build_duration_seconds",
+			Help:    "Duration of builds completed successfully by the builder.",
+			Buckets: []float64{10, 30, 60, 90, 120, 180, 300, 600, 900, 1800},
+		}),
+	}
+}
+
+func (ib *instrumentedBuilder) Build(pkg aur.Pkg) error {
+	ib.buildOps.Inc()
+	defer ib.buildOpsCompleted.Inc()
+	start := time.Now()
+
+	err := ib.builder.Build(pkg)
+	if err != nil {
+		ib.buildOpsFailure.Inc()
+		return err
+	}
+
+	ib.buildDuration.Observe(time.Since(start).Seconds())
+	return nil
+}

pkg/furnace/builder/builder_test.go

@@ -4,6 +4,9 @@ import (
 	"testing"
 
 	"github.com/go-git/go-git/v5"
+	"github.com/mikkeloscar/aur"
+	"github.com/prometheus/client_golang/prometheus"
+	promtest "github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -24,3 +27,43 @@ func TestGitClone(t *testing.T) {
 	_, err = git.PlainOpen(dir)
 	assert.NoError(t, err)
 }
+
+func TestInstrumentedBuilder(t *testing.T) {
+	cases := []struct {
+		shouldFail []bool
+	}{
+		// Single successful build.
+		{shouldFail: []bool{false}},
+		// Single failing build.
+		{shouldFail: []bool{true}},
+		// No builds.
+		{shouldFail: []bool{}},
+		// A mix of passing and failing builds.
+		{shouldFail: []bool{false, true, false, false, true, false}},
+	}
+	for _, v := range cases {
+		tc := v
+		t.Run("builds", func(t *testing.T) {
+			t.Parallel()
+			reg := prometheus.NewRegistry()
+			noop := &NoOpBuilder{}
+			b := BuilderWithMetrics(noop, reg)
+			var numFail int
+			for _, v := range tc.shouldFail {
+				if v {
+					numFail++
+				}
+				noop.ShouldFail = v
+				err := b.Build(aur.Pkg{Name: "example"})
+				if v {
+					assert.Error(t, err)
+				} else {
+					assert.NoError(t, err)
+				}
+			}
+			assert.Equal(t, float64(len(tc.shouldFail)), promtest.ToFloat64(b.buildOps))
+			assert.Equal(t, float64(len(tc.shouldFail)), promtest.ToFloat64(b.buildOpsCompleted))
+			assert.Equal(t, float64(numFail), promtest.ToFloat64(b.buildOpsFailure))
+		})
+	}
+}