diff --git a/book/src/framework/observability/observability_stack.md b/book/src/framework/observability/observability_stack.md index 2648f68ee..165412057 100644 --- a/book/src/framework/observability/observability_stack.md +++ b/book/src/framework/observability/observability_stack.md @@ -2,14 +2,33 @@ You can use a local observability stack, framework is connected to it by default +Default mode includes: +- Prometheus +- Loki +- OTEL-collector +- Grafana ```bash +# start the observability stack ctf obs up +# remove the stack with all the data (volumes) +ctf obs d +# restart the stack removing all the data (volumes) +ctf obs r ``` -To remove it use +Full stack has all the services above but also adds: +- Cadvisor +- Tempo +- Pyroscope +- PostgreSQL exporters for NodeSet databases ```bash -ctf obs down +# start the observability stack +ctf obs up -f +# remove the stack with all the data (volumes) +ctf obs d -f +# restart the stack removing all the data (volumes) +ctf obs r -f ``` Read more about how to check [logs](logs.md) and [profiles](profiling.md) diff --git a/framework/.changeset/v0.10.14.md b/framework/.changeset/v0.10.14.md new file mode 100644 index 000000000..fb08966c6 --- /dev/null +++ b/framework/.changeset/v0.10.14.md @@ -0,0 +1 @@ +- Split the observability stack into two modes: full and standard \ No newline at end of file diff --git a/framework/cmd/main.go b/framework/cmd/main.go index e08e59906..25fadb087 100644 --- a/framework/cmd/main.go +++ b/framework/cmd/main.go @@ -68,28 +68,61 @@ func main() { Usage: "Spins up a local observability stack: Grafana, Loki, Pyroscope", Subcommands: []*cli.Command{ { - Name: "up", - Usage: "ctf obs up", - Aliases: []string{"u"}, - Description: "Spins up a local observability stack: Grafana, Loki, Pyroscope", - Action: func(c *cli.Context) error { return framework.ObservabilityUp() }, + Name: "up", + Usage: "ctf obs up", + Aliases: []string{"u"}, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "full", + Aliases: []string{"f"}, + Usage: "Spin up all the observability services", + Value: false, + }, + }, + Description: "Spins up a local observability stack. Has two modes, standard (Loki, Prometheus, Grafana and OTEL) and full including also Tempo, Cadvisor and PostgreSQL metrics", + Action: func(c *cli.Context) error { + if c.Bool("full") { + return framework.ObservabilityUpFull() + } + return framework.ObservabilityUp() + }, }, { - Name: "down", - Usage: "ctf obs down", - Aliases: []string{"d"}, + Name: "down", + Usage: "ctf obs down", + Aliases: []string{"d"}, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "full", + Aliases: []string{"f"}, + Usage: "Removes all the observability services (this flag exists for compatibility, all the services are always removed with 'down')", + Value: false, + }, + }, Description: "Removes local observability stack", Action: func(c *cli.Context) error { return framework.ObservabilityDown() }, }, { - Name: "restart", - Usage: "ctf obs r", - Aliases: []string{"r"}, + Name: "restart", + Usage: "ctf obs r", + Aliases: []string{"r"}, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "full", + Aliases: []string{"f"}, + Usage: "Restart all observability services (this flag exists for compatibility, all the services are always removed with 'down')", + Value: false, + }, + }, Description: "Restart a local observability stack", Action: func(c *cli.Context) error { + // always remove all the containers and volumes to clean up the data if err := framework.ObservabilityDown(); err != nil { return err } + if c.Bool("full") { + return framework.ObservabilityUpFull() + } return framework.ObservabilityUp() }, }, diff --git a/framework/observability.go b/framework/observability.go index f1364c2d5..a3edf05dd 100644 --- a/framework/observability.go +++ b/framework/observability.go @@ -124,6 +124,30 @@ func ObservabilityUp() error { if err := NewPromtail(); err != nil { return err } + err := RunCommand("bash", "-c", fmt.Sprintf(` + cd %s && \ + docker compose up -d otel-collector prometheus loki grafana + `, "compose")) + if err != nil { + return err + } + fmt.Println() + L.Info().Msgf("Loki: %s", LocalLogsURL) + L.Info().Msgf("Prometheus: %s", LocalPrometheusURL) + L.Info().Msgf("CL Node Errors: %s", LocalCLNodeErrorsURL) + L.Info().Msgf("Workflow Engine: %s", LocalWorkflowEngineURL) + return nil +} + +func ObservabilityUpFull() error { + L.Info().Msg("Creating full local observability stack") + if err := extractAllFiles("observability"); err != nil { + return err + } + _ = DefaultNetwork(nil) + if err := NewPromtail(); err != nil { + return err + } err := RunCommand("bash", "-c", fmt.Sprintf(` cd %s && \ docker compose up -d diff --git a/framework/observability/compose/docker-compose.yaml b/framework/observability/compose/docker-compose.yaml index fc1103852..ae915b3cf 100644 --- a/framework/observability/compose/docker-compose.yaml +++ b/framework/observability/compose/docker-compose.yaml @@ -82,7 +82,6 @@ services: depends_on: - prometheus - loki - - tempo pyroscope: image: 'grafana/pyroscope:1.13.4' diff --git a/framework/observability/dashboards/dummy.json b/framework/observability/dashboards/dummy.json new file mode 100644 index 000000000..b04105f51 --- /dev/null +++ b/framework/observability/dashboards/dummy.json @@ -0,0 +1,3 @@ +{ + "example": "This file is needed for local testing so embedding can work." +} \ No newline at end of file