From 1c4f021d329e32e97a3f8d1b18c6991e879d4639 Mon Sep 17 00:00:00 2001 From: Thomas Wunderlich Date: Thu, 1 Sep 2022 15:53:39 -0400 Subject: [PATCH 1/6] Set up passing in the dev lightstep endpoint Dev satellites should always be localhost. This also primes the way to use private satellites --- pulumi/grapl/Pulumi.local-grapl.yaml | 1 + pulumi/grapl/Pulumi.testing.yaml | 1 + pulumi/grapl/__main__.py | 3 ++- pulumi/infra/observability_env_vars.py | 10 +++++++--- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pulumi/grapl/Pulumi.local-grapl.yaml b/pulumi/grapl/Pulumi.local-grapl.yaml index 19df97e85a..bfbc7131f9 100644 --- a/pulumi/grapl/Pulumi.local-grapl.yaml +++ b/pulumi/grapl/Pulumi.local-grapl.yaml @@ -29,6 +29,7 @@ config: firecracker_kernel.tar.gz: firecracker-v1.0.0-kernel-4.14.174-c595a79 firecracker_rootfs.tar.gz: 20220405165304-c595a79 grapl:confluent-environment-name: "local-grapl" + grapl:lightstep-endpoint: "http://localhost:8360" grapl:scylla-username: "cassandra" grapl:scylla-password: "cassandra" # In prod this is a secret! grapl:scylla-addresses: diff --git a/pulumi/grapl/Pulumi.testing.yaml b/pulumi/grapl/Pulumi.testing.yaml index 16f7ca7221..b369ccdf97 100644 --- a/pulumi/grapl/Pulumi.testing.yaml +++ b/pulumi/grapl/Pulumi.testing.yaml @@ -6,6 +6,7 @@ config: grapl:env_vars: grapl:lightstep-access-token: secure: AAABAKxa4yxFSvciEOaqrBtoUWo6qQUGrDeba1ZGrxfW/yToz6ZjXXMrGCC+69aCZvTZhzWif6r+S2fEJJGbJz8WDuje825m0wxGI7hBOsrjPw/gIJ/huFVoTM7mOgRcPZwzkVaVN0ePtdkFQJOb/PD5dOgcEf+CCcYpRthO8T9YEzVowm9lzg== + grapl:lightstep-endpoint: "ingest.lightstep.com:443" grapl:postgres-instance-type: db.t4g.small grapl:postgres-version: "13.4" grapl:scylla-addresses: diff --git a/pulumi/grapl/__main__.py b/pulumi/grapl/__main__.py index 42e536cfff..4379328058 100644 --- a/pulumi/grapl/__main__.py +++ b/pulumi/grapl/__main__.py @@ -314,7 +314,8 @@ def main() -> None: lightstep_access_token = pulumi.Output.secret( pulumi_config.get(key="lightstep-access-token") or "" ) - otel_configuration = otel_config(lightstep_access_token) + lightstep_endpoint = pulumi_config.require(key="lightstep-endpoint") + otel_configuration = otel_config(lightstep_access_token, lightstep_endpoint) NomadJob( "otel-collector", jobspec=repository_path("nomad/observability.nomad"), diff --git a/pulumi/infra/observability_env_vars.py b/pulumi/infra/observability_env_vars.py index b9d74e7428..3ccdf20a58 100644 --- a/pulumi/infra/observability_env_vars.py +++ b/pulumi/infra/observability_env_vars.py @@ -31,8 +31,12 @@ def observability_env_vars_for_local() -> str: # lightstep_token should be pulumi.Output[str], but the additional type causes pulumi.Output.all to blow up during # typechecking -def otel_config(lightstep_token: pulumi.Output) -> pulumi.Output[str]: - return pulumi.Output.all(lightstep_token=lightstep_token).apply( +def otel_config( + lightstep_token: pulumi.Output, lightstep_endpoint: str = "ingest.lightstep.com:443" +) -> pulumi.Output[str]: + return pulumi.Output.all( + lightstep_endpoint=lightstep_endpoint, lightstep_token=lightstep_token + ).apply( lambda args: f""" receivers: zipkin: @@ -58,7 +62,7 @@ def otel_config(lightstep_token: pulumi.Output) -> pulumi.Output[str]: logging: logLevel: debug otlp/ls: - endpoint: ingest.lightstep.com:443 + endpoint: {args['lightstep_endpoint']} headers: "lightstep-access-token": {args['lightstep_token']} service: From 51124b8eac6bfab05d0ded1f0e9394763c07a84a Mon Sep 17 00:00:00 2001 From: Thomas Wunderlich Date: Thu, 1 Sep 2022 15:54:55 -0400 Subject: [PATCH 2/6] WIP docs for Lightstep dev --- docs/development/observability.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 docs/development/observability.md diff --git a/docs/development/observability.md b/docs/development/observability.md new file mode 100644 index 0000000000..7fff1e74d7 --- /dev/null +++ b/docs/development/observability.md @@ -0,0 +1,9 @@ +## Observability + +We currently use [Lightstep](www.lightstep.com) as our observability platform. + +## Local tracing + +Log into lightstep using google On the left-hand side menu go to developer mode +(the angle brackets `< >`) Copy the command and run that locally This will spin +up a docker container From daedce3a19ea8d9a0769819cc68936a0a663df0e Mon Sep 17 00:00:00 2001 From: Thomas Wunderlich Date: Tue, 6 Sep 2022 10:28:49 -0400 Subject: [PATCH 3/6] Fix TLS error in Lightstep dev workflow The lightstep dev container doesn't have a TLS endpoint by default. As such, the quickest way to get this working is to allow insecure transmission to the local container. Lng-term we should figure out how to get TLS active for this workflow --- pulumi/grapl/Pulumi.local-grapl.yaml | 2 +- pulumi/infra/observability_env_vars.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/pulumi/grapl/Pulumi.local-grapl.yaml b/pulumi/grapl/Pulumi.local-grapl.yaml index bfbc7131f9..f7f2beeb47 100644 --- a/pulumi/grapl/Pulumi.local-grapl.yaml +++ b/pulumi/grapl/Pulumi.local-grapl.yaml @@ -29,7 +29,7 @@ config: firecracker_kernel.tar.gz: firecracker-v1.0.0-kernel-4.14.174-c595a79 firecracker_rootfs.tar.gz: 20220405165304-c595a79 grapl:confluent-environment-name: "local-grapl" - grapl:lightstep-endpoint: "http://localhost:8360" + grapl:lightstep-endpoint: 'http://{{ env "attr.unique.network.ip-address" }}:8360' grapl:scylla-username: "cassandra" grapl:scylla-password: "cassandra" # In prod this is a secret! grapl:scylla-addresses: diff --git a/pulumi/infra/observability_env_vars.py b/pulumi/infra/observability_env_vars.py index 3ccdf20a58..0685ff24df 100644 --- a/pulumi/infra/observability_env_vars.py +++ b/pulumi/infra/observability_env_vars.py @@ -34,8 +34,15 @@ def observability_env_vars_for_local() -> str: def otel_config( lightstep_token: pulumi.Output, lightstep_endpoint: str = "ingest.lightstep.com:443" ) -> pulumi.Output[str]: + # use the endpoint as a way to figure out if we're in local dev or not + if lightstep_endpoint == "ingest.lightstep.com:443": + is_endpoint_insecure = "false" + else: + is_endpoint_insecure = "true" return pulumi.Output.all( - lightstep_endpoint=lightstep_endpoint, lightstep_token=lightstep_token + lightstep_endpoint=lightstep_endpoint, + lightstep_token=lightstep_token, + is_endpoint_insecure=is_endpoint_insecure, ).apply( lambda args: f""" receivers: @@ -63,6 +70,8 @@ def otel_config( logLevel: debug otlp/ls: endpoint: {args['lightstep_endpoint']} + tls: + insecure: {args['is_endpoint_insecure']} headers: "lightstep-access-token": {args['lightstep_token']} service: From 297b406b807e2f4a03920dd4a57c187f11b5eee0 Mon Sep 17 00:00:00 2001 From: Thomas Wunderlich Date: Tue, 6 Sep 2022 10:40:16 -0400 Subject: [PATCH 4/6] Update Lightstep dev docs --- docs/development/observability.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/development/observability.md b/docs/development/observability.md index 7fff1e74d7..906c835c11 100644 --- a/docs/development/observability.md +++ b/docs/development/observability.md @@ -1,9 +1,14 @@ -## Observability +# Observability We currently use [Lightstep](www.lightstep.com) as our observability platform. ## Local tracing -Log into lightstep using google On the left-hand side menu go to developer mode -(the angle brackets `< >`) Copy the command and run that locally This will spin -up a docker container +1. Go to [www.lightstep.com](www.lightstep.com) +1. Log into lightstep using google. +1. On the left-hand side menu go to developer mode (the angle brackets `< >`). +1. Copy the command and run that locally. This will spin up a docker container + configured with an api key. Any data submitted will be forwarded to + Lightstep. +1. Run `make up`. Once everything is up, check the Lightstep developer mode + page. You should start seeing traces appear on the page. From ec62c745efd22ef4ec4bc66caa12fe86b9f7492d Mon Sep 17 00:00:00 2001 From: Thomas Wunderlich Date: Tue, 6 Sep 2022 11:34:57 -0400 Subject: [PATCH 5/6] Pass in whether endpoint is secure instead of extrapolating it --- pulumi/grapl/Pulumi.local-grapl.yaml | 1 + pulumi/grapl/__main__.py | 7 ++++++- pulumi/infra/observability_env_vars.py | 13 +++++-------- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/pulumi/grapl/Pulumi.local-grapl.yaml b/pulumi/grapl/Pulumi.local-grapl.yaml index f7f2beeb47..fb8d9e7fc1 100644 --- a/pulumi/grapl/Pulumi.local-grapl.yaml +++ b/pulumi/grapl/Pulumi.local-grapl.yaml @@ -30,6 +30,7 @@ config: firecracker_rootfs.tar.gz: 20220405165304-c595a79 grapl:confluent-environment-name: "local-grapl" grapl:lightstep-endpoint: 'http://{{ env "attr.unique.network.ip-address" }}:8360' + grapl:lightstep-is-endpoint-secure: "false" grapl:scylla-username: "cassandra" grapl:scylla-password: "cassandra" # In prod this is a secret! grapl:scylla-addresses: diff --git a/pulumi/grapl/__main__.py b/pulumi/grapl/__main__.py index 4379328058..904c354706 100644 --- a/pulumi/grapl/__main__.py +++ b/pulumi/grapl/__main__.py @@ -315,7 +315,12 @@ def main() -> None: pulumi_config.get(key="lightstep-access-token") or "" ) lightstep_endpoint = pulumi_config.require(key="lightstep-endpoint") - otel_configuration = otel_config(lightstep_access_token, lightstep_endpoint) + lightstep_is_endpoint_secure = ( + pulumi_config.get(key="lightstep-is-endpoint-secure") or "true" + ) + otel_configuration = otel_config( + lightstep_access_token, lightstep_endpoint, lightstep_is_endpoint_secure + ) NomadJob( "otel-collector", jobspec=repository_path("nomad/observability.nomad"), diff --git a/pulumi/infra/observability_env_vars.py b/pulumi/infra/observability_env_vars.py index 0685ff24df..fe9e4fc9bf 100644 --- a/pulumi/infra/observability_env_vars.py +++ b/pulumi/infra/observability_env_vars.py @@ -32,17 +32,14 @@ def observability_env_vars_for_local() -> str: # lightstep_token should be pulumi.Output[str], but the additional type causes pulumi.Output.all to blow up during # typechecking def otel_config( - lightstep_token: pulumi.Output, lightstep_endpoint: str = "ingest.lightstep.com:443" + lightstep_token: pulumi.Output, + lightstep_endpoint: str = "ingest.lightstep.com:443", + lightstep_is_endpoint_secure: str = "true", ) -> pulumi.Output[str]: - # use the endpoint as a way to figure out if we're in local dev or not - if lightstep_endpoint == "ingest.lightstep.com:443": - is_endpoint_insecure = "false" - else: - is_endpoint_insecure = "true" return pulumi.Output.all( lightstep_endpoint=lightstep_endpoint, lightstep_token=lightstep_token, - is_endpoint_insecure=is_endpoint_insecure, + lightstep_is_endpoint_secure=lightstep_is_endpoint_secure, ).apply( lambda args: f""" receivers: @@ -71,7 +68,7 @@ def otel_config( otlp/ls: endpoint: {args['lightstep_endpoint']} tls: - insecure: {args['is_endpoint_insecure']} + insecure: {args['lightstep_is_endpoint_secure']} headers: "lightstep-access-token": {args['lightstep_token']} service: From 816963ed6791d48d0c6ae525dfc03759356f300c Mon Sep 17 00:00:00 2001 From: Thomas Wunderlich Date: Tue, 6 Sep 2022 11:48:00 -0400 Subject: [PATCH 6/6] Fix build-docs error --- docs/development/observability.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/development/observability.md b/docs/development/observability.md index 906c835c11..f4454fc2b4 100644 --- a/docs/development/observability.md +++ b/docs/development/observability.md @@ -1,10 +1,11 @@ # Observability -We currently use [Lightstep](www.lightstep.com) as our observability platform. +We currently use [Lightstep](https://lightstep.com) as our observability +platform. ## Local tracing -1. Go to [www.lightstep.com](www.lightstep.com) +1. Go to [lightstep.com](https://lightstep.com) 1. Log into lightstep using google. 1. On the left-hand side menu go to developer mode (the angle brackets `< >`). 1. Copy the command and run that locally. This will spin up a docker container