Merge pull request #1956 from grapl-security/twunderlich/lightstep-de…

…v-mode

Enable lightstep dev mode

docs/development/observability.md

@@ -0,0 +1,15 @@
+# Observability
+
+We currently use [Lightstep](https://lightstep.com) as our observability
+platform.
+
+## Local tracing
+
+1. Go to [lightstep.com](https://lightstep.com)
+1. Log into lightstep using google.
+1. On the left-hand side menu go to developer mode (the angle brackets `< >`).
+1. Copy the command and run that locally. This will spin up a docker container
+   configured with an api key. Any data submitted will be forwarded to
+   Lightstep.
+1. Run `make up`. Once everything is up, check the Lightstep developer mode
+   page. You should start seeing traces appear on the page.

pulumi/grapl/Pulumi.local-grapl.yaml

@@ -29,6 +29,8 @@ config:
     firecracker_kernel.tar.gz: firecracker-v1.0.0-kernel-4.14.174-c595a79
     firecracker_rootfs.tar.gz: 20220405165304-c595a79
   grapl:confluent-environment-name: "local-grapl"
+  grapl:lightstep-endpoint: 'http://{{ env "attr.unique.network.ip-address" }}:8360'
+  grapl:lightstep-is-endpoint-secure: "false"
   grapl:scylla-username: "cassandra"
   grapl:scylla-password: "cassandra" # In prod this is a secret!
   grapl:scylla-addresses:

pulumi/grapl/Pulumi.testing.yaml

@@ -6,6 +6,7 @@ config:
   grapl:env_vars:
   grapl:lightstep-access-token:
     secure: AAABAKxa4yxFSvciEOaqrBtoUWo6qQUGrDeba1ZGrxfW/yToz6ZjXXMrGCC+69aCZvTZhzWif6r+S2fEJJGbJz8WDuje825m0wxGI7hBOsrjPw/gIJ/huFVoTM7mOgRcPZwzkVaVN0ePtdkFQJOb/PD5dOgcEf+CCcYpRthO8T9YEzVowm9lzg==
+  grapl:lightstep-endpoint: "ingest.lightstep.com:443"
   grapl:postgres-instance-type: db.t4g.small
   grapl:postgres-version: "13.4"
   grapl:scylla-addresses:

pulumi/grapl/__main__.py

@@ -314,7 +314,13 @@ def main() -> None:
     lightstep_access_token = pulumi.Output.secret(
         pulumi_config.get(key="lightstep-access-token") or ""
     )
-    otel_configuration = otel_config(lightstep_access_token)
+    lightstep_endpoint = pulumi_config.require(key="lightstep-endpoint")
+    lightstep_is_endpoint_secure = (
+        pulumi_config.get(key="lightstep-is-endpoint-secure") or "true"
+    )
+    otel_configuration = otel_config(
+        lightstep_access_token, lightstep_endpoint, lightstep_is_endpoint_secure
+    )
     NomadJob(
         "otel-collector",
         jobspec=repository_path("nomad/observability.nomad"),

pulumi/infra/observability_env_vars.py

@@ -31,8 +31,16 @@ def observability_env_vars_for_local() -> str:
 
 # lightstep_token should be pulumi.Output[str], but the additional type causes pulumi.Output.all to blow up during
 # typechecking
-def otel_config(lightstep_token: pulumi.Output) -> pulumi.Output[str]:
-    return pulumi.Output.all(lightstep_token=lightstep_token).apply(
+def otel_config(
+    lightstep_token: pulumi.Output,
+    lightstep_endpoint: str = "ingest.lightstep.com:443",
+    lightstep_is_endpoint_secure: str = "true",
+) -> pulumi.Output[str]:
+    return pulumi.Output.all(
+        lightstep_endpoint=lightstep_endpoint,
+        lightstep_token=lightstep_token,
+        lightstep_is_endpoint_secure=lightstep_is_endpoint_secure,
+    ).apply(
         lambda args: f"""
 receivers:
   zipkin:
@@ -58,7 +66,9 @@ def otel_config(lightstep_token: pulumi.Output) -> pulumi.Output[str]:
   logging:
     logLevel: debug
   otlp/ls:
-    endpoint: ingest.lightstep.com:443
+    endpoint: {args['lightstep_endpoint']}
+    tls:
+      insecure: {args['lightstep_is_endpoint_secure']}
     headers:
       "lightstep-access-token": {args['lightstep_token']}
 service: