Setup datadog (#1462)

* build: instead of PROD, use SHUTTLE_ENV to know in which environment we are

* feat: setup logappender to send logs to datadog, fix configuration

- Since Datadog isn't using the `events` from the span to setup logs, we
  instead rely on the logging pipeline from OpenTelemetry.
- Fix the Otel collector configuration to correctly work with Datadog:
  - Correctly set the environment key (need to use the OPTL way to
    set it, not Datadog).
  - Transform the resource.name from name to have span show the right
    resource. Done both through a transfo and with
    span_name_as_resource_name.
  - Change in our tracing configuration for span to use `service.name`
    instead of `service`, because that's the official OPTL way of doing.

fix(otlp): export to honeycomb still

build: update Cargo.lock

* fix(deployer): improved proxy instrumentation

We shouldn't add everything as attributes to the span from the
arguments. And we should avoid logging the display value of some values,
and instead reach for the string. Examble with the FQDN, we want to have
the hostname, not `FQDN(\x06foo.shuttleapp.rs\n)`.

* ref: remove ShuttleEnv tracing configuration, don't log in JSON

Not necessary anymore since we're not relying on ingesting stdout logs.

* fix(tracing): env attributes not set correctly for Datadog

* fmt: common mods.rs

* fix(otel): remove deployment.environment mapping in collector

This messes up Datadog understanding of this tag, for some reason.

* circleci: add missing env when deploying images

* fmt: common/backends/mod.rs

* feat(gateway/project): set attributes in tracing logs

* typo(gateway/project): fix typo in log statement

Co-authored-by: Iulian Barbu <14218860+iulianbarbu@users.noreply.github.com>

* build: remove patch from Cargo.lock

* fmt(gateway/project): format file

---------

Co-authored-by: Iulian Barbu <14218860+iulianbarbu@users.noreply.github.com>

.circleci/config.yml

@@ -363,10 +363,10 @@ jobs:
         description: "Use public registry"
         type: boolean
         default: true
-      production:
-        description: "Push and deploy to production"
-        type: boolean
-        default: false
+      shuttle-env:
+        description: "Push and deploy to the given env"
+        type: string
+        default: staging
     steps:
       - checkout
       - run:
@@ -384,7 +384,7 @@ jobs:
       - run:
           name: Make and push images
           command: |
-            PUSH=true PROD=<< parameters.production >> PLATFORMS=linux/amd64 TAG=$TAG make images
+            PUSH=true SHUTTLE_ENV=<< parameters.shuttle-env >> PLATFORMS=linux/amd64 TAG=$TAG make images
       - save-buildx-cache
   deploy-images:
     executor: machine-ubuntu
@@ -423,10 +423,10 @@ jobs:
       jwt-signing-private-key:
         description: "Auth private key used for JWT signing"
         type: string
-      production:
-        description: "Push and deploy to production"
-        type: boolean
-        default: false
+      shuttle-env:
+        description: "Push and deploy to the given env"
+        type: string
+        default: staging
     steps:
       - checkout
       - run:
@@ -444,27 +444,27 @@ jobs:
           command: |
             DOCKER_HOST=ssh://ec2-user@master.<< parameters.ssh-host >> \
             TAG=$TAG \
-            PROD=<< parameters.production >> \
+            SHUTTLE_ENV=<< parameters.shuttle-env >> \
             USE_TLS=enable \
             POSTGRES_PASSWORD=${<< parameters.postgres-password >>} \
             MONGO_INITDB_ROOT_PASSWORD=${<< parameters.mongodb-password >>} \
             DD_API_KEY=$DD_API_KEY \
+            DD_ENV=<< parameters.shuttle-env >> \
             HONEYCOMB_API_KEY=${<< parameters.honeycomb-api-key >>} \
             DEPLOYS_API_KEY=${<< parameters.deploys-api-key >>} \
             LOGGER_POSTGRES_URI=${<< parameters.logger-postgres-uri >>} \
             STRIPE_SECRET_KEY=${<< parameters.stripe-secret-key >>} \
             AUTH_JWTSIGNING_PRIVATE_KEY=${<< parameters.jwt-signing-private-key >>} \
             make deploy
       - when:
-          condition: << parameters.production >>
+          condition: << parameters.shuttle-env >> == "production"
           steps:
             - run:
                 name: Pull new deployer image on prod
                 command: |
                   ssh ec2-user@controller.<< parameters.ssh-host >> "docker pull public.ecr.aws/shuttle/deployer:$TAG"
       - when:
-          condition:
-            not: << parameters.production >>
+          condition: << parameters.shuttle-env >> == "staging"
           steps:
             - run:
                 name: Pull new deployer image on dev
@@ -810,7 +810,7 @@ workflows:
           name: build-and-push-unstable
           aws-access-key-id: DEV_AWS_ACCESS_KEY_ID
           aws-secret-access-key: DEV_AWS_SECRET_ACCESS_KEY
-          production: false
+          shuttle-env: staging
           requires:
             - approve-push-unstable
       - deploy-images:
@@ -883,7 +883,7 @@ workflows:
           name: build-and-push-production
           aws-access-key-id: PROD_AWS_ACCESS_KEY_ID
           aws-secret-access-key: PROD_AWS_SECRET_ACCESS_KEY
-          production: true
+          shuttle-env: production
           filters:
             branches:
               only: production
@@ -904,7 +904,7 @@ workflows:
           ssh-fingerprint: 6a:c5:33:fe:5b:c9:06:df:99:64:ca:17:0d:32:18:2e
           ssh-config-script: production-ssh-config.sh
           ssh-host: shuttle.prod.internal
-          production: true
+          shuttle-env: production
           requires:
             - build-and-push-production
             - approve-deploy-production

Cargo.toml

@@ -67,10 +67,12 @@ hyper-reverse-proxy = { git = "https://github.com/chesedo/hyper-reverse-proxy",
 jsonwebtoken = "9.0.0"
 once_cell = "1.16.0"
 opentelemetry = "0.21.0"
-opentelemetry_sdk = { version = "0.21.0", features = ["rt-tokio"] }
+opentelemetry_sdk = { version = "0.21.0", features = ["rt-tokio", "logs"] }
 opentelemetry-http = "0.10.0"
-opentelemetry-otlp = "0.14.0"
+opentelemetry-otlp = { version =  "0.14.0", features = ["logs", "grpc-tonic"] }
 opentelemetry-proto = "0.4.0"
+opentelemetry-contrib = { version = "0.4.0", features = ["datadog"] }
+opentelemetry-appender-tracing = "0.2.0"
 percent-encoding = "2.2"
 pin-project = "1.0.12"
 portpicker = "0.1.1"
@@ -98,9 +100,11 @@ tonic = "0.10.2"
 tower = "0.4.13"
 tower-http = { version = "0.4.0", features = ["trace"] }
 tracing = { version = "0.1.37", default-features = false }
+tracing-core = { version = "0.1.32", default-features = false }
 tracing-opentelemetry = "0.22.0"
 tracing-subscriber = { version = "0.3.16", default-features = false, features = [
   "registry",
+  "json"
 ] }
 ttl_cache = "0.5.1"
 utoipa = { version = "4.0.0", features = [ "uuid", "chrono" ] }

Containerfile

@@ -97,7 +97,7 @@ RUN for target_platform in "linux/arm64" "linux/arm64/v8"; do \
     if [ "$TARGETPLATFORM" = "$target_platform" ]; then \
       mv /usr/lib/ulid0_aarch64.so /usr/lib/ulid0.so; fi; done
 # Used as env variable in prepare script
-ARG PROD
+ARG SHUTTLE_ENV
 # Easy way to check if you are running in Shuttle's container
 ARG SHUTTLE=true
 ENV SHUTTLE=${SHUTTLE}