Using existing secret for configuring alloy

[krisztiansala]

Hi,
The Grafana cloud UI auto-generates Helm values for deploying the Kubernetes monitoring chart, but the values file contains the access token and I don't want to push that to version control.
I tried configuring it with a custom secret, but it's not working well.
This is my whole values file:

# Grafana k8s Monitoring Helm Chart Values
# Documentation: https://grafana.com/docs/grafana-cloud/monitor-infrastructure/kubernetes-monitoring/configuration/helm-chart-config/helm-chart/

# Cluster configuration
cluster:
  name: my-cluster

# Grafana Cloud destinations
destinations:
  - name: grafana-cloud-metrics
    type: prometheus
    url: https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push
    auth:
      type: basic
      usernameKey: "username-grafana"
      passwordKey: "access-policy-token"    
    secret:
      create: false
      name: grafana-cloud-credentials
      namespace: infra
  - name: grafana-cloud-logs
    type: loki
    url: https://logs-prod-012.grafana.net/loki/api/v1/push
    auth:
      type: basic
      usernameKey: "username"
      passwordKey: "access-policy-token"
    secret:
      create: false
      name: grafana-cloud-credentials
      namespace: infra
  - name: gc-otlp-endpoint
    type: otlp
    url: https://otlp-gateway-prod-eu-west-2.grafana.net/otlp
    protocol: http
    auth:
      type: basic
      usernameKey: "username"
      passwordKey: "access-policy-token"
    secret:
      create: false
      name: grafana-cloud-credentials
      namespace: infra
    metrics:
      enabled: true
    logs:
      enabled: true
    traces:
      enabled: true
  - name: grafana-cloud-profiles
    type: pyroscope
    url: https://profiles-prod-002.grafana.net:443
    auth:
      type: basic
      usernameKey: "username"
      passwordKey: "access-policy-token"
    secret:
      create: false
      name: grafana-cloud-credentials
      namespace: infra
# Cluster metrics collection
clusterMetrics:
  enabled: true
  opencost:
    enabled: true
    metricsSource: grafana-cloud-metrics
    opencost:
      exporter:
        defaultClusterId: my-cluster
      prometheus:
        existingSecretName: grafana-cloud-credentials
        username_key: "username-grafana"
        password_key: access-policy-token
        external:
          url: https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom
  kepler:
    enabled: true

# Annotation autodiscovery
annotationAutodiscovery:
  enabled: true

# Prometheus operator objects
prometheusOperatorObjects:
  enabled: true

# Cluster events
clusterEvents:
  enabled: true

# Log collection
nodeLogs:
  enabled: true
podLogs:
  enabled: true

# Application observability
applicationObservability:
  enabled: true
  receivers:
    otlp:
      grpc:
        enabled: true
        port: 4317
      http:
        enabled: true
        port: 4318
    zipkin:
      enabled: true
      port: 9411

# Auto instrumentation
autoInstrumentation:
  enabled: true

# Profiling
profiling:
  enabled: true

# Alloy instances configuration
alloy-metrics:
  enabled: true
  alloy:
    extraEnv:
      - name: GCLOUD_RW_API_KEY
        valueFrom:
          secretKeyRef:
            name: grafana-cloud-credentials
            key: access-policy-token
      - name: CLUSTER_NAME
        value: my-cluster
      - name: NAMESPACE
        valueFrom:
          fieldRef:
            fieldPath: metadata.namespace
      - name: POD_NAME
        valueFrom:
          fieldRef:
            fieldPath: metadata.name
      - name: GCLOUD_FM_COLLECTOR_ID
        value: grafana-k8s-monitoring-$(CLUSTER_NAME)-$(NAMESPACE)-$(POD_NAME)
  remoteConfig:
    enabled: true
    url: https://fleet-management-prod-011.grafana.net
    auth:
      type: basic
      usernameKey: "username"
      passwordKey: "access-policy-token"
    secret:
      create: false
      name: grafana-cloud-credentials
      namespace: infra
alloy-singleton:
  enabled: true
  alloy:
    extraEnv:
      - name: GCLOUD_RW_API_KEY
        valueFrom:
          secretKeyRef:
            name: grafana-cloud-credentials
            key: access-policy-token
      - name: CLUSTER_NAME
        value: my-cluster
      - name: NAMESPACE
        valueFrom:
          fieldRef:
            fieldPath: metadata.namespace
      - name: POD_NAME
        valueFrom:
          fieldRef:
            fieldPath: metadata.name
      - name: GCLOUD_FM_COLLECTOR_ID
        value: grafana-k8s-monitoring-$(CLUSTER_NAME)-$(NAMESPACE)-$(POD_NAME)
  remoteConfig:
    enabled: true
    url: https://fleet-management-prod-011.grafana.net
    auth:
      type: basic
      usernameKey: "username"
      passwordKey: "access-policy-token"
    secret:
      create: false
      name: grafana-cloud-credentials
      namespace: infra
alloy-logs:
  enabled: true
  alloy:
    extraEnv:
      - name: GCLOUD_RW_API_KEY
        valueFrom:
          secretKeyRef:
            name: grafana-cloud-credentials
            key: access-policy-token
      - name: CLUSTER_NAME
        value: my-cluster
      - name: NAMESPACE
        valueFrom:
          fieldRef:
            fieldPath: metadata.namespace
      - name: POD_NAME
        valueFrom:
          fieldRef:
            fieldPath: metadata.name
      - name: NODE_NAME
        valueFrom:
          fieldRef:
            fieldPath: spec.nodeName
      - name: GCLOUD_FM_COLLECTOR_ID
        value: grafana-k8s-monitoring-$(CLUSTER_NAME)-$(NAMESPACE)-alloy-logs-$(NODE_NAME)
  remoteConfig:
    enabled: true
    url: https://fleet-management-prod-011.grafana.net
    auth:
      type: basic
      usernameKey: "username"
      passwordKey: "access-policy-token"
    secret:
      create: false
      name: grafana-cloud-credentials
      namespace: infra
alloy-receiver:
  enabled: true
  alloy:
    extraPorts:
      - name: otlp-grpc
        port: 4317
        targetPort: 4317
        protocol: TCP
      - name: otlp-http
        port: 4318
        targetPort: 4318
        protocol: TCP
      - name: zipkin
        port: 9411
        targetPort: 9411
        protocol: TCP
    extraEnv:
      - name: GCLOUD_RW_API_KEY
        valueFrom:
          secretKeyRef:
            name: grafana-cloud-credentials
            key: access-policy-token
      - name: CLUSTER_NAME
        value: my-cluster
      - name: NAMESPACE
        valueFrom:
          fieldRef:
            fieldPath: metadata.namespace
      - name: POD_NAME
        valueFrom:
          fieldRef:
            fieldPath: metadata.name
      - name: NODE_NAME
        valueFrom:
          fieldRef:
            fieldPath: spec.nodeName
      - name: GCLOUD_FM_COLLECTOR_ID
        value: grafana-k8s-monitoring-$(CLUSTER_NAME)-$(NAMESPACE)-alloy-receiver-$(NODE_NAME)
  remoteConfig:
    enabled: true
    url: https://fleet-management-prod-011.grafana.net
    auth:
      type: basic
      usernameKey: "username"
      passwordKey: "access-policy-token"
    secret:
      create: false
      name: grafana-cloud-credentials
      namespace: infra
alloy-profiles:
  enabled: true
  alloy:
    extraEnv:
      - name: GCLOUD_RW_API_KEY
        valueFrom:
          secretKeyRef:
            name: grafana-cloud-credentials
            key: access-policy-token
      - name: CLUSTER_NAME
        value: my-cluster
      - name: NAMESPACE
        valueFrom:
          fieldRef:
            fieldPath: metadata.namespace
      - name: POD_NAME
        valueFrom:
          fieldRef:
            fieldPath: metadata.name
      - name: NODE_NAME
        valueFrom:
          fieldRef:
            fieldPath: spec.nodeName
      - name: GCLOUD_FM_COLLECTOR_ID
        value: grafana-k8s-monitoring-$(CLUSTER_NAME)-$(NAMESPACE)-alloy-profiles-$(NODE_NAME)
  remoteConfig:
    enabled: true
    url: https://fleet-management-prod-011.grafana.net
    auth:
      type: basic
      usernameKey: "username"
      passwordKey: "access-policy-token"
    secret:
      create: false
      name: grafana-cloud-credentials
      namespace: infra

But I'm seeing these errors, for example in the grafana-k8s-monitoring-alloy-logs pod:
Error: /etc/alloy/config.alloy:371:1: Failed to evaluate service: updating service: unauthenticated: authentication error: invalid authentication credentials

370 |
371 | remotecfg {
| _^^^^^^^^^^^
372 | | id = sys.env("GCLOUD_FM_COLLECTOR_ID")
373 | | url = "https://fleet-management-prod-011.grafana.net"
374 | | basic_auth {
375 | | username = convert.nonsensitive(remote.kubernetes.secret.alloy_logs_remote_cfg.data["username"])
376 | | password = remote.kubernetes.secret.alloy_logs_remote_cfg.data["access-policy-token"]
377 | | }
378 | | tls_config {
379 | | insecure_skip_verify = false
380 | | ca_pem = convert.nonsensitive(remote.kubernetes.secret.alloy_logs_remote_cfg.data["ca"])
381 | | cert_pem = convert.nonsensitive(remote.kubernetes.secret.alloy_logs_remote_cfg.data["cert"])
382 | | key_pem = remote.kubernetes.secret.alloy_logs_remote_cfg.data["key"]

If you check my values file, I'm referencing the grafana-cloud-credentials secret, but it's still trying to use alloy_logs_remote_cfg.

What am I doing wrong? I also tried using passwordFrom instead of passwordKey, but it didn't help.

Also, I only want to specify the password like this, I don't want to touch the default values of the SSL certificates. Is this achievable?

[krisztiansala]

It works like this, but I really want to avoid having to specify the token in this file.

# Grafana k8s Monitoring Helm Chart Values
# Documentation: https://grafana.com/docs/grafana-cloud/monitor-infrastructure/kubernetes-monitoring/configuration/helm-chart-config/helm-chart/

# Cluster configuration
cluster:
  name: my-cluster

# Grafana Cloud destinations
destinations:
  - name: grafana-cloud-metrics
    type: prometheus
    url: https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom/push
    auth:
      type: basic
      usernameKey: "username-grafana"
      passwordKey: "access-policy-token"    
    secret:
      create: false
      name: grafana-cloud-credentials
      namespace: infra
  - name: grafana-cloud-logs
    type: loki
    url: https://logs-prod-012.grafana.net/loki/api/v1/push
    auth:
      type: basic
      usernameKey: "username"
      passwordKey: "access-policy-token"
    secret:
      create: false
      name: grafana-cloud-credentials
      namespace: infra
  - name: gc-otlp-endpoint
    type: otlp
    url: https://otlp-gateway-prod-eu-west-2.grafana.net/otlp
    protocol: http
    auth:
      type: basic
      usernameKey: "username"
      passwordKey: "access-policy-token"
    secret:
      create: false
      name: grafana-cloud-credentials
      namespace: infra
    metrics:
      enabled: true
    logs:
      enabled: true
    traces:
      enabled: true
  - name: grafana-cloud-profiles
    type: pyroscope
    url: https://profiles-prod-002.grafana.net:443
    auth:
      type: basic
      usernameKey: "username"
      passwordKey: "access-policy-token"
    secret:
      create: false
      name: grafana-cloud-credentials
      namespace: infra
# Cluster metrics collection
clusterMetrics:
  enabled: true
  opencost:
    enabled: true
    metricsSource: grafana-cloud-metrics
    opencost:
      exporter:
        defaultClusterId: my-cluster
      prometheus:
        existingSecretName: grafana-cloud-credentials
        username_key: "username-grafana"
        password_key: access-policy-token
        external:
          url: https://prometheus-prod-24-prod-eu-west-2.grafana.net/api/prom
  kepler:
    enabled: true

# Annotation autodiscovery
annotationAutodiscovery:
  enabled: true

# Prometheus operator objects
prometheusOperatorObjects:
  enabled: true

# Cluster events
clusterEvents:
  enabled: true

# Log collection
nodeLogs:
  enabled: true
podLogs:
  enabled: true

# Application observability
applicationObservability:
  enabled: true
  receivers:
    otlp:
      grpc:
        enabled: true
        port: 4317
      http:
        enabled: true
        port: 4318
    zipkin:
      enabled: true
      port: 9411

# Auto instrumentation
autoInstrumentation:
  enabled: true

# Profiling
profiling:
  enabled: true

# Alloy instances configuration
alloy-metrics:
  enabled: true
  alloy:
    extraEnv:
      - name: GCLOUD_RW_API_KEY
        valueFrom:
          secretKeyRef:
            name: alloy-metrics-remote-cfg-grafana-k8s-monitoring
            key: password
      - name: CLUSTER_NAME
        value: my-cluster
      - name: NAMESPACE
        valueFrom:
          fieldRef:
            fieldPath: metadata.namespace
      - name: POD_NAME
        valueFrom:
          fieldRef:
            fieldPath: metadata.name
      - name: GCLOUD_FM_COLLECTOR_ID
        value: grafana-k8s-monitoring-$(CLUSTER_NAME)-$(NAMESPACE)-$(POD_NAME)
  remoteConfig:
    enabled: true
    url: https://fleet-management-prod-011.grafana.net
    auth:
      type: basic
      username: "<my_username>"
      password: <my_access_token>
alloy-singleton:
  enabled: true
  alloy:
    extraEnv:
      - name: GCLOUD_RW_API_KEY
        valueFrom:
          secretKeyRef:
            name: alloy-singleton-remote-cfg-grafana-k8s-monitoring
            key: password
      - name: CLUSTER_NAME
        value: my-cluster
      - name: NAMESPACE
        valueFrom:
          fieldRef:
            fieldPath: metadata.namespace
      - name: POD_NAME
        valueFrom:
          fieldRef:
            fieldPath: metadata.name
      - name: GCLOUD_FM_COLLECTOR_ID
        value: grafana-k8s-monitoring-$(CLUSTER_NAME)-$(NAMESPACE)-$(POD_NAME)
  remoteConfig:
    enabled: true
    url: https://fleet-management-prod-011.grafana.net
    auth:
      type: basic
      username: "<my_username>"
      password: <my_access_token>
alloy-logs:
  enabled: true
  alloy:
    extraEnv:
      - name: GCLOUD_RW_API_KEY
        valueFrom:
          secretKeyRef:
            name: alloy-logs-remote-cfg-grafana-k8s-monitoring
            key: password
      - name: CLUSTER_NAME
        value: my-cluster
      - name: NAMESPACE
        valueFrom:
          fieldRef:
            fieldPath: metadata.namespace
      - name: POD_NAME
        valueFrom:
          fieldRef:
            fieldPath: metadata.name
      - name: NODE_NAME
        valueFrom:
          fieldRef:
            fieldPath: spec.nodeName
      - name: GCLOUD_FM_COLLECTOR_ID
        value: grafana-k8s-monitoring-$(CLUSTER_NAME)-$(NAMESPACE)-alloy-logs-$(NODE_NAME)
  remoteConfig:
    enabled: true
    url: https://fleet-management-prod-011.grafana.net
    auth:
      type: basic
      username: "<my_username>"
      password: <my_access_token>
alloy-receiver:
  enabled: true
  alloy:
    extraPorts:
      - name: otlp-grpc
        port: 4317
        targetPort: 4317
        protocol: TCP
      - name: otlp-http
        port: 4318
        targetPort: 4318
        protocol: TCP
      - name: zipkin
        port: 9411
        targetPort: 9411
        protocol: TCP
    extraEnv:
      - name: GCLOUD_RW_API_KEY
        valueFrom:
          secretKeyRef:
            name: alloy-receiver-remote-cfg-grafana-k8s-monitoring
            key: password
      - name: CLUSTER_NAME
        value: my-cluster
      - name: NAMESPACE
        valueFrom:
          fieldRef:
            fieldPath: metadata.namespace
      - name: POD_NAME
        valueFrom:
          fieldRef:
            fieldPath: metadata.name
      - name: NODE_NAME
        valueFrom:
          fieldRef:
            fieldPath: spec.nodeName
      - name: GCLOUD_FM_COLLECTOR_ID
        value: grafana-k8s-monitoring-$(CLUSTER_NAME)-$(NAMESPACE)-alloy-receiver-$(NODE_NAME)
  remoteConfig:
    enabled: true
    url: https://fleet-management-prod-011.grafana.net
    auth:
      type: basic
      username: "<my_username>"
      password: <my_access_token>
alloy-profiles:
  enabled: true
  alloy:
    extraEnv:
      - name: GCLOUD_RW_API_KEY
        valueFrom:
          secretKeyRef:
            name: alloy-profiles-remote-cfg-grafana-k8s-monitoring
            key: password
      - name: CLUSTER_NAME
        value: my-cluster
      - name: NAMESPACE
        valueFrom:
          fieldRef:
            fieldPath: metadata.namespace
      - name: POD_NAME
        valueFrom:
          fieldRef:
            fieldPath: metadata.name
      - name: NODE_NAME
        valueFrom:
          fieldRef:
            fieldPath: spec.nodeName
      - name: GCLOUD_FM_COLLECTOR_ID
        value: grafana-k8s-monitoring-$(CLUSTER_NAME)-$(NAMESPACE)-alloy-profiles-$(NODE_NAME)
  remoteConfig:
    enabled: true
    url: https://fleet-management-prod-011.grafana.net
    auth:
      type: basic
      username: "<my_username>"
      password: <my_access_token>

[rlankfo]

Did you create the pre-existing secret first?

I believe the reference alloy_logs_remote_cfg is to the alloy config object. The helm chart will render a remote.kubernetes.secret block that references your secret by name.

If you take a look at this example, you can see a remote.kubernetes.secret block is created that reference a secret named my-monitoring-secret:

    remote.kubernetes.secret "metrics_service" {
      name      = "my-monitoring-secret"
      namespace = "monitoring"
    }

You can use helm template command to render the output with your values.yaml and check the configuration points to the expected secret.

I notice in some places, you define the usernameKey as username-grafana and in others as username. It also looks like you may be using the same username value for different destinations (e.g. Grafana Cloud logs, metrics, otlp, etc) but these should have different tenant IDs.

Hope this helps, let me know if we can assist troubleshooting further.

[krisztiansala]

I have created the pre-existing secret.

I used two separate usernames, because the grafana cloud UI generated 2 usernames: 265.... (for grafana-cloud-metrics)
and 136..... (for everything else).

I ran the helm template and it's indeed creating the remote.kubernetes.secret blocks.
But I still get the authentication issue. And it's trying to reference the SSL settings from my custom secret, but in that I only specified the username and password. And I don't want to touch the default SSL settings. Is that possible?

This is part of the helm template:

 remote.kubernetes.secret "alloy_logs_remote_cfg" {
      name      = "grafana-cloud-credentials"
      namespace = "infra"
    }
    
    remotecfg {
      id = sys.env("GCLOUD_FM_COLLECTOR_ID")
      url = "https://fleet-management-prod-011.grafana.net"
      basic_auth {
        username = convert.nonsensitive(remote.kubernetes.secret.alloy_logs_remote_cfg.data["username"])
        password = remote.kubernetes.secret.alloy_logs_remote_cfg.data["access-policy-token"]
      }
      tls_config {
        insecure_skip_verify = false
        ca_pem = convert.nonsensitive(remote.kubernetes.secret.alloy_logs_remote_cfg.data["ca"])
        cert_pem = convert.nonsensitive(remote.kubernetes.secret.alloy_logs_remote_cfg.data["cert"])
        key_pem = remote.kubernetes.secret.alloy_logs_remote_cfg.data["key"]
      }
      poll_frequency = "5m"
    }

and this is the error when I apply it:

226 | |   url = "https://fleet-management-prod-011.grafana.net"
227 | |   basic_auth {
228 | |     username = convert.nonsensitive(remote.kubernetes.secret.alloy_logs_remote_cfg.data["username"])
229 | |     password = remote.kubernetes.secret.alloy_logs_remote_cfg.data["access-policy-token"]
230 | |   }
231 | |   tls_config {
232 | |     insecure_skip_verify = false
233 | |     ca_pem = convert.nonsensitive(remote.kubernetes.secret.alloy_logs_remote_cfg.data["ca"])
234 | |     cert_pem = convert.nonsensitive(remote.kubernetes.secret.alloy_logs_remote_cfg.data["cert"])
235 | |     key_pem = remote.kubernetes.secret.alloy_logs_remote_cfg.data["key"]
236 | |   }
237 | |   poll_frequency = "5m"
248 | | }
    | |_^

ts=2025-10-05T06:48:40.162897918Z level=error msg="failed to evaluate config" controller_path=/ controller_id="" node=remotecfg err="updating service: unauthenticated: authentication error: invalid authentication credentials"
Error: could not perform the initial load successfully

[petewall]

It looks like it's only the remote config piece that's having problems authenticating. So if this works:

alloy-metrics:
  enabled: true
  alloy:
    extraEnv: {...}
  remoteConfig:
    enabled: true
    url: https://fleet-management-prod-011.grafana.net
    auth:
      type: basic
      username: "<my_username>"
      password: <my_access_token>

But this doesn't:

alloy-metrics:
  enabled: true
  alloy:
    extraEnv: {...}
  remoteConfig:
    enabled: true
    url: https://fleet-management-prod-011.grafana.net
    auth:
      type: basic
      usernameKey: "username"
      passwordKey: "access-policy-token"
    secret:
      create: false
      name: grafana-cloud-credentials
      namespace: infra

Then either:

1. It's not loading the right secret
2. The secret it's loading is bad or doesn't have the fleet-management:read access policy scope.

[jjaimon]

I'm encountering the same issue now. The chart expects actual secret values in the remoteConfig section, not references from externalSecrets. This is insecure because we can't version control the configuration. How should we resolve this?

[jason-berk-k1x]

my first experience with Grafana Cloud and I've been fighting this issue from step one.... why in the HELL would you generate a values file with a hard coded token like that....this makes NO SENSE.

[dmouse]

Same problem here, I'm trying to migrate my setup from 1.x to 3.x and now is so confusing if you want to use external secrets.

I'm going to test with the example

Update:

No lucky.

When I used this configuration

   destinations:
          - name: grafana-cloud-metrics
            type: prometheus
            url: "https://prometheus.grafana.net/api/prom/push"
            auth:
              type: basic
              usernameFrom: username
              passwordFrom: password
            secret:
              create: false
              name: "grafana"
              namespace: "monitoring"

The alloy log pod shows this error:

alloy | Error: /etc/alloy/config.alloy:250:18: component "username" does not exist or is out of scope
10/21/2025 6:16:18 PM alloy |
10/21/2025 6:16:18 PM alloy |
10/21/2025 6:16:18 PM alloy | 249 |     basic_auth {
10/21/2025 6:16:18 PM alloy | 250 |       username = username
10/21/2025 6:16:18 PM alloy |     |                  ^^^^^^^^
10/21/2025 6:16:18 PM alloy | 251 |       password = password
10/21/2025 6:16:18 PM alloy |
10/21/2025 6:16:18 PM alloy | Error: /etc/alloy/config.alloy:251:18: component "password" does not exist or is out of scope
10/21/2025 6:16:18 PM alloy |
10/21/2025 6:16:18 PM alloy | 250 |       username = username
10/21/2025 6:16:18 PM alloy | 251 |       password = password
10/21/2025 6:16:18 PM alloy |     |                  ^^^^^^^^
10/21/2025 6:16:18 PM alloy | 252 |     }
10/21/2025 6:16:18 PM alloy | interrupt received
10/21/2025 6:16:18 PM alloy | Error: could not perform the initial load successfully
10/21/2025 6:16:44 PM alloy | EOF: No more logs available.

[petewall]

Hi everybody! I've updated the example to show how to use a pre-existing secret for the remote config section. This should work, as it matches what we do with a routine test to ensure that the remote config section works as expected.

I apologize for any confusion you've had with configuring this. Hopefully the updated example makes it clearer.

[jason-berk-k1x]

@petewall - thanks...I'll give this a shot. Silly question, do we have to put the username and password in the secret? when I used the GC UI to generate the values file for my stack, my destinations all have a different username (a number) but use the same password. Seems like the only data I really need to put in the secret is the password......I'll try it out but figured I'd ask here as well

[petewall]

@jason-berk-k1x , the k8s monitoring helm chart tries to stick any "credentials" into the same secret. So there isn't a built-in way to mix and match credential sources, aside from using the ___From fields. Those allow you to directly inject Alloy config.

This might do what you want:

destinations:
  - name: ...
    type: ...
    auth:
      usernameFrom: '"12345"'  # <-- use ' and " to make sure we set the value as a quoted string
      passwordKey: my-access-policy-token
    secret:
      create: false
      name: my-secret-with-token-only

This same pattern should work for all destinations, and the remoteConfig section within Alloy.

I hope this makes sense. I have plans to make a doc about the difference between:

• a field (e.g. username
• its key in a secret (e.g. usernameKey)
• the raw Alloy config override (e.g. usernameFrom)

[jason-berk-k1x]

@petewall thanks for that.... I can confirm this does work:

destinations:
  - name: grafana-cloud-metrics
    type: prometheus
    url: https://blahblahblah./api/prom/push
    auth:
      type: basic
      usernameFrom: '"0000000"'
      passwordKey: token <------------- this is the dataKey of the AKS secret
    secret:
      create: false
      name: grafana-cloud <------------ the name of the AKS secret
      namespace: monitoring

now, it sounds like I could, if I wanted to, add another dataKey into my AKS secret named prom-username and do this:

destinations:
  - name: grafana-cloud-metrics
    type: prometheus
    url: https://blahblahblah./api/prom/push
    auth:
      type: basic
      usernameKey: prom-username <------------- this is one of the dataKeys of the AKS secret
      passwordKey: token <------------------------ this is another of the dataKeys of the AKS secret
    secret:
      create: false
      name: grafana-cloud <------------ the name of the AKS secret
      namespace: monitoring

that line up with you?

[petewall]

Yep. Using both usernameKey and passwordKey means it'll look for those fields within the same grafana-cloud k8s Secret.

[dmouse]

Thanks you for your support @petewall!

For me, the confusion starts in the Grafana Cloud integration dashboard. Enable Remote Configuration is on by default, which creates a remoteConfig alongside each alloy-* config. That remoteConfig includes a different set of auth key–values, and it also references a secretKeyRef pointing to a random Secret name. My first thought was "maybe this is an auto-generated secret based on the destinations config." but no! you need to configure too if you want that option not just the destinations.

Example:

alloy-profiles:
  enabled: true
  alloy:
    extraEnv:
      - name: GCLOUD_RW_API_KEY
        valueFrom:
          secretKeyRef:
            name: alloy-profiles-remote-cfg-grafana-k8s-monitoring
            key: password
      - name: CLUSTER_NAME
        value: develop
      - name: NAMESPACE
        valueFrom:
          fieldRef:
            fieldPath: metadata.namespace
      - name: POD_NAME
        valueFrom:
          fieldRef:
            fieldPath: metadata.name
      - name: NODE_NAME
        valueFrom:
          fieldRef:
            fieldPath: spec.nodeName
      - name: GCLOUD_FM_COLLECTOR_ID
        value: grafana-k8s-monitoring-$(CLUSTER_NAME)-$(NAMESPACE)-alloy-profiles-$(NODE_NAME)
  extraEnv:
    enabled: true
    url: https://fleet-management-prod.grafana.net
    auth:
      type: basic
      username: "user"
      password: "token"

At the end I just removed the extraEnv and extraEnv and just keep the yaml like the example.

Thank you again

[krisztiansala]

Hi @petewall could you please take a look what am I doing wrong? It still doesn't work for me in remoteConfig. This is the relevant part of my values file:

alloy-metrics:
  enabled: true
  scrapeInterval: 60s
  # Drop high-cardinality histogram metrics
  extraMetricProcessingRules: |
    rule {
      source_labels = ["__name__"]
      regex = "alloy_component_dependencies_wait_seconds_bucket|alloy_component_evaluation_seconds_bucket|prometheus_remote_storage_sent_batch_duration_seconds_bucket"
      action = "drop"
    }
  alloy:
    extraEnv:
      - name: GCLOUD_RW_API_KEY
        valueFrom:
          secretKeyRef:
            name: grafana-cloud-credentials
            key: access-token
      - name: NAMESPACE
        valueFrom:
          fieldRef:
            fieldPath: metadata.namespace
      - name: POD_NAME
        valueFrom:
          fieldRef:
            fieldPath: metadata.name
  remoteConfig:
    enabled: true
    url: https://fleet-management-prod-011.grafana.net
    auth:
      type: basic
      usernameKey: "username"
      passwordKey: "access-token"
    secret:
      create: false
      name: grafana-cloud-credentials
      

alloy-singleton:
  enabled: true
  alloy:
    extraEnv:
      - name: GCLOUD_RW_API_KEY
        valueFrom:
          secretKeyRef:
            name: grafana-cloud-credentials
            key: access-token
      - name: NAMESPACE
        valueFrom:
          fieldRef:
            fieldPath: metadata.namespace
      - name: POD_NAME
        valueFrom:
          fieldRef:
            fieldPath: metadata.name
  remoteConfig:
    enabled: true
    url: https://fleet-management-prod-011.grafana.net
    auth:
      type: basic
      usernameKey: "username"
      passwordKey: "access-token"
    secret:
      create: false
      name: grafana-cloud-credentials
      

alloy-logs:
  enabled: true
  alloy:
    extraEnv:
      - name: GCLOUD_RW_API_KEY
        valueFrom:
          secretKeyRef:
            name: grafana-cloud-credentials
            key: access-token
      - name: NAMESPACE
        valueFrom:
          fieldRef:
            fieldPath: metadata.namespace
      - name: POD_NAME
        valueFrom:
          fieldRef:
            fieldPath: metadata.name
      - name: NODE_NAME
        valueFrom:
          fieldRef:
            fieldPath: spec.nodeName
  remoteConfig:
    enabled: true
    url: https://fleet-management-prod-011.grafana.net
    auth:
      type: basic
      usernameKey: "username"
      passwordKey: "access-token"
    secret:
      create: false
      name: grafana-cloud-credentials
      

alloy-receiver:
  enabled: true
  alloy:
    extraPorts:
      - name: otlp-grpc
        port: 4317
        targetPort: 4317
        protocol: TCP
      - name: otlp-http
        port: 4318
        targetPort: 4318
        protocol: TCP
      - name: zipkin
        port: 9411
        targetPort: 9411
        protocol: TCP
    extraEnv:
      - name: GCLOUD_RW_API_KEY
        valueFrom:
          secretKeyRef:
            name: grafana-cloud-credentials
            key: access-token
      - name: NAMESPACE
        valueFrom:
          fieldRef:
            fieldPath: metadata.namespace
      - name: POD_NAME
        valueFrom:
          fieldRef:
            fieldPath: metadata.name
      - name: NODE_NAME
        valueFrom:
          fieldRef:
            fieldPath: spec.nodeName
  remoteConfig:
    enabled: true
    url: https://fleet-management-prod-011.grafana.net
    auth:
      type: basic
      usernameKey: "username"
      passwordKey: "access-token"
    secret:
      create: false
      name: grafana-cloud-credentials
      

alloy-profiles:
  enabled: false
  alloy:
    extraEnv:
      - name: GCLOUD_RW_API_KEY
        valueFrom:
          secretKeyRef:
            name: grafana-cloud-credentials
            key: access-token
      - name: NAMESPACE
        valueFrom:
          fieldRef:
            fieldPath: metadata.namespace
      - name: POD_NAME
        valueFrom:
          fieldRef:
            fieldPath: metadata.name
      - name: NODE_NAME
        valueFrom:
          fieldRef:
            fieldPath: spec.nodeName
  remoteConfig:
    enabled: true
    url: https://fleet-management-prod-011.grafana.net
    auth:
      type: basic
      usernameKey: "username"
      passwordKey: "access-token"
    secret:
      create: false
      name: grafana-cloud-credentials

The grafana-cloud-credentials exists in the namespace where this is deployed and contains the access-token and username values. It doesn't contain any TLS details, as I don't want to manage those. When applying, I get errors like this:

Error: /etc/alloy/config.alloy:224:1: Failed to evaluate service: updating service: unauthenticated: authentication error: invalid authentication credentials


223 |   
224 |   remotecfg {
    |  _^^^^^^^^^^^
225 | |   id = sys.env("GCLOUD_FM_COLLECTOR_ID")
226 | |   url = "https://fleet-management-prod-011.grafana.net"
227 | |   basic_auth {
228 | |     username = convert.nonsensitive(remote.kubernetes.secret.alloy_logs_remote_cfg.data["username"])
229 | |     password = remote.kubernetes.secret.alloy_logs_remote_cfg.data["access-token"]
230 | |   }
231 | |   tls_config {
232 | |     insecure_skip_verify = false
233 | |     ca_pem = convert.nonsensitive(remote.kubernetes.secret.alloy_logs_remote_cfg.data["ca"])
234 | |     cert_pem = convert.nonsensitive(remote.kubernetes.secret.alloy_logs_remote_cfg.data["cert"])
235 | |     key_pem = remote.kubernetes.secret.alloy_logs_remote_cfg.data["key"]
236 | |   }
237 | |   poll_frequency = "5m"
238 | |   attributes = {
239 | |     "cluster" = "***",
240 | |     "namespace" = "***",
241 | |     "platform" = "kubernetes",
242 | |     "release" = "***",
243 | |     "source" = "k8s-monitoring",
244 | |     "sourceVersion" = "3.6.2",
245 | |     "workloadName" = "alloy-logs",
246 | |     "workloadType" = "daemonset",

It's still referencing the wrong secret.

[petewall]

The error:

unauthenticated: authentication error: invalid authentication credentials

tells me that it's finding a secret and using that. But that secret isn't working for remote config. Go into your Grafana Cloud where you can manage your access policy tokens. find the one that you're using and make sure you have the fleet-management:read scope.