Prometheus: scrape tor instance

Blockstream · Jan 2, 2019 · f333851 · f333851
1 parent efc16f0
commit f333851
Show file tree

Hide file tree

Showing 7 changed files with 183 additions and 22 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -6,6 +6,7 @@ variables:
 image: blockstream/gcloud-docker@sha256:31c1a01d143558f0ba5677d121891a958fa600195679fe325980ec72e5264f2a
 stages:
   - build
+  - plan
   - deploy
 
 before_script:
@@ -70,6 +71,37 @@ build:
           -t blockstream/esplora:$CI_COMMIT_SHA .
         && docker push blockstream/esplora:$CI_COMMIT_SHA)
 
+plan:
+  except:
+    - /^bitcoin_mainnet.*/
+    - /^bitcoin_testnet.*/
+    - /^liquid_mainnet.*/
+    - master@greenaddress/esplora
+    - schedules
+  only:
+    - branches@greenaddress/esplora
+  stage: plan
+  image:
+    name: blockstream/gcloud-docker@sha256:31c1a01d143558f0ba5677d121891a958fa600195679fe325980ec72e5264f2a
+    entrypoint: [""]
+  script:
+    - (echo -n "$V2_PK" > terraform/modules/tor/v2.pk)
+    - (echo -n "$V3_PK" > terraform/modules/tor/v3.pk)
+    - (echo -n "$V3_PUBK" > terraform/modules/tor/v3.pubk)
+    - (cd terraform && terraform init -input=false &&
+       terraform workspace select main &&
+       terraform plan
+        -var "prometheus_allowed_source_ip=$PROMETHEUS_ALLOWED_SOURCE_IP"
+        -var "hosts=$HOSTS"
+        -var "hosts_onion=$HOSTS_ONION"
+        -var "cluster_size=$NODE_CLUSTER_SIZE"
+        -var "instance_type=$NODE_INSTANCE_TYPE"
+        -var "regions=$REGIONS"
+        -var "zones=$ZONES"
+        -var "ssl_certs=$SSL_CERTS"
+        -var "opsgenie_api_key=$OPSGENIE_API_KEY"
+        -input=false)
+
 deploy:
   except:
     - schedules
@@ -83,9 +115,9 @@ deploy:
     - (echo -n "$V2_PK" > terraform/modules/tor/v2.pk)
     - (echo -n "$V3_PK" > terraform/modules/tor/v3.pk)
     - (echo -n "$V3_PUBK" > terraform/modules/tor/v3.pubk)
-    - (cd terraform && terraform init -input=false)
-    - (cd terraform && terraform workspace select main)
-    - (cd terraform && terraform apply
+    - (cd terraform && terraform init -input=false &&
+       terraform workspace select main &&
+       terraform apply
         -var "prometheus_allowed_source_ip=$PROMETHEUS_ALLOWED_SOURCE_IP"
         -var "hosts=$HOSTS"
         -var "hosts_onion=$HOSTS_ONION"

diff --git a/terraform/main.tf b/terraform/main.tf
@@ -35,20 +35,22 @@ module "prometheus" {
 module "tor" {
   source = "modules/tor"
 
-  name             = "explorer-tor"
-  network          = "default"
-  zones            = "${var.zones[0]}"
-  region           = "${var.regions[0]}"
-  instances        = 1
-  project          = "${var.project}"
-  tor_machine_type = "${var.instance_type[3]}"
-  tor_lb           = "${element(concat(google_compute_global_address.onion-lb.*.address, list("")), 0)}"
-  docker_tag       = "${var.docker_tag_tor}"
-  hosts_onion      = "${var.hosts_onion}"
-  kms_key          = "${element(concat(google_kms_crypto_key.esplora-crypto-key.*.name, list("")), 0)}"
-  kms_key_link     = "${element(concat(google_kms_crypto_key.esplora-crypto-key.*.self_link, list("")), 0)}"
-  kms_key_ring     = "${element(concat(google_kms_key_ring.esplora-key-ring.*.name, list("")), 0)}"
-  kms_location     = "${var.kms_location}"
+  name                     = "explorer-tor"
+  network                  = "default"
+  zones                    = "${var.zones[0]}"
+  region                   = "${var.regions[0]}"
+  instances                = 1
+  project                  = "${var.project}"
+  tor_machine_type         = "${var.instance_type[3]}"
+  tor_lb                   = "${element(concat(google_compute_global_address.onion-lb.*.address, list("")), 0)}"
+  docker_tag               = "${var.docker_tag_tor}"
+  hosts_onion              = "${var.hosts_onion}"
+  kms_key                  = "${element(concat(google_kms_crypto_key.esplora-crypto-key.*.name, list("")), 0)}"
+  kms_key_link             = "${element(concat(google_kms_crypto_key.esplora-crypto-key.*.self_link, list("")), 0)}"
+  kms_key_ring             = "${element(concat(google_kms_key_ring.esplora-key-ring.*.name, list("")), 0)}"
+  kms_location             = "${var.kms_location}"
+  service_account_prom     = "${terraform.workspace == "main" ? module.prometheus.service_account : data.terraform_remote_state.main.prometheus_service_account}"
+  docker_tag_node_exporter = "${var.docker_tag_node_exporter}"
 
   create_resources = "${local.create_main}"
 }

diff --git a/terraform/modules/prometheus/cloud-init/prometheus.yml b/terraform/modules/prometheus/cloud-init/prometheus.yml
@@ -155,6 +155,70 @@ write_files:
               zone: us-east1-c
               filter: (labels.type = "prometheus")
               port: 9100
+        - job_name: tor
+          relabel_configs:
+            - source_labels:
+              - '__meta_gce_label_network'
+              target_label: 'network'
+            - source_labels:
+              - '__meta_gce_label_name'
+              target_label: 'name'
+            - source_labels:
+              - '__meta_gce_instance_name'
+              target_label: 'instance_name'
+          gce_sd_configs:
+            - project: green-address-explorer
+              zone: asia-northeast1-a
+              filter: (labels.type = "tor")
+              port: 9100
+            - project: green-address-explorer
+              zone: asia-northeast1-b
+              filter: (labels.type = "tor")
+              port: 9100
+            - project: green-address-explorer
+              zone: asia-northeast1-c
+              filter: (labels.type = "tor")
+              port: 9100
+            - project: green-address-explorer
+              zone: europe-west4-a
+              filter: (labels.type = "tor")
+              port: 9100
+            - project: green-address-explorer
+              zone: europe-west4-b
+              filter: (labels.type = "tor")
+              port: 9100
+            - project: green-address-explorer
+              zone: europe-west4-c
+              filter: (labels.type = "tor")
+              port: 9100
+            - project: green-address-explorer
+              zone: us-central1-a
+              filter: (labels.type = "tor")
+              port: 9100
+            - project: green-address-explorer
+              zone: us-central1-b
+              filter: (labels.type = "tor")
+              port: 9100
+            - project: green-address-explorer
+              zone: us-central1-c
+              filter: (labels.type = "tor")
+              port: 9100
+            - project: green-address-explorer
+              zone: us-central1-f
+              filter: (labels.type = "tor")
+              port: 9100
+            - project: green-address-explorer
+              zone: us-east1-d
+              filter: (labels.type = "tor")
+              port: 9100
+            - project: green-address-explorer
+              zone: us-east1-b
+              filter: (labels.type = "tor")
+              port: 9100
+            - project: green-address-explorer
+              zone: us-east1-c
+              filter: (labels.type = "tor")
+              port: 9100
         - job_name: http-tor
           relabel_configs:
             - source_labels:

diff --git a/terraform/modules/tor/cloud-init/tor.yaml b/terraform/modules/tor/cloud-init/tor.yaml
@@ -142,6 +142,38 @@ write_files:
         [Install]
         WantedBy=multi-user.target
 
+  - path: /etc/systemd/system/node-exporter.service
+    permissions: 0644
+    owner: root
+    content: |
+        [Unit]
+        Description=prometheus node-exporter
+        Wants=gcr-online.target docker.service
+        After=gcr-online.service docker.service
+
+        [Service]
+        Restart=always
+        RestartSec=1
+        Environment=HOME=/home/exec
+        ExecStartPre=/usr/bin/docker-credential-gcr configure-docker
+        ExecStartPre=/usr/bin/docker pull ${docker_tag_node_exporter}
+        ExecStartPre=/sbin/iptables -A INPUT -m tcp -p tcp --dport 9100 -j ACCEPT
+        ExecStart=/usr/bin/docker run \
+            --name=node-exporter \
+            --network=host \
+            --read-only \
+            -v /proc:/host/proc:ro \
+            -v /sys:/host/sys:ro \
+            -v /:/rootfs:ro \
+            -v metrics:/metrics:ro \
+            "${docker_tag_node_exporter}" --path.procfs /host/proc --path.sysfs /host/sys --collector.textfile.directory /metrics --collector.filesystem.ignored-mount-points "^/(sys|proc|dev|host|etc($|/))"
+        ExecStop=/usr/bin/docker stop node-exporter
+        ExecStopPost=/usr/bin/docker rm node-exporter
+        ExecStopPost=/sbin/iptables -D INPUT -m tcp -p tcp --dport 9100 -j ACCEPT
+
+        [Install]
+        WantedBy=multi-user.target
+
 runcmd:
   - systemctl daemon-reload
   - base64 -d /home/bs/tor/hidden_service_v3/hs_ed25519_public_key.enc.b64 > /home/bs/tor/hidden_service_v3/hs_ed25519_public_key.enc
@@ -151,3 +183,5 @@ runcmd:
   - systemctl enable decrypt.service
   - systemctl start decrypt.service
   - systemctl start tor.service
+  - systemctl enable node-exporter.service
+  - systemctl start node-exporter.service
diff --git a/terraform/modules/tor/data.tf b/terraform/modules/tor/data.tf
@@ -13,11 +13,12 @@ data "template_file" "tor" {
     v3_pk   = "${file("${path.module}/v3.pk")}"
     v3_pubk = "${file("${path.module}/v3.pubk")}"
 
-    docker_tag        = "${var.docker_tag}"
-    docker_tag_gcloud = "${var.docker_tag_gcloud}"
-    kms_key           = "${var.kms_key}"
-    kms_key_ring      = "${var.kms_key_ring}"
-    kms_location      = "${var.kms_location}"
+    docker_tag               = "${var.docker_tag}"
+    docker_tag_gcloud        = "${var.docker_tag_gcloud}"
+    kms_key                  = "${var.kms_key}"
+    kms_key_ring             = "${var.kms_key_ring}"
+    kms_location             = "${var.kms_location}"
+    docker_tag_node_exporter = "${var.docker_tag_node_exporter}"
   }
 }
 

diff --git a/terraform/modules/tor/firewall.tf b/terraform/modules/tor/firewall.tf
@@ -15,3 +15,23 @@ resource "google_compute_firewall" "tor-healthcheck" {
     "${google_service_account.tor.email}",
   ]
 }
+
+resource "google_compute_firewall" "prom-traffic" {
+  name    = "tor-${var.name}-prometheus-access"
+  network = "${data.google_compute_network.default.self_link}"
+
+  count = "${var.create_resources}"
+
+  allow {
+    protocol = "tcp"
+    ports    = ["9100"]
+  }
+
+  source_service_accounts = [
+    "${var.service_account_prom}",
+  ]
+
+  target_service_accounts = [
+    "${google_service_account.tor.email}",
+  ]
+}
diff --git a/terraform/modules/tor/variables.tf b/terraform/modules/tor/variables.tf
@@ -68,3 +68,11 @@ variable "kms_key_ring" {
 variable "kms_location" {
   type = "string"
 }
+
+variable "docker_tag_node_exporter" {
+  type = "string"
+}
+
+variable "service_account_prom" {
+  type = "string"
+}