diff --git a/.travis.yml b/.travis.yml index 418a3f7..a6c4eb9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,7 +44,7 @@ deploy: ############################################################################ # Sandbox. - provider: script - script: $TRAVIS_BUILD_DIR/deploy_prometheus_targets.sh mlab-sandbox global + script: $TRAVIS_BUILD_DIR/deploy_scraper_prometheus_targets.sh mlab-sandbox skip_cleanup: true on: repo: m-lab/operator @@ -54,7 +54,7 @@ deploy: ############################################################################ # Staging, on master. - provider: script - script: $TRAVIS_BUILD_DIR/deploy_prometheus_targets.sh mlab-staging global + script: $TRAVIS_BUILD_DIR/deploy_scraper_prometheus_targets.sh mlab-staging skip_cleanup: true on: repo: m-lab/operator @@ -63,7 +63,7 @@ deploy: ############################################################################ # Production, on tags. - provider: script - script: $TRAVIS_BUILD_DIR/deploy_prometheus_targets.sh mlab-oti global + script: $TRAVIS_BUILD_DIR/deploy_scraper_prometheus_targets.sh mlab-oti skip_cleanup: true on: repo: m-lab/operator diff --git a/deploy_prometheus_targets.sh b/deploy_scraper_prometheus_targets.sh similarity index 86% rename from deploy_prometheus_targets.sh rename to deploy_scraper_prometheus_targets.sh index 5e41b39..a7a2d6f 100755 --- a/deploy_prometheus_targets.sh +++ b/deploy_scraper_prometheus_targets.sh @@ -8,14 +8,13 @@ CACHE_CONTROL="Cache-Control:private, max-age=0, no-transform" USAGE="Usage: $0 " PROJECT=${1:?Please provide project name: $USAGE} -GROUP=${2:?Please provide monitoring group name: $USAGE} # Root directory of this script. SCRIPTDIR=$( dirname "${BASH_SOURCE[0]}" ) BASEDIR=${PWD} # Generate the configs. -${SCRIPTDIR}/generate_prometheus_targets.sh ${GROUP} > /dev/null +${SCRIPTDIR}/generate_scraper_prometheus_targets.sh > /dev/null # Be sure that gcloud is PATH source "${HOME}/google-cloud-sdk/path.bash.inc" diff --git a/generate_prometheus_targets.sh b/generate_prometheus_targets.sh deleted file mode 100755 index e31aa84..0000000 --- a/generate_prometheus_targets.sh +++ /dev/null @@ -1,253 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -# Root directory of this script. -SCRIPTDIR=$( dirname "${BASH_SOURCE[0]}" ) -BASEDIR=${PWD} - -USAGE="Usage: $0 " -GROUP=${1:?Please provide monitoring group name: $USAGE} - -# Create all output directories. -for project in mlab-sandbox mlab-staging mlab-oti ; do - mkdir -p ${BASEDIR}/gen/${project}/prometheus/{legacy-targets,blackbox-targets,blackbox-targets-ipv6,snmp-targets,script-targets} -done - -# All testing sites and machines. -SELECT_mlab_sandbox=$( cat ${SCRIPTDIR}/plsync/testing_patterns.txt | xargs | sed -e 's/ /|/g' ) - -# All mlab4's and the set of canary machines. -SELECT_mlab_staging=$( cat ${SCRIPTDIR}/plsync/staging_patterns.txt ${SCRIPTDIR}/plsync/canary_machines.txt | xargs | sed -e 's/ /|/g' ) - -# All sites *excluding* test sites. -SELECT_mlab_oti=$( cat ${SCRIPTDIR}/plsync/production_patterns.txt | xargs | sed -e 's/ /|/g' ) - -# GCP doesn't support IPv6, so we have a Linode VM running three instances of -# the blackbox_exporter, on three separate ports... one port/instance for each -# project. These variables map projects to ports, and will be transmitted to -# Prometheus in the form of a new label that will be rewritten. -BBE_IPV6_PORT_mlab_oti="9115" -BBE_IPV6_PORT_mlab_staging="8115" -BBE_IPV6_PORT_mlab_sandbox="7115" - - -for project in mlab-sandbox mlab-staging mlab-oti ; do - pushd ${SCRIPTDIR}/plsync - output=${BASEDIR}/gen/${project}/prometheus - - # Construct the per-project SELECT variable name to use below. - pattern=SELECT_${project/-/_} - - # Construct the per-project blackbox_exporter port variable to use below. - # blackbox_exporter on for IPv6 targets. - bbe_port=BBE_IPV6_PORT_${project/-/_} - - if [[ ${GROUP} == scraper ]] ; then - # Rsyncd on port 7999. - ./legacyconfig.py --format=prom-targets \ - --template_target={{hostname}}:7999 \ - --label service=rsyncd \ - --label module=rsyncd_online \ - --rsync \ - --select "${!pattern}" > ${output}/blackbox-targets/rsyncd.json - - # SSH on port 806 over IPv4 - ./legacyconfig.py --format=prom-targets-nodes \ - --template_target={{hostname}}:806 \ - --label service=ssh806 \ - --label module=ssh_v4_online \ - --select "${!pattern}" > ${output}/blackbox-targets/ssh806.json - - # SSH on port 806 over IPv6 - ./legacyconfig.py --format=prom-targets-nodes \ - --template_target={{hostname}}:806 \ - --label service=ssh806 \ - --label module=ssh_v6_online \ - --label __blackbox_port=${!bbe_port} \ - --select "${!pattern}" \ - --decoration "v6" > ${output}/blackbox-targets-ipv6/ssh806_ipv6.json - - # Sidestream exporter in the npad experiment. - ./legacyconfig.py --format=prom-targets \ - --template_target={{hostname}}:9090 \ - --label service=sidestream \ - --select "npad.iupui.(${!pattern})" > \ - ${output}/legacy-targets/sidestream.json - - elif [[ ${GROUP} == global ]] ; then - - ######################################################################## - # Note: The following configs select all servers. This allows us to - # experiment with monitoring many sites in sandbox or staging before - # production. - ######################################################################## - - # ndt7 SSL on port 443 over IPv4 - ./mlabconfig.py --format=prom-targets \ - --template_target={{hostname}}:443 \ - --label service=ndt7 \ - --label module=tcp_v4_online \ - --select "ndt.iupui.(${!pattern})" > \ - ${output}/blackbox-targets/ndt7.json - - # ndt7 SSL on port 443 over IPv6 - ./mlabconfig.py --format=prom-targets \ - --template_target={{hostname}}:443 \ - --label service=ndt7_ipv6 \ - --label module=tcp_v6_online \ - --label __blackbox_port=${!bbe_port} \ - --select "ndt.iupui.(${!pattern})" \ - --decoration "v6" > \ - ${output}/blackbox-targets-ipv6/ndt7_ipv6.json - - # NDT "raw" on port 3001 over IPv4 - ./mlabconfig.py --format=prom-targets \ - --template_target={{hostname}}:3001 \ - --label service=ndt_raw \ - --label module=tcp_v4_online \ - --select "ndt.iupui.(${!pattern})" > \ - ${output}/blackbox-targets/ndt_raw.json - - # NDT "raw" on port 3001 over IPv6 - ./mlabconfig.py --format=prom-targets \ - --template_target={{hostname}}:3001 \ - --label service=ndt_raw_ipv6 \ - --label module=tcp_v6_online \ - --label __blackbox_port=${!bbe_port} \ - --select "ndt.iupui.(${!pattern})" \ - --decoration "v6" > \ - ${output}/blackbox-targets-ipv6/ndt_raw_ipv6.json - - # NDT SSL on port 3010 over IPv4 - ./mlabconfig.py --format=prom-targets \ - --template_target={{hostname}}:3010 \ - --label service=ndt_ssl \ - --label module=tcp_v4_tls_online \ - --use_flatnames \ - --select "ndt.iupui.(${!pattern})" > \ - ${output}/blackbox-targets/ndt_ssl.json - - # NDT SSL on port 3010 over IPv6 - ./mlabconfig.py --format=prom-targets \ - --template_target={{hostname}}:3010 \ - --label service=ndt_ssl_ipv6 \ - --label module=tcp_v6_tls_online \ - --label __blackbox_port=${!bbe_port} \ - --use_flatnames \ - --select "ndt.iupui.(${!pattern})" \ - --decoration "v6" > \ - ${output}/blackbox-targets-ipv6/ndt_ssl_ipv6.json - - # script_exporter for NDT end-to-end monitoring - ./mlabconfig.py --format=prom-targets \ - --template_target={{hostname}} \ - --label service=ndt_e2e \ - --use_flatnames \ - --select "ndt.iupui.(${!pattern})" > \ - ${output}/script-targets/ndt_e2e.json - - # script_exporter for NDT queueing check - ./mlabconfig.py --format=prom-targets \ - --template_target={{hostname}} \ - --label service=ndt_queue \ - --use_flatnames \ - --select "ndt.iupui.(${!pattern})" > \ - ${output}/script-targets/ndt_queue.json - - # Mobiperf on ports 6001, 6002, 6003 over IPv4. - ./mlabconfig.py --format=prom-targets \ - --template_target={{hostname}}:6001 \ - --template_target={{hostname}}:6002 \ - --template_target={{hostname}}:6003 \ - --label service=mobiperf \ - --label module=tcp_v4_online \ - --physical \ - --select "1.michigan.(${!pattern})" > \ - ${output}/blackbox-targets/mobiperf.json - - # Mobiperf on ports 6001, 6002, 6003 over IPv6. - ./mlabconfig.py --format=prom-targets \ - --template_target={{hostname}}:6001 \ - --template_target={{hostname}}:6002 \ - --template_target={{hostname}}:6003 \ - --label service=mobiperf_ipv6 \ - --label module=tcp_v6_online \ - --label __blackbox_port=${!bbe_port} \ - --physical \ - --select "1.michigan.(${!pattern})" \ - --decoration "v6" > ${output}/blackbox-targets-ipv6/mobiperf_ipv6.json - - # neubot on port 9773 over IPv4. - ./mlabconfig.py --format=prom-targets \ - --template_target={{hostname}}:9773/sapi/state \ - --label service=neubot \ - --label module=neubot_online_v4 \ - --physical \ - --select "neubot.mlab.(${!pattern})" > \ - ${output}/blackbox-targets/neubot.json - - # neubot on port 9773 over IPv6. - ./mlabconfig.py --format=prom-targets \ - --template_target={{hostname}}:9773/sapi/state \ - --label service=neubot_ipv6 \ - --label module=neubot_online_v6 \ - --label __blackbox_port=${!bbe_port} \ - --physical \ - --select "neubot.mlab.(${!pattern})" \ - --decoration "v6" > ${output}/blackbox-targets-ipv6/neubot_ipv6.json - - # snmp_exporter on port 9116. - ./mlabconfig.py --format=prom-targets-sites \ - --physical \ - --select "${!pattern}" \ - --template_target=s1.{{sitename}}.measurement-lab.org \ - --label service=snmp > \ - ${output}/snmp-targets/snmpexporter.json - - # inotify_exporter for NDT on port 9393. - ./mlabconfig.py --format=prom-targets \ - --template_target={{hostname}}:9393 \ - --label service=inotify \ - --physical \ - --select "ndt.iupui.(${!pattern})" > \ - ${output}/legacy-targets/ndt_inotify.json - - # node_exporter on port 9100. - ./mlabconfig.py --format=prom-targets-nodes \ - --template_target={{hostname}}:9100 \ - --label service=nodeexporter \ - --select "${!pattern}" > \ - ${output}/legacy-targets/nodeexporter.json - - # ICMP probe for platform switches - ./mlabconfig.py --format=prom-targets-sites \ - --physical \ - --select "${!pattern}" \ - --template_target=s1.{{sitename}}.measurement-lab.org \ - --label module=icmp > \ - ${output}/blackbox-targets/switches_ping.json - - # SSH on port 22 over IPv4 - ./mlabconfig.py --format=prom-targets-nodes \ - --template_target={{hostname}}:22 \ - --label service=ssh \ - --label module=ssh_v4_online \ - --physical \ - --select "${!pattern}" > ${output}/blackbox-targets/ssh.json - - # SSH on port 22 over IPv6 - ./mlabconfig.py --format=prom-targets-nodes \ - --template_target={{hostname}}:22 \ - --label service=ssh \ - --label module=ssh_v6_online \ - --label __blackbox_port=${!bbe_port} \ - --physical \ - --select "${!pattern}" \ - --decoration "v6" > ${output}/blackbox-targets-ipv6/ssh_ipv6.json - - else - echo "Unknown group name: ${GROUP} for ${project}" - fi - popd -done diff --git a/generate_scraper_prometheus_targets.sh b/generate_scraper_prometheus_targets.sh new file mode 100755 index 0000000..399d949 --- /dev/null +++ b/generate_scraper_prometheus_targets.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +set -euxo pipefail + +# Root directory of this script. +SCRIPTDIR=$( dirname "${BASH_SOURCE[0]}" ) +BASEDIR=${PWD} + +# Create all output directories. +for project in mlab-sandbox mlab-staging mlab-oti ; do + mkdir -p ${BASEDIR}/gen/${project}/prometheus/{legacy-targets,blackbox-targets,blackbox-targets-ipv6,snmp-targets,script-targets} +done + +# All testing sites and machines. +SELECT_mlab_sandbox=$( cat ${SCRIPTDIR}/plsync/testing_patterns.txt | xargs | sed -e 's/ /|/g' ) + +# All mlab4's and the set of canary machines. +SELECT_mlab_staging=$( cat ${SCRIPTDIR}/plsync/staging_patterns.txt | xargs | sed -e 's/ /|/g' ) + +# All sites *excluding* test sites. +SELECT_mlab_oti=$( cat ${SCRIPTDIR}/plsync/production_patterns.txt | xargs | sed -e 's/ /|/g' ) + +# GCP doesn't support IPv6, so we have a Linode VM running three instances of +# the blackbox_exporter, on three separate ports... one port/instance for each +# project. These variables map projects to ports, and will be transmitted to +# Prometheus in the form of a new label that will be rewritten. +BBE_IPV6_PORT_mlab_oti="9115" +BBE_IPV6_PORT_mlab_staging="8115" +BBE_IPV6_PORT_mlab_sandbox="7115" + + +for project in mlab-sandbox mlab-staging mlab-oti ; do + pushd ${SCRIPTDIR}/plsync + output=${BASEDIR}/gen/${project}/prometheus + + # Construct the per-project SELECT variable name to use below. + pattern=SELECT_${project/-/_} + + # Construct the per-project blackbox_exporter port variable to use below. + # blackbox_exporter on for IPv6 targets. + bbe_port=BBE_IPV6_PORT_${project/-/_} + + # Rsyncd on port 7999. + ./legacyconfig.py --format=prom-targets \ + --template_target={{hostname}}:7999 \ + --label service=rsyncd \ + --label module=rsyncd_online \ + --rsync \ + --select "${!pattern}" > ${output}/blackbox-targets/rsyncd.json + + # SSH on port 806 over IPv4 + ./legacyconfig.py --format=prom-targets-nodes \ + --template_target={{hostname}}:806 \ + --label service=ssh806 \ + --label module=ssh_v4_online \ + --select "${!pattern}" > ${output}/blackbox-targets/ssh806.json + + # SSH on port 806 over IPv6 + ./legacyconfig.py --format=prom-targets-nodes \ + --template_target={{hostname}}:806 \ + --label service=ssh806 \ + --label module=ssh_v6_online \ + --label __blackbox_port=${!bbe_port} \ + --select "${!pattern}" \ + --decoration "v6" > ${output}/blackbox-targets-ipv6/ssh806_ipv6.json + + # Sidestream exporter in the npad experiment. + ./legacyconfig.py --format=prom-targets \ + --template_target={{hostname}}:9090 \ + --label service=sidestream \ + --select "npad.iupui.(${!pattern})" > \ + ${output}/legacy-targets/sidestream.json + popd +done