diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile new file mode 100644 index 0000000000..8d135bec91 --- /dev/null +++ b/.ci/load/Jenkinsfile @@ -0,0 +1,167 @@ +// For documentation on this pipeline, please see the README.md in this directory +pipeline { + agent any + environment { + REPO = 'apm-agent-java' + APP = 'spring-petclinic' + APP_BASE_DIR = "src/${APP}" + METRICS_BASE_DIR="metrics/" + AGENT_BASE_DIR = "agent/" + ORCH_URL = 'obs-load-orch.app.elastic.co:8000' + // Set below for local development + // ORCH_URL='10.0.2.2:8000' + DEBUG_MODE = '0' // set to '0' for production + LOCUST_RUN_TIME = "${params.duration}" + LOCUST_USERS = "${params.concurrent_requests}" + + } + options { + timeout(time: 72, unit: 'HOURS') + buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) + timestamps() + ansiColor('xterm') + durabilityHint('PERFORMANCE_OPTIMIZED') + + } + parameters { + // The following snippet is auto-generated. To update it, run the script located in .ci/load/scripts/param_gen and copy in the output + choice(choices: ['1.18.1', '1.18.0', '1.18.0.RC1', '1.17.0', '1.16.0', '1.15.0', '1.14.0', '1.13.0', '1.12.0', '1.11.0', '1.10.0', '1.9.0', '1.8.0', '1.7.0', '1.6.1', '1.6.0', '1.5.0', '1.4.0', '1.3.0', '1.2.0', '1.1.0', '1.0.1', '1.0.0', '1.0.0.RC1', '0.7.1', '0.7.0', '0.6.2', '0.6.1', '0.6.0', '0.5.1', '0.1.2', '0.1.1'], name: "apm_version", description: "APM Java Agent version") + choice(choices: ['adoptopenjdk-11+28-linux', 'adoptopenjdk-11.0.1+13-linux', 'adoptopenjdk-11.0.2+7-linux', 'adoptopenjdk-11.0.2+9-linux', 'adoptopenjdk-11.0.3+7-linux', 'adoptopenjdk-11.0.4+11-linux', 'adoptopenjdk-11.0.5+10-linux', 'adoptopenjdk-11.0.6+10-linux', 'adoptopenjdk-11.0.7+10-linux', 'adoptopenjdk-11.0.8+10-linux', 'adoptopenjdk-11.0.9+11-linux', 'adoptopenjdk-12+33-linux', 'adoptopenjdk-12.0.1+12-linux', 'adoptopenjdk-12.0.2+10-linux', 'adoptopenjdk-13.0.1+9-linux', 'adoptopenjdk-13.0.2+8-linux', 'adoptopenjdk-14.0.1+7-linux', 'adoptopenjdk-14.0.2+12-linux', 'adoptopenjdk-15+36-linux', 'adoptopenjdk-15.0.1+9-linux', 'openjdk-10+43-linux', 'openjdk-10-linux', 'openjdk-10.0.1-linux', 'openjdk-10.0.2-linux', 'openjdk-11+11-linux', 'openjdk-11+12-linux', 'openjdk-11+13-linux', 'openjdk-11+14-linux', 'openjdk-11+15-linux', 'openjdk-11+16-linux', 'openjdk-11+17-linux', 'openjdk-11+18-linux', 'openjdk-11+19-linux', 'openjdk-11+20-linux', 'openjdk-11+21-linux', 'openjdk-11+22-linux', 'openjdk-11+23-linux', 'openjdk-11+24-linux', 'openjdk-11+25-linux', 'openjdk-11+26-linux', 'openjdk-11+27-linux', 'openjdk-11+28-linux', 'openjdk-11+5-linux', 'openjdk-11-linux', 'openjdk-11.0.1-linux', 'openjdk-11.0.2-linux', 'openjdk-12+23-linux', 'openjdk-12+24-linux', 'openjdk-12+25-linux', 'openjdk-12+27-linux', 'openjdk-12+28-linux', 'openjdk-12+29-linux', 'openjdk-12+30-linux', 'openjdk-12+31-linux', 'openjdk-12+32-linux', 'openjdk-12+33-linux', 'openjdk-12-linux', 'openjdk-12.0.1-linux', 'openjdk-12.0.2-linux', 'openjdk-13+14-linux', 'openjdk-13+15-linux', 'openjdk-13+16-linux', 'openjdk-13+17-linux', 'openjdk-13+18-linux', 'openjdk-13+19-linux', 'openjdk-13+20-linux', 'openjdk-13+21-linux', 'openjdk-13+22-linux', 'openjdk-13+23-linux', 'openjdk-13+24-linux', 'openjdk-13+25-linux', 'openjdk-13+26-linux', 'openjdk-13+27-linux', 'openjdk-13+28-linux', 'openjdk-13+29-linux', 'openjdk-13+30-linux', 'openjdk-13+31-linux', 'openjdk-13+32-linux', 'openjdk-13-linux', 'openjdk-13.0.1-linux', 'openjdk-13.0.2-linux', 'openjdk-14+10-linux', 'openjdk-14+11-linux', 'openjdk-14+12-linux', 'openjdk-14+13-linux', 'openjdk-14+14-linux', 'openjdk-14+15-linux', 'openjdk-14+16-linux', 'openjdk-14+17-linux', 'openjdk-14+25-linux', 'openjdk-14+26-linux', 'openjdk-14+27-linux', 'openjdk-14+28-linux', 'openjdk-14+30-linux', 'openjdk-14+31-linux', 'openjdk-14+32-linux', 'openjdk-14+33-linux', 'openjdk-14+34-linux', 'openjdk-14+9-linux', 'openjdk-14-linux', 'openjdk-14.0.1-linux', 'openjdk-14.0.2+12-linux', 'openjdk-14.0.2-linux', 'openjdk-15+10-linux', 'openjdk-15+11-linux', 'openjdk-15+12-linux', 'openjdk-15+13-linux', 'openjdk-15+14-linux', 'openjdk-15+15-linux', 'openjdk-15+16-linux', 'openjdk-15+17-linux', 'openjdk-15+18-linux', 'openjdk-15+19-linux', 'openjdk-15+20-linux', 'openjdk-15+21-linux', 'openjdk-15+22-linux', 'openjdk-15+23-linux', 'openjdk-15+24-linux', 'openjdk-15+25-linux', 'openjdk-15+26-linux', 'openjdk-15+27-linux', 'openjdk-15+28-linux', 'openjdk-15+29-linux', 'openjdk-15+30-linux', 'openjdk-15+31-linux', 'openjdk-15+32-linux', 'openjdk-15+33-linux', 'openjdk-15+34-linux', 'openjdk-15+36-linux', 'openjdk-15+4-linux', 'openjdk-15+5-linux', 'openjdk-15+6-linux', 'openjdk-15+7-linux', 'openjdk-15+8-linux', 'openjdk-15+9-linux', 'openjdk-15-linux', 'openjdk-15.0.1+9-linux', 'openjdk-9.0.4-linux', 'oracle-10+43-linux', 'oracle-10+46-linux', 'oracle-11+11-linux', 'oracle-11+12-linux', 'oracle-11+13-linux', 'oracle-11+14-linux', 'oracle-11+15-linux', 'oracle-11+16-linux', 'oracle-11+17-linux', 'oracle-11+18-linux', 'oracle-11+19-linux', 'oracle-11+20-linux', 'oracle-11+21-linux', 'oracle-11+22-linux', 'oracle-11+23-linux', 'oracle-11+24-linux', 'oracle-11+25-linux', 'oracle-11+26-linux', 'oracle-11+27-linux', 'oracle-11+28-linux', 'oracle-11+5-linux', 'oracle-11.0.2+7-linux', 'oracle-11.0.2+9-linux', 'oracle-11.0.3+12-linux', 'oracle-11.0.4+10-linux', 'oracle-11.0.5+10-linux', 'oracle-11.0.6+8-linux', 'oracle-12+33-linux', 'oracle-12.0.1+12-linux', 'oracle-12.0.2+10-linux', 'oracle-13+33-linux', 'oracle-13.0.1+9-linux', 'oracle-13.0.2+8-linux', 'oracle-9.0.4+11-linux', 'zulu-10.0.0-linux', 'zulu-10.0.1-linux', 'zulu-10.0.2-linux', 'zulu-11.0.1-linux', 'zulu-11.0.2-linux', 'zulu-11.0.3-linux', 'zulu-11.0.4-linux', 'zulu-11.0.5-linux', 'zulu-11.0.6-linux', 'zulu-11.0.7-linux', 'zulu-11.0.8-linux', 'zulu-11.0.9-linux', 'zulu-12-linux', 'zulu-12.0.0-linux', 'zulu-12.0.1-linux', 'zulu-12.0.2-linux', 'zulu-13-linux', 'zulu-13.0.0-linux', 'zulu-13.0.1-linux', 'zulu-13.0.2-linux', 'zulu-13.0.3-linux', 'zulu-13.0.4-linux', 'zulu-13.0.5-linux', 'zulu-14-linux', 'zulu-14.0.0-linux', 'zulu-14.0.1-linux', 'zulu-14.0.2-linux', 'zulu-15.0.0-linux', 'zulu-15.0.1-linux', 'zulu-7.0.101-linux', 'zulu-7.0.111-linux', 'zulu-7.0.121-linux', 'zulu-7.0.131-linux', 'zulu-7.0.141-linux', 'zulu-7.0.154-linux', 'zulu-7.0.161-linux', 'zulu-7.0.171-linux', 'zulu-7.0.181-linux', 'zulu-7.0.191-linux', 'zulu-7.0.201-linux', 'zulu-7.0.211-linux', 'zulu-7.0.222-linux', 'zulu-7.0.232-linux', 'zulu-7.0.242-linux', 'zulu-7.0.252-linux', 'zulu-7.0.262-linux', 'zulu-7.0.272-linux', 'zulu-7.0.282-linux', 'zulu-7.0.95-linux', 'zulu-8.0.102-linux', 'zulu-8.0.112-linux', 'zulu-8.0.121-linux', 'zulu-8.0.131-linux', 'zulu-8.0.144-linux', 'zulu-8.0.152-linux', 'zulu-8.0.162-linux', 'zulu-8.0.163-linux', 'zulu-8.0.172-linux', 'zulu-8.0.181-linux', 'zulu-8.0.192-linux', 'zulu-8.0.201-linux', 'zulu-8.0.202-linux', 'zulu-8.0.212-linux', 'zulu-8.0.222-linux', 'zulu-8.0.232-linux', 'zulu-8.0.242-linux', 'zulu-8.0.252-linux', 'zulu-8.0.262-linux', 'zulu-8.0.265-linux', 'zulu-8.0.272-linux', 'zulu-8.0.71-linux', 'zulu-8.0.72-linux', 'zulu-8.0.91-linux', 'zulu-8.0.92-linux', 'zulu-9.0.0-linux', 'zulu-9.0.1-linux', 'zulu-9.0.4-linux', 'zulu-9.0.7-linux'], name: "jvm_version", description: "JVM") + string(name: "concurrent_requests", defaultValue: "100", description: "The number of concurrent requests to test with") + string(name: "duration", defaultValue: "10", description: "Test duration in minutes. Max: 280") + // num_of_runs currently unsupported + // string(name: "num_of_runs", defaultValue: "1", description: "Number of test runs to execute") + text(name: "agent_config", "defaultValue": "", description: "Custom APM Agent configuration. (WARNING: May echo to console. Do not supply sensitive data.)") + text(name: "locustfile", "defaultValue": "", description: "Locust load-generator plan") + booleanParam(name: "local_metrics", description: "Enable local metrics collection?", defaultValue: false) + // End script auto-generation + } + + stages { + stage('Pre-flight'){ + steps { + echo 'Getting authentication information from Vault' + withSecretVault(secret: 'secret/apm-team/ci/bandstand', user_var_name: 'APP_TOKEN_TYPE', pass_var_name: 'APP_TOKEN'){ + setEnvVar('SESSION_TOKEN', sh(script: ".ci/load/scripts/start.sh", returnStdout: true).trim()) + } + } + } + stage('Load test') { + parallel { + stage('Load generation') { + agent { label 'metal' } + steps { + withSecretVault(secret: 'secret/apm-team/ci/bandstand', user_var_name: 'APP_TOKEN_TYPE', pass_var_name: 'APP_TOKEN'){ + echo 'Preparing load generation..' + whenTrue(Boolean.valueOf(params.locustfile)) { + echo 'Using user-supplied plan for load-generation with Locust' + sh script: "echo \"${params.locustfile}\">.ci/load/scripts/locustfile.py" + } + sh(script: ".ci/load/scripts/load_agent.sh") + } + } + } + stage('Test application') { + agent { label 'benchmark' } + stages{ + stage('Provision Java') { + steps { + echo "Provisioning Java version: ${params.jvm_version}" + setEnvVar('JAVA_HOME', sh(script: ".ci/load/scripts/fetch_sdk.sh ${params.jvm_version}", returnStdout: true).trim()) + setEnvVar('JAVACMD', "${env.JAVA_HOME}/bin/java") + setEnvVar('PATH', "${env.JAVA_HOME}/bin:$PATH") + } + } + stage ('Provision agent') { + steps { + echo 'Checking out master branch' + dir("${AGENT_BASE_DIR}") { + gitCheckout( + basedir: "apm-agent-java", + branch: 'master', + repo: "https://github.com/elastic/${REPO}.git", + credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba', + shallow: false + ) + dir("apm-agent-java"){ + echo 'Switching to requested version' + sh(script: "git checkout v${params.apm_version}") + echo 'Building agent' + sh(script: './mvnw clean install -DskipTests=true -Dmaven.javadoc.skip=true') + } + } + whenTrue(Boolean.valueOf(params.agent_config)) { + echo 'Writing user-supplied agent configuration' + dir("${AGENT_BASE_DIR}") { + sh script: "echo \"${params.agent_config}\">custom_config.cfg" + } + } + } + } + stage('Provision test application') { + steps { + echo 'Checking out test application' + gitCheckout( + basedir: "${APP_BASE_DIR}", + branch: 'main', + repo: "https://github.com/spring-projects/${APP}.git", + credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba', + shallow: false + ) + } + } + stage('Provision local metrics collection') { + when { + expression { + return params.local_metrics + } + } + steps { + echo 'Enable local metric collection' + gitCheckout( + basedir: "${METRICS_BASE_DIR}", + branch: 'master', + repo: "https://github.com/pstadler/metrics.sh", + credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba', + shallow: false + ) + sh(script: "touch metrics.out") + dir("${METRICS_BASE_DIR}"){ + withEnv(["FILE_LOCATION=./metrics.out"]) { + sh(script: "./metrics.sh -r file &") + } + } + } + } + stage('Application load') { + steps { + echo 'Starting test application in background..' + dir("${APP_BASE_DIR}"){ + // Launch app in background + withSecretVault(secret: 'secret/apm-team/ci/apm-load-test-server', user_var_name: 'APM_SERVER_URL', pass_var_name: 'ELASTIC_APM_API_KEY'){ + // Start with packaging things up + sh(script: "./mvnw package") + sh(script: "java -jar -javaagent:${WORKSPACE}/${AGENT_BASE_DIR}/apm-agent-java/elastic-apm-agent/target/elastic-apm-agent-${params.apm_version}.jar -Delastic.apm.server_urls=${env.APM_SERVER_URL} -Delastic.apm.secret_token=${env.ELASTIC_APM_API_KEY} -XX:+FlightRecorder -XX:StartFlightRecording=filename=flight.jfr ./target/spring-petclinic-*.jar &") + } + } + echo 'Starting bandstand client..' + // Foreground the orchestrator script for execution control + withSecretVault(secret: 'secret/apm-team/ci/bandstand', user_var_name: 'APP_TOKEN_TYPE', pass_var_name: 'APP_TOKEN'){ + sh(script: ".ci/load/scripts/app.sh") + } + } + } + stage('Collecting results') { + steps { + echo "To view results, JMC is required. Get it here: https://jdk.java.net/jmc/" + archiveArtifacts(allowEmptyArchive: true, + artifacts: "${APP_BASE_DIR}/**/*.jfr,${METRICS_BASE_DIR}/**/*.out", + onlyIfSuccessful: false) + } + } + } + } + } + } + } +} diff --git a/.ci/load/README.md b/.ci/load/README.md new file mode 100644 index 0000000000..61de0b46ad --- /dev/null +++ b/.ci/load/README.md @@ -0,0 +1,61 @@ +# APM Java Agent Load Geneator + +This directory contains configuration files for load-testing the APM Java Agent. + +## Load Tests + +Load tests are run using two bare-metal workers. One worker is dedicated to the role of load generation, which other worker is dedicated to the role of hosting the application under test, which is instrumented by the APM Java Agent. + +Load generation is performed using [Locust](https://locust.io/) and the application which is instrumented by the APM Java Agent and placed under load is [Spring Petclinic](https://projects.spring.io/spring-petclinic/). + +## Orchestration + +Because of the nature of these tests, they require the use of a dedicated orchestration layer. For this project, that orchestrator is called Bandstand. It is an internal project to Elastic. + +Bandstand performs two primary functions. + +1. *Service discovery* The pipeline utilizes a load-generation machine and an application server which run in parallel to each other. These machines need to know how to find each other and Bandstand gives them that information. +2. *Orchestration* We need to have a system which tells various services when to start an stop based on the state of other services. For example, we can't start load-generation until we are assured that the application is up and in a coherent state. By using an independent orchestrator, we can allow each service to report on its own state. This avoid a number of otherwise difficult-to-maintain dependencies between service state through the lifetime of the load generation. + +While the orchestration layer is currently relatively lightweight and simple, it is built to be able to be easily extended for more sophisticated needs, such as multiple test runs inside a single test execution. + +The current design uses a persistant orchestator. However, it is a future goal to have a dynamic orchestrator which is unique to each test execution and is spun up and torn down alongside the rest of the services. + +## Running Load Tests + +To run a load test, navigate to the [APM Java Agent CI](https://apm-ci.elastic.co/job/apm-agent-java/) and look for the `Load Test` pipeline. Tests can be executed with the following parameters: + +|Parameter|Description| +|:-------:|:---------| +|`apm_version`|The version of the APM Java Agent to use. After checking out the agent from source control, this is the git tag that will be used.| +|`jvm_version`|The version of the JVM which will be used. This will be used as the version of Java both when building the agent and when launching the application.| +|`concurrent_requests`|The number of concurrent, simulated users which will make requests against the test applicaiton.| +|`duration`|The duration for the load test. Specify this value using time units: (300s, 20m, 3h, 1h30m, etc.)| +|`agent_config`|To override the defaults of the APM Java Agent config, paste a configuration file.| +|`locustfile`|A Locust file controls the specifics of how load will be applied to the test application, such as which URLs are requested and at what rate. If this option is not specified, then `.ci/load/scripts/locustfile.py` will be used. For more information on writing a locustfile, [please see the documentation](https://docs.locust.io/en/stable/writing-a-locustfile.html).| + + +After specifying the options, a test run will begin. One can view the progress of the test run by following along in the Jenkins Blue Ocean interface for the given job after it is launched. Typically, it takes several minutes for the machines to be provisioned before tests can begin. + +## Viewing test results + +Because the [test application](https://projects.spring.io/spring-petclinic/) is instrumented with [JFR](https://docs.oracle.com/javacomponents/jmc-5-4/jfr-runtime-guide/about.htm#JFRUH170), it is necessary to have a copy of the [JDK Mission Control application](https://www.baeldung.com/java-flight-recorder-monitoring#3-visualize-data) which is distributed with the JDK. + +The results of the JFR instrumentation are available after a test run is complete by navigating to Jenkins and viewing the job results. A link to `Artifacts` will bring you to a page which shows the files generated by the test run. Download `src/spring-petclinic/flight.jfr` and then view its contents with JDK MIssion Control to examine the moment-by-moment profile of the application under test. + +To view the load generation output, see the console output in Jenkins for the `Load Generation` step. + +## Hacking + +*Caution*: The following requires access to libraries which are internal to Elastic. Do not attempt to follow these instructions if you are an open-source user +of the APM Java Agent. Instead, contact the core development team with questions. + +To set up a local development environment for executing load tests and for developing the test framework, perform the following steps: + +1. Checkout the Bandstand application and build a docker container from it tagged as `bandstand`. +2. Checkout the [fork of the APM Pipeline library](https://github.com/cachedout/apm-pipeline-library-1/tree/perf) which contains the necessary modifications to provision a load-testing environment locally. +3. In the `apm-pipeline-library` checkout from step 2, create a soft link from `local/jenkins_home` to the your checkout of the `apm-java-agent` codebase. +4. Start up Jenkins using `make start` from your copy of the APM Pipeline library. Start up a worker as well, using the modified `Vagrantfile` from step 2. +5. Modify the Jenkinsfile to use Bandstand. Set `ORCH_URL` to `10.0.2.2:8000` to use the local instance. + +To verify this environment, run `docker ps` and ensure you have a copy of `bandstand` running. It may be necessary to manipulate certain variables in the pipeline to avoid looking up certain credentials from the production secret store. diff --git a/.ci/load/scripts/app.sh b/.ci/load/scripts/app.sh new file mode 100755 index 0000000000..b5ede85b95 --- /dev/null +++ b/.ci/load/scripts/app.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash + +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -exuo pipefail + +POLL_FREQ=1 + +APP_PORT=8080 + +function setUp() { + echo "Setting CPU frequency to base frequency" + + CPU_MODEL=$(lscpu | grep "Model name" | awk '{for(i=3;i<=NF;i++){printf "%s ", $i}; printf "\n"}') + if [ "${CPU_MODEL}" == "Intel(R) Xeon(R) CPU E3-1246 v3 @ 3.50GHz " ] + then + # could also use `nproc` + CORE_INDEX=7 + BASE_FREQ="3.5GHz" + elif [ "${CPU_MODEL}" == "Intel(R) Core(TM) i7-6700 CPU @ 3.40GHz " ] + then + CORE_INDEX=7 + BASE_FREQ="3.4GHz" + elif [ "${CPU_MODEL}" == "Intel(R) Core(TM) i7-7700 CPU @ 3.60GHz " ] + then + CORE_INDEX=7 + BASE_FREQ="3.6GHz" + elif [ "${CPU_MODEL}" == "Intel(R) Core(TM) i7-8665U CPU @ 1.90GHz " ] + then + CORE_INDEX=7 + BASE_FREQ="1.9GHz" + else + >&2 echo "Cannot determine base frequency for CPU model [${CPU_MODEL}]. Please adjust the build script." + exit 1 + fi + MIN_FREQ=$(cpufreq-info -l -c 0 | awk '{print $1}') + # This is the frequency including Turbo Boost. See also http://ark.intel.com/products/80916/Intel-Xeon-Processor-E3-1246-v3-8M-Cache-3_50-GHz + MAX_FREQ=$(cpufreq-info -l -c 0 | awk '{print $2}') + + # set all CPUs to the base frequency + for (( cpu=0; cpu<=${CORE_INDEX}; cpu++ )) + do + sudo -n cpufreq-set -c ${cpu} --min ${BASE_FREQ} --max ${BASE_FREQ} + done + + # Build cgroups to isolate microbenchmarks and JVM threads + echo "Creating groups for OS and microbenchmarks" + # Isolate the OS to the first core + sudo -n cset set --set=/os --cpu=0-1 + sudo -n cset proc --move --fromset=/ --toset=/os + + # Isolate the microbenchmarks to all cores except the first two (first physical core) + # On a 4 core CPU with hyper threading, this would be 6 cores (3 physical cores) + sudo -n cset set --set=/benchmark --cpu=2-${CORE_INDEX} +} + +function appIsReady() { + # Poll the app until it is ready + curl -Is http://localhost:$APP_PORT/| head -1|egrep 200 +} + +function sendAppReady() { + curl -s -X POST -H "Content-Type: application/json" -d \ + "{\"app_token\": \""$APP_TOKEN"\", \ + \"session_token\": \""$SESSION_TOKEN"\", \ + \"service\": \"application\", \ + \"hostname\": \""$(hostname)"\", \ + \"port\": \"8080\"}" \ + $ORCH_URL/api/ready + + +} + +function waitForApp() { + # Wait for the load generation to finish before we kill the app + while : + do + if appIsReady; then + break + fi + sleep $POLL_FREQ; + done +} + +function waitForLoad() { + # Wait for the load generation to finish before we kill the app + while : + do + if checkLoadGen; then + break + fi + sleep $POLL_FREQ; + done +} + +function waitForLoadFinish() { + # Wait for the load generation to finish before we kill the app + while : + do + if checkLoadGenFinish; then + break + fi + sleep $POLL_FREQ; + done +} + + +function checkLoadGen(){ + # Check to see if the load generation piece is sending requests + curl -s -X POST -H "Content-Type: application/json" -d \ + "{\"app_token\": \""$APP_TOKEN"\", \ + \"session_token\": \""$SESSION_TOKEN"\", \ + \"service\": \"load_generation\", \ + \"hostname\": \"test_app\", \ + \"port\": \"8080\"}" \ + $ORCH_URL/api/poll | jq '.services.load_generation.state' | egrep 'ready' +} + + +function checkLoadGenFinish(){ + # Check to see if the load generation piece is sending requests + curl -s -X POST -H "Content-Type: application/json" -d \ + "{\"app_token\": \""$APP_TOKEN"\", \ + \"session_token\": \""$SESSION_TOKEN"\", \ + \"service\": \"load_generation\", \ + \"hostname\": \"test_app\", \ + \"port\": \"8080\"}" \ + $ORCH_URL/api/poll | jq '.services.load_generation.state' | egrep 'stopped' +} + + +function stopApp() { + ps -ef|egrep spring-petclinic|egrep java|awk '{print $2}'|xargs kill +} + +function tearDown() { + echo "Destroying cgroups" + sudo -n cset set --destroy /os + sudo -n cset set --destroy /benchmark + + echo "Setting normal frequency range" + for (( cpu=0; cpu<=${CORE_INDEX}; cpu++ )) + do + sudo -n cpufreq-set -c ${cpu} --min ${MIN_FREQ} --max ${MAX_FREQ} + done +} + +if [ ! $DEBUG_MODE ]; then +trap "tearDown" EXIT + setUp +fi +waitForApp +sendAppReady +waitForLoad +waitForLoadFinish +stopApp diff --git a/.ci/load/scripts/fetch_sdk.sh b/.ci/load/scripts/fetch_sdk.sh new file mode 100755 index 0000000000..eb58844705 --- /dev/null +++ b/.ci/load/scripts/fetch_sdk.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# ================================================================ + +# Call this script with an SDK to grab. The initial list should +# be populated in the Jenkinsfile by executing the script in the +# `param_gen` directory. + +# This script will download and unpack whatever SDK is requested +# and then return to the caller the complete name of the directory +# which should be available to set as JAVA_HOME. + +# Example execution for retreiving the SDK: + +# ./fetch_sdk.sh oracle-13+33-linux +# +# This script requires the following tooling to be available on the +# system path prior to execution: +# +# 1. cURL [https://curl.haxx.se/] +# 2. jq [https://stedolan.github.io/jq/] +# 3. tar [https://www.gnu.org/software/tar/] +# ================================================================ + +# set -exuo pipefail + + +CATALOG_URL="https://jvm-catalog.elastic.co/jdks" + +read -r SDK_URL SDK_FILENAME <<<$(curl -s $CATALOG_URL|jq -Mr '.['\"$1\"'].url, .['\"$1\"'].filename') + +curl -s -o $SDK_FILENAME $SDK_URL + +if [ ! -e tmp_java ]; then + mkdir tmp_java/ +fi + +tar xfz $SDK_FILENAME -C tmp_java/ + +UNPACKED_JDK_DIR=$(ls tmp_java) +echo $PWD/tmp_java/$UNPACKED_JDK_DIR diff --git a/.ci/load/scripts/load_agent.sh b/.ci/load/scripts/load_agent.sh new file mode 100755 index 0000000000..336e6c645f --- /dev/null +++ b/.ci/load/scripts/load_agent.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash + +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -exuo pipefail + +POLL_FREQ=1 + +LOCUST_LOCUSTFILE="../locust.py" +LOCUST_PRINT_STATS=1 + + +function setUp() { + echo "Setting CPU frequency to base frequency" + + CPU_MODEL=$(lscpu | grep "Model name" | awk '{for(i=3;i<=NF;i++){printf "%s ", $i}; printf "\n"}') + if [ "${CPU_MODEL}" == "Intel(R) Xeon(R) CPU E3-1246 v3 @ 3.50GHz " ] + then + # could also use `nproc` + CORE_INDEX=7 + BASE_FREQ="3.5GHz" + elif [ "${CPU_MODEL}" == "Intel(R) Core(TM) i7-6700 CPU @ 3.40GHz " ] + then + CORE_INDEX=7 + BASE_FREQ="3.4GHz" + elif [ "${CPU_MODEL}" == "Intel(R) Core(TM) i7-7700 CPU @ 3.60GHz " ] + then + CORE_INDEX=7 + BASE_FREQ="3.6GHz" + elif [ "${CPU_MODEL}" == "Intel(R) Core(TM) i7-8665U CPU @ 1.90GHz " ] + then + CORE_INDEX=7 + BASE_FREQ="1.9GHz" + else + >&2 echo "Cannot determine base frequency for CPU model [${CPU_MODEL}]. Please adjust the build script." + exit 1 + fi + MIN_FREQ=$(cpufreq-info -l -c 0 | awk '{print $1}') + # This is the frequency including Turbo Boost. See also http://ark.intel.com/products/80916/Intel-Xeon-Processor-E3-1246-v3-8M-Cache-3_50-GHz + MAX_FREQ=$(cpufreq-info -l -c 0 | awk '{print $2}') + + # set all CPUs to the base frequency + for (( cpu=0; cpu<=${CORE_INDEX}; cpu++ )) + do + sudo -n cpufreq-set -c ${cpu} --min ${BASE_FREQ} --max ${BASE_FREQ} + done + + # Build cgroups to isolate microbenchmarks and JVM threads + echo "Creating groups for OS and microbenchmarks" + # Isolate the OS to the first core + sudo -n cset set --set=/os --cpu=0-1 + sudo -n cset proc --move --fromset=/ --toset=/os + + # Isolate the microbenchmarks to all cores except the first two (first physical core) + # On a 4 core CPU with hyper threading, this would be 6 cores (3 physical cores) + sudo -n cset set --set=/benchmark --cpu=2-${CORE_INDEX} +} + +# 1. Loop until we get a signal that the application is ready. + +function appIsReady() { + curl -s -X POST -H "Content-Type: application/json" -d \ + "{\"app_token\": \""$APP_TOKEN"\", \"session_token\": \""$SESSION_TOKEN"\"}" $ORCH_URL/api/poll | \ + jq -e '.services.application.state == "ready"' +} + +function waitForApp() { + while : + do + if appIsReady; then + break + fi + sleep $POLL_FREQ; + done +} + +function buildArgs() { + LOCUST_HOSTNAME="$(curl -s -X POST -H "Content-Type: application/json" -d \ + "{\"app_token\": \ + \""$APP_TOKEN"\", \ + \"session_token\": \""$SESSION_TOKEN"\"}" \ + $ORCH_URL/api/poll | \ + jq '.services.application.hostname')" + + LOCUST_PORT="$(curl -s -X POST -H "Content-Type: application/json" -d \ + "{\"app_token\": \ + \""$APP_TOKEN"\", \ + \"session_token\": \""$SESSION_TOKEN"\"}" \ + $ORCH_URL/api/poll | \ + jq '.services.application.port')" + export LOCUST_HOST=http://$(echo $LOCUST_HOSTNAME|sed 's/"//g'):$(echo $LOCUST_PORT|sed 's/"//g') + export LOCUST_RUN_TIME=30s +} + +function startLoad() { + curl -s -X POST -H "Content-Type: application/json" -d \ + "{\"app_token\": \""$APP_TOKEN"\", \ + \"session_token\": \""$SESSION_TOKEN"\", \ + \"service\": \"load_generation\", \ + \"hostname\": \"test_app\", \ + \"port\": \"8080\"}" \ + $ORCH_URL/api/ready + docker run -e "LOCUST_HOST=$LOCUST_HOST" -e "LOCUST_RUN_TIME=$LOCUST_RUN_TIME" -e "LOCUST_USERS=$LOCUST_USERS" -p 8089:8089 -v ${PWD}/.ci/load/scripts:/locust locustio/locust -f /locust/locustfile.py -u 10 --headless +} + +function stopLoad() { + # This happens as soon as the container exits so there is nothing to kill + curl -s -X POST -H "Content-Type: application/json" -d \ + "{\"app_token\": \""$APP_TOKEN"\", \ + \"session_token\": \""$SESSION_TOKEN"\", \ + \"service\": \"load_generation\", \ + \"hostname\": \"test_app\", \ + \"port\": \"8080\"}" \ + $ORCH_URL/api/stop +} + +function tearDown() { + echo "Destroying cgroups" + sudo -n cset set --destroy /os + sudo -n cset set --destroy /benchmark + + echo "Setting normal frequency range" + for (( cpu=0; cpu<=${CORE_INDEX}; cpu++ )) + do + sudo -n cpufreq-set -c ${cpu} --min ${MIN_FREQ} --max ${MAX_FREQ} + done +} +if [ ! $DEBUG_MODE ]; then +trap "tearDown" EXIT +setUp +fi + +waitForApp +buildArgs +startLoad +stopLoad diff --git a/.ci/load/scripts/locustfile.py b/.ci/load/scripts/locustfile.py new file mode 100644 index 0000000000..37c2f39ade --- /dev/null +++ b/.ci/load/scripts/locustfile.py @@ -0,0 +1,13 @@ +from locust import HttpUser, task, constant_pacing + + +class QuickstartUser(HttpUser): + wait_time = constant_pacing(1) + + @task + def index_page(self): + self.client.get("/owners/find") + + @task(3) + def gen_error(self): + self.client.get("/oups") diff --git a/.ci/load/scripts/param_gen/gen_params.py b/.ci/load/scripts/param_gen/gen_params.py new file mode 100755 index 0000000000..89fef43cca --- /dev/null +++ b/.ci/load/scripts/param_gen/gen_params.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python + +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Description: + +This is an ad-hoc script used to generate the drop-down selections +for the Jenkinsfile to allow users to test against various combinations +of JDKs available in the Elastic JVM catalog which is hosted at: +https://jvm-catalog.elastic.co. + +It dymamically generates a list of available JDKs and of APM Java Agent +releases. These are returns as a snippet of code which can then be pasted +directly into a Jenkinsfile. Below is an example of the kind of output +produced by this script: + + parameters { + string(name: 'agent_version', defaultValue: 'v1.9.0', description: 'Version of agent. Should correspond to tag, e.g. `v1.9.0`.') + string(name: 'jvm_version', defaultValue: '9.0.4', description: 'Version of JVM.') + string(name: 'concurrent_requests', defaultValue: '100', description: 'The number of concurrent requests to test with.') + string(name: 'duration', defaultValue: '10', description: 'Test duration in minutes. Max: 280') + string(name: 'num_of_runs', defaultValue: '1', description: 'Number of test runs to execute.') + } + +Author: Mike Place + +Maintainers: Observability Developer Productivity + +""" + +import requests +import argparse +import github +from packaging.version import parse as parse_version + +CATALOG_URL = 'https://jvm-catalog.elastic.co' +# The limit of total choices must be < 256 or Jenkins will stacktrace. +# The list below represents all choices currently in the catalog +# *except* `amazon` and `jdk`. +# See https://github.com/elastic/apm-agent-java/pull/1467#discussion_r516464187 +# for additional context and discussion. +SUPPORTED_JDKS = ['oracle', 'openjdk', 'adoptopenjdk', 'zulu'] +MIN_JDK_VERSION = '7' + +parser = argparse.ArgumentParser(description="Jenkins JDK snippet generator") +parser.add_argument( + '--platforms', + nargs='+', + help='platforms help', + type=str, + default='linux', + choices=['linux', 'darwin', 'windows'] + ) +parser.add_argument( + '--gh-token', + help='GitHub token to gather supported releases', + type=str, + required=False +) +parser.add_argument( + '--min-ver', + help='Minimum version of JDK to include', + type=str, + required=False, + default=MIN_JDK_VERSION +) + +parsed_args = parser.parse_args() + +# Gather JDKs we can support +r = requests.get(CATALOG_URL + '/jdks') +if r.status_code != 200: + raise Exception('Error encountered trying to download JDK manifest') + +supported_jdks = [] + +for jdk in r.json(): + arch = None + try: + flavor, ver, dist = jdk.split('-', 3) + except ValueError: + flavor, ver, dist, arch = jdk.split('-', 4) + + if dist in parsed_args.platforms and \ + flavor in SUPPORTED_JDKS and \ + parse_version(ver) >= parse_version(parsed_args.min_ver): + if arch: + continue + supported_jdks.append(jdk) + +# Gather releases of the agent we can support +agent_releases = [] +if parsed_args.gh_token: + gh = github.Github(login_or_token=parsed_args.gh_token) + releases = gh.get_repo('elastic/apm-agent-java').get_releases() + for release in releases: + _, rel_number = release.title.split(' ') + agent_releases.append(rel_number) + +print('Paste the following into the Jenkinsfile:\n\n\n\n') + +print( + '// The following snippet is auto-generated. To update it, run the script located in .ci/load/scripts/param_gen and copy in the output', # noqa E501 + 'choice(choices: {}, name: "apm_version", description: "APM Java Agent version")'.format(agent_releases), # noqa E501 + 'choice(choices: {}, name: "jvm_version", description: "JVM")'.format(supported_jdks), # noqa E501 + 'string(name: "concurrent_requests", defaultValue: "100", description: "The number of concurrent requests to test with")', # noqa E501 + 'string(name: "duration", defaultValue: "10", description: "Test duration in minutes. Max: 280")', # noqa E501 + '// num_of_runs currently unsupported', + '// string(name: "num_of_runs", defaultValue: "1", description: "Number of test runs to execute")', # noqa E501 + 'text(name: "agent_config", "defaultValue": "", description: "Custom APM Agent configuration. (WARNING: May echo to console. Do not supply sensitive data.)")', # noqa E501 + 'text(name: "locustfile", "defaultValue": "", description: "Locust load-generator plan")', # noqa E5011 + 'booleanParam(name: "local_metrics", description: "Enable local metrics collection?", defaultValue: false)', # noqa E501 + '// End script auto-generation', + sep="\n" +) diff --git a/.ci/load/scripts/start.sh b/.ci/load/scripts/start.sh new file mode 100755 index 0000000000..4b7d704578 --- /dev/null +++ b/.ci/load/scripts/start.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +curl -s -X POST -H "Content-Type: application/json" -d \ +"{\"app_token\": \""$APP_TOKEN"\", \ +\"service\": \"application\", \ +\"hostname\": \"test_app\", \ +\"port\": \"999\"}" \ +$ORCH_URL/api/register | jq -Mr '.session_created.session' diff --git a/.ci/load/scripts/tests/simulate_app_start.sh b/.ci/load/scripts/tests/simulate_app_start.sh new file mode 100755 index 0000000000..7950473e6e --- /dev/null +++ b/.ci/load/scripts/tests/simulate_app_start.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -exuo pipefail + +POLL_FREQ=1 + +function startApp() { + curl -s -X POST -H "Content-Type: application/json" -d \ + "{\"app_token\": \""$APP_TOKEN"\", \ + \"session_token\": \""$SESSION_TOKEN"\", \ + \"service\": \"application\", \ + \"hostname\": \"test_app\", \ + \"port\": \"999\"}" \ + $ORCH_URL/api/ready +} + +startApp diff --git a/.ci/load/scripts/tests/simulate_register.sh b/.ci/load/scripts/tests/simulate_register.sh new file mode 100755 index 0000000000..0ae322f2e8 --- /dev/null +++ b/.ci/load/scripts/tests/simulate_register.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -exuo pipefail + +function registerSession() { + curl -s -X POST -H "Content-Type: application/json" -d \ + "{\"app_token\": \""$APP_TOKEN"\", \ + \"service\": \"application\", \ + \"hostname\": \"test_app\", \ + \"port\": \"999\"}" \ + $ORCH_URL/api/register | jq '.session_created.session' +} + +registerSession