Profile low-privileged processes with perf_events (#411)

Makefile

@@ -19,7 +19,7 @@ JAR=$(JAVA_HOME)/bin/jar
 JAVAC_RELEASE_VERSION=7
 
 SOURCES := $(wildcard src/*.cpp)
-HEADERS := $(wildcard src/*.h)
+HEADERS := $(wildcard src/*.h src/fdtransfer/*.h)
 JAVA_HEADERS := $(patsubst %.java,%.class.h,$(wildcard src/helper/one/profiler/*.java))
 API_SOURCES := $(wildcard src/api/one/profiler/*.java)
 CONVERTER_SOURCES := $(shell find src/converter -name '*.java')
@@ -35,6 +35,7 @@ ifeq ($(OS), Darwin)
   SOEXT=dylib
   PACKAGE_EXT=zip
   OS_TAG=macos
+  FDTRANSFER_BIN=
 else
   LIBS += -lrt
   INCLUDES += -I$(JAVA_HOME)/include/linux
@@ -45,6 +46,7 @@ else
   else
     OS_TAG=linux
   endif
+  FDTRANSFER_BIN=build/fdtransfer
 endif
 
 ARCH:=$(shell uname -m)
@@ -69,7 +71,7 @@ endif
 
 .PHONY: all release test clean
 
-all: build build/$(LIB_PROFILER) build/$(JATTACH) build/$(API_JAR) build/$(CONVERTER_JAR)
+all: build build/$(LIB_PROFILER) build/$(JATTACH) build/$(API_JAR) build/$(CONVERTER_JAR) $(FDTRANSFER_BIN)
 
 release: build $(PACKAGE_NAME).$(PACKAGE_EXT)
 
@@ -103,6 +105,9 @@ build/$(LIB_PROFILER_SO): $(SOURCES) $(HEADERS) $(JAVA_HEADERS)
 build/$(JATTACH): src/jattach/*.c src/jattach/*.h
 	$(CC) $(CFLAGS) -DJATTACH_VERSION=\"$(PROFILER_VERSION)-ap\" -o $@ src/jattach/*.c
 
+build/fdtransfer: src/fdtransfer/fdtransfer_server.cpp src/fdtransfer/fdTransfer_shared_linux.h src/jattach/psutil.c
+	$(CXX) $(CFLAGS) -o $@ $^
+
 build/$(API_JAR): $(API_SOURCES)
 	mkdir -p build/api
 	$(JAVAC) -source $(JAVAC_RELEASE_VERSION) -target $(JAVAC_RELEASE_VERSION) -d build/api $^
@@ -126,6 +131,7 @@ test: all
 	test/thread-smoke-test.sh
 	test/alloc-smoke-test.sh
 	test/load-library-test.sh
+	test/fdtransfer-smoke-test.sh
 	echo "All tests passed"
 
 clean:

README.md

@@ -427,6 +427,11 @@ The following is a complete list of the command-line options accepted by
   only events between the safepoint request and the start of the VM operation
   will be recorded.
 
+* `--fdtransfer` - runs "fdtransfer" alongside, which is a small program providing an interface
+  for the profiler to access, `perf_event_open` even while this syscall is unavailable for the
+  profiled process (due to low privileges).
+  See [Profiling Java in a container](#profiling-java-in-a-container).
+
 * `-v`, `--version` - prints the version of profiler library. If PID is specified,
   gets the version of the library loaded into the given process.
 
@@ -446,13 +451,12 @@ the target container can access `libasyncProfiler.so` by the same
 absolute path as on the host.
 
 By default, Docker container restricts the access to `perf_event_open`
-syscall. So, in order to allow profiling inside a container, you'll need
-to modify [seccomp profile](https://docs.docker.com/engine/security/seccomp/)
+syscall. There are 3 alternatives to allow profiling in a container:
+1. You can modify the [seccomp profile](https://docs.docker.com/engine/security/seccomp/)
 or disable it altogether with `--security-opt seccomp=unconfined` option. In
 addition, `--cap-add SYS_ADMIN` may be required.
-
-Alternatively, if changing Docker configuration is not possible,
-you may fall back to `-e itimer` profiling mode, see [Troubleshooting](#troubleshooting).
+2. You can use "fdtransfer": see the help for `--fdtransfer`.
+3. Last, you may fall back to `-e itimer` profiling mode, see [Troubleshooting](#troubleshooting).
 
 ## Restrictions/Limitations
 
@@ -537,7 +541,7 @@ Make sure the user of JVM process has permissions to access `libasyncProfiler.so
 For more information see [#78](https://github.com/jvm-profiling-tools/async-profiler/issues/78).
 
 ```
-No access to perf events. Try --all-user option or 'sysctl kernel.perf_event_paranoid=1'
+No access to perf events. Try --fdtransfer or --all-user option or 'sysctl kernel.perf_event_paranoid=1'
 ```
 or
 ```
@@ -547,10 +551,13 @@ Perf events unavailable
 
 Typical reasons include:
 1. `/proc/sys/kernel/perf_event_paranoid` is set to restricted mode (>=2).
-2. seccomp disables perf_event_open API in a container.
+2. seccomp disables `perf_event_open` API in a container.
 3. OS runs under a hypervisor that does not virtualize performance counters.
 4. perf_event_open API is not supported on this system, e.g. WSL.
 
+For permissions-related reasons (such as 1 and 2), using `--fdtransfer` while running the profiler
+as a privileged user will allow using perf_events.
+
 If changing the configuration is not possible, you may fall back to
 `-e itimer` profiling mode. It is similar to `cpu` mode, but does not
 require perf_events support. As a drawback, there will be no kernel

profiler.sh

@@ -41,6 +41,7 @@ usage() {
     echo "  --begin function  begin profiling when function is executed"
     echo "  --end function    end profiling when function is executed"
     echo "  --ttsp            time-to-safepoint profiling"
+    echo "  --fdtransfer      use fdtransfer to pass perf & kallsyms fds to the lower privileged target"
     echo ""
     echo "<pid> is a numeric process ID of the target JVM"
     echo "      or 'jps' keyword to find running JVM automatically"
@@ -85,7 +86,21 @@ check_if_terminated() {
     fi
 }
 
+run_fdtransfer() {
+    if [ "$ACTION" = "start" ] || [ "$ACTION" = "resume" ] ; then
+        nohup "$FDTRANSFER" "$PID"
+    elif [ "$ACTION" = "collect" ]; then
+        case "$1" in
+        "start"*) "$FDTRANSFER" "$PID"
+        esac
+    fi
+}
+
 jattach() {
+    if [ "$UNAME_S" = "Linux" ] && [ "$USE_FDTRANSFER" = "true" ]; then
+        run_fdtransfer "$1"
+    fi
+
     set +e
     "$JATTACH" "$PID" load "$PROFILER" true "$1,log=$LOG" > /dev/null
     RET=$?
@@ -113,6 +128,8 @@ jattach() {
 OPTIND=1
 SCRIPT_DIR="$(cd "$(dirname "$0")" > /dev/null 2>&1; pwd -P)"
 JATTACH=$SCRIPT_DIR/build/jattach
+FDTRANSFER=$SCRIPT_DIR/build/fdtransfer
+USE_FDTRANSFER="false"
 PROFILER=$SCRIPT_DIR/build/libasyncProfiler.so
 ACTION="collect"
 DURATION="60"
@@ -221,6 +238,10 @@ while [ $# -gt 0 ]; do
         --ttsp)
             PARAMS="$PARAMS,begin=SafepointSynchronize::begin,end=RuntimeService::record_safepoint_synchronized"
             ;;
+        --fdtransfer)
+            PARAMS="$PARAMS,fdtransfer"
+            USE_FDTRANSFER="true"
+            ;;
         --safe-mode)
             PARAMS="$PARAMS,safemode=$2"
             shift

src/arguments.cpp

@@ -84,6 +84,7 @@ static const Multiplier UNIVERSAL[] = {{'n', 1}, {'u', 1000}, {'m', 1000000}, {'
 //                       MODE is 'fp' (Frame Pointer), 'lbr' (Last Branch Record) or 'no'
 //     allkernel       - include only kernel-mode events
 //     alluser         - include only user-mode events
+//     fdtransfer      - use fdtransfer to pass fds to the profiler
 //     simple          - simple class names instead of FQN
 //     dot             - dotted class names
 //     sig             - print method signatures
@@ -264,6 +265,12 @@ Error Arguments::parse(const char* args) {
                     }
                 }
 
+            CASE("fdtransfer")
+                _fdtransfer = true;
+                if (value != NULL) {
+                    _fdtransfer_path = value;
+                }
+
             // Output style modifiers
             CASE("simple")
                 _style |= STYLE_SIMPLE;

src/arguments.h

@@ -147,6 +147,8 @@ class Arguments {
     bool _sched;
     int _style;
     CStack _cstack;
+    bool _fdtransfer;
+    const char *_fdtransfer_path;
     Output _output;
     long _chunk_size;
     long _chunk_time;
@@ -181,6 +183,8 @@ class Arguments {
         _sched(false),
         _style(0),
         _cstack(CSTACK_DEFAULT),
+        _fdtransfer(false),
+        _fdtransfer_path(NULL),
         _output(OUTPUT_NONE),
         _chunk_size(100 * 1024 * 1024),
         _chunk_time(3600),

src/fdTransfer_client.h

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2021 Andrei Pangin
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _FDTRANSFER_H
+#define _FDTRANSFER_H
+
+#ifdef __linux__
+
+#include "fdtransfer/fdTransfer_shared_linux.h"
+
+class FdTransferClient {
+  private:
+    static int _peer;
+
+    static int recvFd(unsigned int request_id, struct fd_response *resp, size_t resp_size);
+
+  public:
+    static bool connectToServer(const char *path, int pid);
+    static bool hasPeer() { return _peer != -1; }
+    static void closePeer() {
+        if (_peer != -1) {
+            close(_peer);
+            _peer = -1;
+        }
+    }
+
+    static int requestPerfFd(int *tid, struct perf_event_attr *attr);
+    static int requestKallsymsFd();
+};
+
+#else
+
+class FdTransferClient {
+  public:
+    static bool connectToServer(const char *path, int pid) { return false; }
+    static bool hasPeer() { return false; }
+    static void closePeer() { }
+};
+
+#endif // __linux__
+
+#endif // _FDTRANSFER_H