Merge pull request #6 from umanwizard/capi

Introduce `capi` package.

Cargo.toml

@@ -1,3 +1,6 @@
+[workspace]
+members = ["pure", "capi"]
+
 [package]
 name = "jemalloc_pprof"
 description = "Convert jemalloc heap profiles to pprof to understand memory usage, fix memory leaks, and fix OOM Kills."
@@ -15,7 +18,7 @@ categories = ["development-tools", "development-tools::profiling", "development-
 documentation = "https://docs.rs/jemalloc_pprof/latest/jemalloc_pprof/"
 homepage = "https://crates.io/crates/jemalloc_pprof"
 
-[dependencies]
+[workspace.dependencies]
 anyhow = "1.0.66"
 flate2 = "1.0.24"
 libc = "0.2.138"
@@ -27,3 +30,14 @@ tracing = "0.1.37"
 tokio = { version = "1.32.0", features = ["time", "sync"] }
 paste = "1.0.11"
 num = "0.4.0"
+errno = "0.3.8"
+
+[dependencies]
+pure = { path = "./pure" }
+libc.workspace = true
+anyhow.workspace = true
+tikv-jemalloc-ctl.workspace = true
+once_cell.workspace = true
+tracing.workspace = true
+tempfile.workspace = true
+tokio.workspace = true

README.md

@@ -104,3 +104,240 @@ The way this library works is that it creates a new temporary file (in the [plat
 Polar Signals Cloud allows continuously collecting heap profiling data, so you always have the right profiling data available, and don't need to search for the right data, you already have it!
 
 Polar Signals Cloud supports anything in the pprof format, so a process exposing the above explained pprof endpoint, can then be scraped as elaborated in the [scraping docs](https://www.polarsignals.com/docs/setup-scraper).
+
+## Use from C or C++
+
+The functionality to dump the current jemalloc heap profile in pprof
+format is exposed to C and C++ (or any other language that can use
+jemalloc and can link against libraries via the C ABI). This
+functionality is exposed via the `capi` (C API) package.
+
+### Building
+
+The following prerequisites are necessary to build the C API package:
+
+* Working Rust and C toolchains. The former can be installed by
+  following the instructions at https://rustup.rs . The latter can be
+  installed via the distribution's package manager. For example, on
+  Ubuntu, run `sudo apt install build-essential`.
+* `jemalloc` and its development headers. For example, on Ubuntu, run
+  `sudo apt install jemalloc-dev`.
+
+Once the prerequisites are installed, the library can be built by
+running `cargo build -p capi --release`. There are three files of
+interest:
+
+* The library itself, produced at
+  `target/release/libjemalloc_pprof.so`
+* A header file, at `capi/include/jemalloc_pprof.h`
+* A manual page, at `capi/man/jemalloc_pprof.3`.
+
+The procedure for installing and using these files depends on your
+distribution and build system.
+
+### Use
+
+Ensure that your binaries link against both jemalloc and
+jemalloc_pprof by passing the linker flags `-ljemalloc
+-ljemalloc_pprof`. The procedure for ensuring that these flags are
+passed depends on your build system and is currently outside the scope
+of this document.
+
+Once that is done, profiling can be enabled either by setting the
+`MALLOC_CONF` variable or by defining a symbol called `malloc_conf` in
+the binary. For example:
+
+``` shell
+export MALLOC_CONF="prof:true,prof_active:true,lg_prof_sample:19"
+```
+
+See the `jemalloc` man page for more details. When profiling is
+enabled, a profile may be dumped in pprof format via the
+`dump_jemalloc_pprof` function.
+
+### Example
+
+This program allocates between 1 and 10 MiB every 100 milliseconds,
+and dumps a profile to the file `my_profile` every 2 seconds.
+
+``` c
+#include <assert.h>
+#include <errno.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <pthread.h>
+#include <stdio.h>
+
+#include <jemalloc_pprof.h>
+
+void
+a()
+{
+        size_t sz = 1 * 1024 * 1024;
+        char *x = malloc(sz);
+        for (size_t i = 0; i < sz; ++i) {
+                x[i] = '\0';
+        }
+}
+
+void
+b()
+{
+        size_t sz = 2 * 1024 * 1024;
+        char *x = malloc(sz);
+        for (size_t i = 0; i < sz; ++i) {
+                x[i] = '\0';
+        }
+}
+
+void
+c()
+{
+        size_t sz = 3 * 1024 * 1024;
+        char *x = malloc(sz);
+        for (size_t i = 0; i < sz; ++i) {
+                x[i] = '\0';
+        }
+}
+
+void
+d()
+{
+        size_t sz = 4 * 1024 * 1024;
+        char *x = malloc(sz);
+        for (size_t i = 0; i < sz; ++i) {
+                x[i] = '\0';
+        }
+}
+
+void
+e()
+{
+        size_t sz = 5 * 1024 * 1024;
+        char *x = malloc(sz);
+        for (size_t i = 0; i < sz; ++i) {
+                x[i] = '\0';
+        }
+}
+
+void
+f()
+{
+        size_t sz = 6 * 1024 * 1024;
+        char *x = malloc(sz);
+        for (size_t i = 0; i < sz; ++i) {
+                x[i] = '\0';
+        }
+}
+
+void
+g()
+{
+        size_t sz = 7 * 1024 * 1024;
+        char *x = malloc(sz);
+        for (size_t i = 0; i < sz; ++i) {
+                x[i] = '\0';
+        }
+}
+
+void
+h()
+{
+        size_t sz = 8 * 1024 * 1024;
+        char *x = malloc(sz);
+        for (size_t i = 0; i < sz; ++i) {
+                x[i] = '\0';
+        }
+}
+
+void
+j()
+{
+        size_t sz = 9 * 1024 * 1024;
+        char *x = malloc(sz);
+        for (size_t i = 0; i < sz; ++i) {
+                x[i] = '\0';
+        }
+}
+
+void
+k()
+{
+        size_t sz = 10 * 1024 * 1024;
+        char *x = malloc(sz);
+        for (size_t i = 0; i < sz; ++i) {
+                x[i] = '\0';
+        }
+}
+
+void *
+repeatedly_dump(void *ignored)
+{
+        char *buf;
+        size_t len = 0;
+        int result;
+        for (;;) {
+                sleep(2);
+                result = dump_jemalloc_pprof(&buf, &len);
+                if (result != JP_SUCCESS) {
+                        fprintf(stderr, "errno: %d\n", errno);
+                        continue;
+                }
+                if (buf) {                        
+                        FILE *file = fopen("my_profile", "w");
+                        assert(file);
+
+                        fwrite(buf, sizeof(char), len, file);    
+                        fclose(file);
+                        printf("dumped pprof of size %lu\n", len);
+                        free(buf);
+                }
+        }
+        return NULL;
+}
+
+int
+main()
+{
+        pthread_t tid;
+        int result;
+
+        result = pthread_create(&tid, NULL, repeatedly_dump, NULL);
+        assert(!result);
+        for (;;) {
+                usleep(100000);
+                switch (rand() % 10) {
+                case 0:
+                        a();
+                        break;
+                case 1:
+                        b();
+                        break;
+                case 2:
+                        c();
+                        break;
+                case 3:
+                        d();
+                        break;
+                case 4:
+                        e();
+                        break;
+                case 5:
+                        f();
+                        break;
+                case 6:
+                        g();
+                        break;
+                case 7:
+                        h();
+                        break;
+                case 8:
+                        j();
+                        break;
+                case 9:
+                        k();
+                        break;
+                }
+        }
+}
+```

capi/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "capi"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+crate-type = ["cdylib"]
+name = "jemalloc_pprof"
+
+[dependencies]
+libc.workspace = true
+anyhow.workspace = true
+tempfile.workspace = true
+pure.path = "../pure"
+errno.workspace = true

capi/include/jemalloc_pprof.h

@@ -0,0 +1,14 @@
+#ifndef JEMALLOC_PPROF_H
+#define JEMALLOC_PPROF_H
+
+#include <stddef.h>
+
+#define JP_SUCCESS 0
+#define JP_FAILURE 1
+
+#ifdef __cplusplus
+extern "C"
+#endif
+int dump_jemalloc_pprof(char **buf_out, size_t *n_out);
+
+#endif // include guard

capi/man/dump_jemalloc_pprof.3

@@ -0,0 +1,33 @@
+.TH dump_jemalloc_pprof 3 "rust-jemalloc-pprof manual"
+.SH NAME
+dump_jemalloc_pprof \- dump the current jemalloc allocation profile in pprof format
+
+.SH SYNOPSIS
+\fB#include <jemalloc_pprof.h>\fR
+
+\fBint dump_jemalloc_pprof(\fR\fIunsigned char **buf_out, size_t *n_out\fR\fB);\fR
+
+.SH DESCRIPTION
+The \fBdump_jemalloc_pprof()\fR function is intended to be called from C code to dump the current jemalloc allocation profile in pprof format. It allocates a buffer and stores a pointer to this buffer in \fI*buf_out\fR. The size of the buffer is stored in \fI*n_out\fR. The function returns \fIJP_SUCCESS\fR if the operation succeeds or \fIJP_FAILURE\fR if it fails. In the case of failure, an error code may be stored in \fIerrno\fR if meaningful.
+
+If \fIJP_FAILURE\fR is returned, the values pointed to by \fIbuf_out\fR and \fIn_out\fR are unspecified.
+
+This function requires jemalloc profiling to be enabled and active. For more information, see \fBjemalloc(3)\fR.
+
+.SH SAFETY
+This function is marked as unsafe and should not be called directly from Rust code. A corresponding Rust API should be used instead.
+
+.SH "RETURN VALUES"
+.TP
+\fBJP_SUCCESS\fR
+Indicates that the operation succeeded.
+.TP
+\fBJP_FAILURE\fR
+Indicates that the operation failed. The specific error code can be found in \fIerrno\fR.
+
+.SH "SEE ALSO"
+\fBjemalloc(3)\fR, \fBerrno(3)\fR
+
+.SH AUTHOR
+This library was written by the Polar Signals team. See
+https://polarsignals.com for more information.