0005-pythonrun.c-telemetry-patch.patch

From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: William Douglas <william.douglas@intel.com>
Date: Thu, 31 Oct 2019 14:16:48 -0700
Subject: [PATCH] pythonrun.c: telemetry patch

If the telemetry library is available, the patch taps into Python unhandled exception code
and creates a telemetry probe payload. The payload is typically a traceback, identical
to the output as displayed via stderr. The payload is sent to the backend server only if
telemetry is enabled and the user is opted in telemetry as well. The telemetry backend server
is based on telemetry configuration file. .

Implemetation:
If the telemetry library "libtelemetry.so" is available, (and we are not in Python
interactive mode), we pipe the error text produced by stderr (traceback) into a temporary
file. At the same time we also output all messages via the original stderr as well so the
user receives all messages as usual.

Once we captured the traceback in a file, we validate if the traceback originated
in /usr/bin. If so, we create a payload for the backend server using the libtelemetry
API functions. Note that even if the telemetry is installed but the user is not opted in
telemetry, nothing is reported to the backed server, this is how the telemetry API
is implemented.

If we encounter any errors, we do not create the telemetry payload and we do not
print any error messages. This should be the worst case scenario.

Signed-off-by: Juro Bystricky <juro.bystricky@intel.com>
---
 Python/pythonrun.c | 274 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 274 insertions(+)

diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index 70748dc..c7e23ce 100644
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -698,10 +698,281 @@ handle_system_exit(void)
 }
 
 
+/* *** Telemetry patch (code/definitions) start *** */
+
+#define TM_RECORD_VERSION 1
+#define TM_CLASS "org.clearlinux/python/python-exception"
+#define TM_SEVERITY 3
+
+#include <stdbool.h>
+#include <dlfcn.h>
+
+static int telemetry_fd_pipe[2];
+static int telemetry_fd_stderr_orig;
+static int telemetry_fd_file;
+static pthread_t telemetry_tid;
+static char telemetry_filename[]="/var/tmp/telemetry_temp.XXXXXX";
+
+#define TELEMETRY_LIB "/usr/lib64/libtelemetry.so.4"
+
+static void *tm_dlhandle;
+
+struct telem_ref {
+    struct telem_record *record;
+};
+
+static int (*tm_create_record)(struct telem_ref **, uint32_t, char *, uint32_t);
+static int (*tm_set_payload)(struct telem_ref *, char *);
+static int (*tm_send_record)(struct telem_ref *);
+static void (*tm_free_record)(struct telem_ref *);
+
+/* attempt to load libtelemetry functions */
+static bool load_telem_api(void)
+{
+    char *error;
+
+    tm_dlhandle = dlopen(TELEMETRY_LIB, RTLD_NOW);
+    if (!tm_dlhandle) {
+        /* No error, we just don't have telemetry */
+        return false;
+    }
+
+    tm_create_record = dlsym(tm_dlhandle, "tm_create_record");
+    if ((error = dlerror()) != NULL) {
+        goto err_out;
+    }
+
+    tm_set_payload = dlsym(tm_dlhandle, "tm_set_payload");
+    if ((error = dlerror()) != NULL) {
+        goto err_out;
+    }
+
+    tm_send_record = dlsym(tm_dlhandle, "tm_send_record");
+    if ((error = dlerror()) != NULL) {
+        goto err_out;
+    }
+
+    tm_free_record = dlsym(tm_dlhandle, "tm_free_record");
+    if ((error = dlerror()) != NULL) {
+         goto err_out;
+    }
+
+    return true;
+
+err_out:
+    if (tm_dlhandle) {
+        dlclose(tm_dlhandle);
+        tm_dlhandle = NULL;
+    }
+    return false;
+}
+
+/* send record to telemetry server
+ * only called if load_telem_api is successful
+ */
+static bool send_record(FILE* fp)
+{
+    struct telem_ref *handle = NULL;
+    struct stat st;
+    bool ret = false;
+    char *payload = NULL;
+    long fsize = 0, bytes_read = 0;
+
+    /* find the size of the record and allocate memory */
+    if (fstat(fileno(fp), &st) < 0) {
+        goto out;
+    }
+
+    if (!st.st_size) {
+        goto out;
+    }
+
+    fsize = st.st_size;
+
+    payload = (char *) malloc(fsize + 1);
+
+    /* Read the payload from the record file */
+    rewind(fp);
+    bytes_read = fread(payload, (size_t) fsize, 1, fp);
+
+    if (!bytes_read) {
+        goto out;
+    }
+
+    /* Make sure payload ends with a null char */
+    payload[fsize - 1] = '\0';
+
+    /* Create telemetry record with severity and classification */
+    if (tm_create_record(&handle, TM_SEVERITY, TM_CLASS, TM_RECORD_VERSION) < 0) {
+        goto out;
+    }
+
+    /* Set the payload for the telemetry record from the payload read */
+    if (tm_set_payload(handle, payload) < 0) {
+        goto out;
+    }
+
+    /* Send the record to the telemetry server based on configuration file.
+     * This call may fail if the telemetry is installed but user not opted-in
+     */
+    if (tm_send_record(handle) < 0 ) {
+        goto out;
+    }
+
+    /* Clean up: free the record, free the payload */
+    ret = true;
+
+out:
+    if (handle) {
+        tm_free_record(handle);
+    }
+
+    if (payload) {
+        free(payload);
+    }
+
+    return ret;
+}
+
+/* Parse a line in captured tracback. Return false to prevent sending */
+/* Beware some traceback messages do not contain any path, we don't want those */
+static bool telemetry_parse_line(char *line, bool *path_ok)
+{
+    char *src = line;
+
+    while (isspace((unsigned char)*src)) {
+        src++;
+    }
+
+    /* Only interested in exceptions in /usr/bin */
+    if (strncmp(src, "File \"/usr/bin/", (sizeof("File \"/usr/bin/") - 1)) == 0) {
+        *path_ok = true;
+        return true;
+    }
+
+    /* Reject any other file path */
+    if (strncmp(src, "File \"", (sizeof("File \"") - 1)) == 0) {
+        return false;
+    }
+
+   return true;
+}
+
+static bool telemetry_process_log(int fd)
+{
+    bool result, path_ok;
+    char line[256];
+
+    FILE* fp = fdopen(fd, "r");
+    if (fp == NULL) {
+        close(fd);
+        return false;
+    }
+
+    rewind(fp);
+    result = true;
+    path_ok = false;
+
+    while (fgets(line, sizeof(line), fp)) {
+        if (telemetry_parse_line(line, &path_ok) == false) {
+            result = false;
+            break;
+        }
+    }
+
+    if (result && path_ok) {
+        send_record(fp);
+    }
+
+    fclose(fp); /* this will also close fd */
+    return (result && path_ok);
+}
+
+static void *telemetry_reader(void *dummy)
+{
+    while (1) {
+        char c;
+
+        /* Check for errors */
+        if (read (telemetry_fd_pipe[0], &c, 1) < 0) {
+            break;
+        }
+
+        /* Check for our EOF */
+        if (c == '\0')
+            break;
+
+        write(telemetry_fd_stderr_orig, &c, 1);
+        write(telemetry_fd_file, &c, 1);
+    }
+
+    return NULL;
+}
+
+/* Start monitoring/capturing stderr output */
+static int telemetry_file_open(void)
+{
+    /* Nothing to do if telemetry not installed */
+    if (load_telem_api() == false) {
+        return -1;
+    }
+
+    /* Ignore exceptions in interactive mode */
+    if (PyId_ps1.object != NULL) {
+        return -1;
+    }
+
+    /* Create a file collecting the traceback */
+    telemetry_fd_file = mkstemp(telemetry_filename);
+    if (telemetry_fd_file < 0) {
+        return -1;
+    }
+
+    if (pipe(telemetry_fd_pipe) == -1) {
+        return -1;
+    }
+
+    /* Save stderr, redirect stderr to pipe and start monitoring the pipe*/
+    telemetry_fd_stderr_orig = dup(STDERR_FILENO);
+    dup2(telemetry_fd_pipe[1], STDERR_FILENO);
+    pthread_create(&telemetry_tid, NULL, telemetry_reader, NULL);
+    return telemetry_fd_file;
+}
+
+/* Stop monitoring/capturing stderr output */
+static void telemetry_file_close(int fd)
+{
+    if (fd == -1) {
+        return;
+    }
+
+    /* Flush internal buffers to our pipe */
+    flush_io();
+
+    /* Wait for the telemetry_reader thread to finish */
+    write(telemetry_fd_pipe[1],"\0",1);
+    pthread_join(telemetry_tid, NULL);
+
+    /* Restore the original stderr */
+    close(telemetry_fd_pipe[1]);
+    close(telemetry_fd_pipe[0]);
+    fsync(telemetry_fd_file);
+    dup2(telemetry_fd_stderr_orig, STDERR_FILENO);
+
+    /* Send the traceback to the server and clean up */
+    telemetry_process_log(fd);
+    dlclose(tm_dlhandle);
+    unlink(telemetry_filename);
+}
+
+/* *** Telemetry patch (code/definitions) end *** */
+
+
 static void
 _PyErr_PrintEx(PyThreadState *tstate, int set_sys_last_vars)
 {
     PyObject *exception, *v, *tb, *hook;
+    int fd_telem;
 
     handle_system_exit();
 
@@ -732,6 +1003,8 @@ _PyErr_PrintEx(PyThreadState *tstate, int set_sys_last_vars)
             _PyErr_Clear(tstate);
         }
     }
+
+    fd_telem = telemetry_file_open();
     hook = _PySys_GetObjectId(&PyId_excepthook);
     if (_PySys_Audit(tstate, "sys.excepthook", "OOOO", hook ? hook : Py_None,
                      exception, v, tb) < 0) {
@@ -786,6 +1059,7 @@ done:
     Py_XDECREF(exception);
     Py_XDECREF(v);
     Py_XDECREF(tb);
+    telemetry_file_close(fd_telem);
 }
 
 void