Add Usage Telemetry

.github/workflows/e2e-test.yml

@@ -64,6 +64,7 @@ jobs:
     env:
       MAX_MEMORY: 28000
       MAX_AVG_DURATION: 24
+      LIGHTPANDA_DISABLE_TELEMETRY: true
 
     runs-on: ubuntu-latest
 

.gitignore

@@ -3,3 +3,4 @@ zig-cache
 zig-out
 /vendor/netsurf/out
 /vendor/libiconv/
+lightpanda.id

README.md

@@ -111,6 +111,9 @@ await context.close();
 await browser.disconnect();
 ```
 
+### Telemetry
+By default, Lightpanda collects and sends usage telemetry. This can be disabled by setting an environment variable `LIGHTPANDA_DISABLE_TELEMETRY=true`. You can read Lightpanda's privacy policy at: [https://lightpanda.io/privacy-policy](https://lightpanda.io/privacy-policy).
+
 ## Status
 
 Lightpanda is still a work in progress and is currently at a Beta stage.

src/app.zig

@@ -0,0 +1,67 @@
+const std = @import("std");
+
+const Loop = @import("jsruntime").Loop;
+const Allocator = std.mem.Allocator;
+const Telemetry = @import("telemetry/telemetry.zig").Telemetry;
+
+const log = std.log.scoped(.app);
+
+pub const RunMode = enum {
+    serve,
+    fetch,
+};
+
+// Container for global state / objects that various parts of the system
+// might need.
+pub const App = struct {
+    loop: *Loop,
+    app_dir_path: ?[]const u8,
+    allocator: Allocator,
+    telemetry: Telemetry,
+
+    pub fn init(allocator: Allocator, run_mode: RunMode) !App {
+        const loop = try allocator.create(Loop);
+        errdefer allocator.destroy(loop);
+
+        loop.* = try Loop.init(allocator);
+        errdefer loop.deinit();
+
+        const app_dir_path = getAndMakeAppDir(allocator);
+        const telemetry = Telemetry.init(allocator, run_mode, app_dir_path);
+        errdefer telemetry.deinit();
+
+        return .{
+            .loop = loop,
+            .allocator = allocator,
+            .telemetry = telemetry,
+            .app_dir_path = app_dir_path,
+        };
+    }
+
+    pub fn deinit(self: *App) void {
+        if (self.app_dir_path) |app_dir_path| {
+            self.allocator.free(app_dir_path);
+        }
+
+        self.telemetry.deinit();
+        self.loop.deinit();
+        self.allocator.destroy(self.loop);
+    }
+};
+
+fn getAndMakeAppDir(allocator: Allocator) ?[]const u8 {
+    const app_dir_path = std.fs.getAppDataDir(allocator, "lightpanda") catch |err| {
+        log.warn("failed to get lightpanda data dir: {}", .{err});
+        return null;
+    };
+
+    std.fs.makeDirAbsolute(app_dir_path) catch |err| switch (err) {
+        error.PathAlreadyExists => return app_dir_path,
+        else => {
+            allocator.free(app_dir_path);
+            log.warn("failed to create lightpanda data dir: {}", .{err});
+            return null;
+        },
+    };
+    return app_dir_path;
+}

src/browser/browser.zig

@@ -33,6 +33,7 @@ const Loop = jsruntime.Loop;
 const Env = jsruntime.Env;
 const Module = jsruntime.Module;
 
+const App = @import("../app.zig").App;
 const apiweb = @import("../apiweb.zig");
 
 const Window = @import("../html/window.zig").Window;
@@ -59,7 +60,7 @@ pub const user_agent = "Lightpanda/1.0";
 // A browser contains only one session.
 // TODO allow multiple sessions per browser.
 pub const Browser = struct {
-    loop: *Loop,
+    app: *App,
     session: ?*Session,
     allocator: Allocator,
     http_client: HttpClient,
@@ -68,9 +69,10 @@ pub const Browser = struct {
 
     const SessionPool = std.heap.MemoryPool(Session);
 
-    pub fn init(allocator: Allocator, loop: *Loop) Browser {
+    pub fn init(app: *App) Browser {
+        const allocator = app.allocator;
         return .{
-            .loop = loop,
+            .app = app,
             .session = null,
             .allocator = allocator,
             .http_client = .{ .allocator = allocator },
@@ -109,6 +111,8 @@ pub const Browser = struct {
 // You can create successively multiple pages for a session, but you must
 // deinit a page before running another one.
 pub const Session = struct {
+    app: *App,
+
     browser: *Browser,
 
     // The arena is used only to bound the js env init b/c it leaks memory.
@@ -133,8 +137,10 @@ pub const Session = struct {
     jstypes: [Types.len]usize = undefined,
 
     fn init(self: *Session, browser: *Browser, ctx: anytype) !void {
-        const allocator = browser.allocator;
+        const app = browser.app;
+        const allocator = app.allocator;
         self.* = .{
+            .app = app,
             .env = undefined,
             .browser = browser,
             .inspector = undefined,
@@ -145,7 +151,7 @@ pub const Session = struct {
         };
 
         const arena = self.arena.allocator();
-        Env.init(&self.env, arena, browser.loop, null);
+        Env.init(&self.env, arena, app.loop, null);
         errdefer self.env.deinit();
         try self.env.load(&self.jstypes);
 
@@ -238,7 +244,7 @@ pub const Session = struct {
         std.debug.assert(self.page != null);
 
         // Reset all existing callbacks.
-        self.browser.loop.reset();
+        self.app.loop.reset();
 
         self.env.stop();
         // TODO unload document: https://html.spec.whatwg.org/#unloading-documents
@@ -359,6 +365,11 @@ pub const Page = struct {
 
         // TODO handle fragment in url.
 
+        self.session.app.telemetry.record(.{ .navigate = .{
+            .proxy = false,
+            .tls = std.ascii.eqlIgnoreCase(self.uri.scheme, "https"),
+        } });
+
         // load the data
         var resp = try self.session.loader.get(arena, self.uri);
         defer resp.deinit();

src/cdp/cdp.zig

@@ -20,7 +20,7 @@ const std = @import("std");
 const Allocator = std.mem.Allocator;
 const json = std.json;
 
-const Loop = @import("jsruntime").Loop;
+const App = @import("../app.zig").App;
 const asUint = @import("../str/parser.zig").asUint;
 const Incrementing = @import("../id.zig").Incrementing;
 
@@ -34,7 +34,6 @@ pub const TimestampEvent = struct {
 };
 
 pub const CDP = CDPT(struct {
-    const Loop = *@import("jsruntime").Loop;
     const Client = *@import("../server.zig").Client;
     const Browser = @import("../browser/browser.zig").Browser;
     const Session = @import("../browser/browser.zig").Session;
@@ -47,8 +46,6 @@ const BrowserContextIdGen = Incrementing(u32, "BID");
 // Generic so that we can inject mocks into it.
 pub fn CDPT(comptime TypeProvider: type) type {
     return struct {
-        loop: TypeProvider.Loop,
-
         // Used for sending message to the client and closing on error
         client: TypeProvider.Client,
 
@@ -73,13 +70,13 @@ pub fn CDPT(comptime TypeProvider: type) type {
         pub const Browser = TypeProvider.Browser;
         pub const Session = TypeProvider.Session;
 
-        pub fn init(allocator: Allocator, client: TypeProvider.Client, loop: TypeProvider.Loop) Self {
+        pub fn init(app: *App, client: TypeProvider.Client) Self {
+            const allocator = app.allocator;
             return .{
-                .loop = loop,
                 .client = client,
                 .allocator = allocator,
                 .browser_context = null,
-                .browser = Browser.init(allocator, loop),
+                .browser = Browser.init(app),
                 .message_arena = std.heap.ArenaAllocator.init(allocator),
                 .browser_context_pool = std.heap.MemoryPool(BrowserContext(Self)).init(allocator),
             };

src/cdp/testing.zig

@@ -7,6 +7,7 @@ const Testing = @This();
 
 const main = @import("cdp.zig");
 const parser = @import("netsurf");
+const App = @import("../app.zig").App;
 
 pub const expectEqual = std.testing.expectEqual;
 pub const expectError = std.testing.expectError;
@@ -16,10 +17,9 @@ const Browser = struct {
     session: ?*Session = null,
     arena: std.heap.ArenaAllocator,
 
-    pub fn init(allocator: Allocator, loop: anytype) Browser {
-        _ = loop;
+    pub fn init(app: *App) Browser {
         return .{
-            .arena = std.heap.ArenaAllocator.init(allocator),
+            .arena = std.heap.ArenaAllocator.init(app.allocator),
         };
     }
 
@@ -112,13 +112,13 @@ const Client = struct {
 };
 
 const TestCDP = main.CDPT(struct {
-    pub const Loop = void;
     pub const Browser = Testing.Browser;
     pub const Session = Testing.Session;
     pub const Client = *Testing.Client;
 });
 
 const TestContext = struct {
+    app: App,
     client: ?Client = null,
     cdp_: ?TestCDP = null,
     arena: std.heap.ArenaAllocator,
@@ -127,6 +127,7 @@ const TestContext = struct {
         if (self.cdp_) |*c| {
             c.deinit();
         }
+        self.app.deinit();
         self.arena.deinit();
     }
 
@@ -135,7 +136,7 @@ const TestContext = struct {
             self.client = Client.init(self.arena.allocator());
             // Don't use the arena here. We want to detect leaks in CDP.
             // The arena is only for test-specific stuff
-            self.cdp_ = TestCDP.init(std.testing.allocator, &self.client.?, {});
+            self.cdp_ = TestCDP.init(&self.app, &self.client.?);
         }
         return &self.cdp_.?;
     }
@@ -262,6 +263,7 @@ const TestContext = struct {
 
 pub fn context() TestContext {
     return .{
+        .app = App.init(std.testing.allocator, .serve) catch unreachable,
         .arena = std.heap.ArenaAllocator.init(std.testing.allocator),
     };
 }

src/id.zig

@@ -66,7 +66,7 @@ pub fn Incrementing(comptime T: type, comptime prefix: []const u8) type {
     };
 }
 
-fn uuidv4(hex: []u8) void {
+pub fn uuidv4(hex: []u8) void {
     std.debug.assert(hex.len == 36);
 
     var bin: [16]u8 = undefined;

src/main.zig

@@ -31,6 +31,7 @@ const apiweb = @import("apiweb.zig");
 pub const Types = jsruntime.reflect(apiweb.Interfaces);
 pub const UserContext = apiweb.UserContext;
 pub const IO = @import("asyncio").Wrapper(jsruntime.Loop);
+const version = @import("build_info").git_commit;
 
 const log = std.log.scoped(.cli);
 
@@ -60,7 +61,7 @@ pub fn main() !void {
     switch (args.mode) {
         .help => args.printUsageAndExit(args.mode.help),
         .version => {
-            std.debug.print("{s}\n", .{@import("build_info").git_commit});
+            std.debug.print("{s}\n", .{version});
             return std.process.cleanExit();
         },
         .serve => |opts| {
@@ -69,28 +70,29 @@ pub fn main() !void {
                 return args.printUsageAndExit(false);
             };
 
-            var loop = try jsruntime.Loop.init(alloc);
-            defer loop.deinit();
+            var app = try @import("app.zig").App.init(alloc, .serve);
+            defer app.deinit();
+            app.telemetry.record(.{ .run = {} });
 
             const timeout = std.time.ns_per_s * @as(u64, opts.timeout);
-            server.run(alloc, address, timeout, &loop) catch |err| {
+            server.run(&app, address, timeout) catch |err| {
                 log.err("Server error", .{});
                 return err;
             };
         },
         .fetch => |opts| {
             log.debug("Fetch mode: url {s}, dump {any}", .{ opts.url, opts.dump });
 
+            var app = try @import("app.zig").App.init(alloc, .fetch);
+            defer app.deinit();
+            app.telemetry.record(.{ .run = {} });
+
             // vm
             const vm = jsruntime.VM.init();
             defer vm.deinit();
 
-            // loop
-            var loop = try jsruntime.Loop.init(alloc);
-            defer loop.deinit();
-
             // browser
-            var browser = Browser.init(alloc, &loop);
+            var browser = Browser.init(&app);
             defer browser.deinit();
 
             var session = try browser.newSession({});

src/server.zig

@@ -34,6 +34,7 @@ const CloseError = jsruntime.IO.CloseError;
 const CancelError = jsruntime.IO.CancelOneError;
 const TimeoutError = jsruntime.IO.TimeoutError;
 
+const App = @import("app.zig").App;
 const CDP = @import("cdp/cdp.zig").CDP;
 
 const TimeoutCheck = std.time.ns_per_ms * 100;
@@ -48,6 +49,7 @@ const MAX_HTTP_REQUEST_SIZE = 2048;
 const MAX_MESSAGE_SIZE = 256 * 1024 + 14;
 
 const Server = struct {
+    app: *App,
     allocator: Allocator,
     loop: *jsruntime.Loop,
 
@@ -70,7 +72,6 @@ const Server = struct {
 
     fn deinit(self: *Server) void {
         self.client_pool.deinit();
-        self.allocator.free(self.json_version_response);
     }
 
     fn queueAccept(self: *Server) void {
@@ -465,7 +466,7 @@ pub const Client = struct {
         };
 
         self.mode = .websocket;
-        self.cdp = CDP.init(self.server.allocator, self, self.server.loop);
+        self.cdp = CDP.init(self.server.app, self);
         return self.send(arena, response);
     }
 
@@ -1014,10 +1015,9 @@ fn websocketHeader(buf: []u8, op_code: OpCode, payload_len: usize) []const u8 {
 }
 
 pub fn run(
-    allocator: Allocator,
+    app: *App,
     address: net.Address,
     timeout: u64,
-    loop: *jsruntime.Loop,
 ) !void {
     // create socket
     const flags = posix.SOCK.STREAM | posix.SOCK.CLOEXEC | posix.SOCK.NONBLOCK;
@@ -1040,9 +1040,13 @@ pub fn run(
     const vm = jsruntime.VM.init();
     defer vm.deinit();
 
+    var loop = app.loop;
+    const allocator = app.allocator;
     const json_version_response = try buildJSONVersionResponse(allocator, address);
+    defer allocator.free(json_version_response);
 
     var server = Server{
+        .app = app,
         .loop = loop,
         .timeout = timeout,
         .listener = listener,