feat: NDJSON protocol mode + parallel test execution

Common/source/tcpverbs.c

@@ -95,6 +95,9 @@ static pthread_mutex_t g_listeners_mutex = PTHREAD_MUTEX_INITIALIZER;
  * tcp_process_callbacks() periodically (e.g., in REPL loop or idle handler).
  * ======================================================================== */
 
+#define TCP_CONNECT_TIMEOUT_SECS 10   /* Non-blocking connect timeout */
+#define TCP_SOCKET_TIMEOUT_SECS  30   /* Send/receive timeout */
+
 #define TCP_CALLBACK_QUEUE_SIZE 64
 
 typedef struct tcp_callback_item {
@@ -509,6 +512,70 @@ boolean tcp_address_decode(long addr, bigstring ip_string_out) {
     return true;
 }
 
+/* Connect with timeout using non-blocking socket + select().
+ * Returns 0 on success, -1 on failure/timeout. */
+static int tcp_connect_with_timeout(int sockfd, const struct sockaddr *addr,
+                                     socklen_t addrlen, int timeout_secs) {
+    int flags, ret, err;
+    socklen_t len;
+    fd_set wfds;
+    struct timeval tv;
+
+    /* Set non-blocking for connect */
+    flags = fcntl(sockfd, F_GETFL, 0);
+    if (flags < 0)
+        return -1;
+    if (fcntl(sockfd, F_SETFL, flags | O_NONBLOCK) < 0)
+        return -1;
+
+    ret = connect(sockfd, addr, addrlen);
+    if (ret == 0) {
+        /* Connected immediately (e.g. localhost) — restore blocking */
+        fcntl(sockfd, F_SETFL, flags);
+        return 0;
+    }
+    if (errno != EINPROGRESS) {
+        fcntl(sockfd, F_SETFL, flags);
+        return -1;
+    }
+
+    /* Wait for connect to complete or timeout */
+    FD_ZERO(&wfds);
+    FD_SET(sockfd, &wfds);
+    tv.tv_sec = timeout_secs;
+    tv.tv_usec = 0;
+
+    ret = select(sockfd + 1, NULL, &wfds, NULL, &tv);
+    if (ret <= 0) {
+        /* Timeout (ret==0) or error (ret<0) */
+        fcntl(sockfd, F_SETFL, flags);
+        return -1;
+    }
+
+    /* Check actual connection result */
+    err = 0;
+    len = sizeof(err);
+    if (getsockopt(sockfd, SOL_SOCKET, SO_ERROR, &err, &len) < 0 || err != 0) {
+        fcntl(sockfd, F_SETFL, flags);
+        return -1;
+    }
+
+    /* Restore blocking mode */
+    fcntl(sockfd, F_SETFL, flags);
+    return 0;
+}
+
+/* Set send/receive timeouts on a connected socket */
+static void tcp_set_socket_timeouts(int sockfd, int timeout_secs) {
+    struct timeval tv;
+    tv.tv_sec = timeout_secs;
+    tv.tv_usec = 0;
+    if (setsockopt(sockfd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) < 0)
+        log_debug(LOG_COMP_LANG, "tcp_set_socket_timeouts: SO_RCVTIMEO failed (errno=%d)", errno);
+    if (setsockopt(sockfd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)) < 0)
+        log_debug(LOG_COMP_LANG, "tcp_set_socket_timeouts: SO_SNDTIMEO failed (errno=%d)", errno);
+}
+
 /* ========================================================================
  * Phase 1A: Core Socket Operations
  * ======================================================================== */
@@ -564,14 +631,17 @@ boolean tcp_open_stream_addr(long addr, long port, long *stream_id_out) {
     server_addr.sin_port = htons((uint16_t)port);
     server_addr.sin_addr.s_addr = htonl((uint32_t)addr);  /* Convert to network byte order */
 
-    /* Connect (blocking) */
-    if (connect(sockfd, (struct sockaddr*)&server_addr, sizeof(server_addr)) < 0) {
-        int saved_errno = errno;
+    /* Connect with timeout */
+    if (tcp_connect_with_timeout(sockfd, (struct sockaddr*)&server_addr,
+                                  sizeof(server_addr), TCP_CONNECT_TIMEOUT_SECS) < 0) {
         close(sockfd);
-        tcp_set_error(tcp_map_errno(saved_errno), "Connection refused");
+        tcp_set_error(TCP_ERR_CONNECTION_FAILED, "Connection failed or timed out");
         return false;
     }
 
+    /* Set send/receive timeouts */
+    tcp_set_socket_timeouts(sockfd, TCP_SOCKET_TIMEOUT_SECS);
+
     log_debug(LOG_COMP_LANG, "tcp_open_stream_addr: connected, allocating stream ID");
 
     /* Allocate stream ID */
@@ -803,6 +873,15 @@ boolean tcp_write_stream(long stream_id, Handle hdata) {
         }
 
         total_written += bytes_written;
+
+        /* Yield to other threads every 64KB */
+        if ((total_written % 65536) < bytes_written && !langbackgroundtask(false)) {
+            /* User cancelled */
+            unlockhandle(hdata);
+            tcp_stream_release(stream);
+            tcp_set_error(TCP_ERR_SOCKET_ERROR, "Write cancelled");
+            return false;
+        }
     }
 
     unlockhandle(hdata);
@@ -1121,9 +1200,11 @@ boolean tcp_open_stream_name(bigstring hostname, long port, long *stream_id_out)
         int optval = 1;
         setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval));
 
-        /* Attempt connection (blocking) */
-        if (connect(sockfd, rp->ai_addr, rp->ai_addrlen) == 0) {
-            /* Success! */
+        /* Attempt connection with timeout */
+        if (tcp_connect_with_timeout(sockfd, rp->ai_addr, rp->ai_addrlen,
+                                      TCP_CONNECT_TIMEOUT_SECS) == 0) {
+            /* Success! Set send/receive timeouts */
+            tcp_set_socket_timeouts(sockfd, TCP_SOCKET_TIMEOUT_SECS);
             break;
         }
 

frontier-cli/Makefile

@@ -77,6 +77,8 @@ CLI_SOURCES = \
     cli_parser.c \
     cli_utils.c \
     cli_executor.c \
+    cli_json_output.c \
+    protocol_handler.c \
     headless_scripts_loader.c \
     terminal_control.c \
     dialog_prompts.c \

frontier-cli/cli_executor.c

@@ -7,6 +7,7 @@
  */
 
 #include "cli_executor.h"
+#include "cli_json_output.h"
 #include "repl_output.h"  /* For kMacRomanHighToUnicode, putc_utf8 */
 
 /* 2025-12-08 Codex: Route long inline CLI scripts through langrunhandle so compiled evals return results without bogus empty verb names. */
@@ -16,7 +17,6 @@
 #include <string.h>
 
 // Forward declarations for static helper functions
-static void cli_print_json_escaped_string(const char* str);
 static void cli_print_execution_result_json(const usertalk_execution_t* execution, boolean success);
 
 /* Duplicates a C string using CLI memory allocation. */
@@ -279,47 +279,10 @@ void cli_print_execution_result(const usertalk_execution_t* execution) {
     putc('\n', stdout);
 }
 
-// Helper function to escape JSON strings
-//
-// NOTE: This function expects UTF-8 encoded strings and passes multi-byte
-// UTF-8 sequences through directly (valid per JSON RFC 8259). Control characters
-// (< 32) are escaped as \uXXXX. Invalid UTF-8 sequences may produce malformed JSON.
-//
-// Current Frontier string encoding: Pascal strings (bigstring/pstring) are
-// length-prefixed byte arrays, typically ASCII or MacRoman. Future roadmap
-// includes UTF-8 migration for runtime strings.
-//
-// For now, this handles ASCII and valid UTF-8 correctly. Edge cases with invalid
-// multi-byte sequences will be addressed when runtime string encoding is unified.
+// Helper function to escape JSON strings - delegates to shared utility.
+// See cli_json_output.c for encoding notes (ASCII, UTF-8, MacRoman).
 static void cli_print_json_escaped_string(const char* str) {
-    if (str == NULL) {
-        printf( "null");
-        return;
-    }
-
-    printf( "\"");
-    for (const char* p = str; *p != '\0'; p++) {
-        switch (*p) {
-            case '"':  printf( "\\\""); break;
-            case '\\': printf( "\\\\"); break;
-            case '\b': printf( "\\b"); break;
-            case '\f': printf( "\\f"); break;
-            case '\n': printf( "\\n"); break;
-            case '\r': printf( "\\r"); break;
-            case '\t': printf( "\\t"); break;
-            default:
-                if ((unsigned char)*p < 32) {
-                    // Escape control characters
-                    printf( "\\u%04x", (unsigned char)*p);
-                } else {
-                    // Pass through printable ASCII and UTF-8 multi-byte sequences
-                    // JSON spec (RFC 8259) allows unescaped UTF-8
-                    putchar(*p);
-                }
-                break;
-        }
-    }
-    printf( "\"");
+    cli_json_write_escaped_string(stdout, str);
 }
 
 // Print execution result as JSON to stdout for clean separation from prompts/logs

frontier-cli/cli_json_output.c

@@ -0,0 +1,61 @@
+/*
+ * Frontier CLI - Shared JSON Output Utilities
+ *
+ * cli_json_output.c - JSON string escaping and output helpers
+ *
+ * Extracted from cli_executor.c to be shared with protocol_handler.c.
+ *
+ * Copyright (C) 1992-2004 UserLand Software, Inc.
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include "cli_json_output.h"
+
+/* Internal: escape one character to the stream. */
+static void write_escaped_char(FILE *stream, char c) {
+    switch (c) {
+        case '"':  fputs("\\\"", stream); break;
+        case '\\': fputs("\\\\", stream); break;
+        case '\b': fputs("\\b", stream);  break;
+        case '\f': fputs("\\f", stream);  break;
+        case '\n': fputs("\\n", stream);  break;
+        case '\r': fputs("\\r", stream);  break;
+        case '\t': fputs("\\t", stream);  break;
+        default:
+            if ((unsigned char)c < 32) {
+                fprintf(stream, "\\u%04x", (unsigned char)c);
+            } else {
+                fputc(c, stream);
+            }
+            break;
+    }
+}
+
+void cli_json_write_escaped_string(FILE *stream, const char *str) {
+    if (str == NULL) {
+        fputs("null", stream);
+        return;
+    }
+
+    fputc('"', stream);
+    for (const char *p = str; *p != '\0'; p++) {
+        write_escaped_char(stream, *p);
+    }
+    fputc('"', stream);
+}
+
+void cli_json_write_escaped_buffer(FILE *stream, const char *buf, long len) {
+    if (buf == NULL) {
+        fputs("null", stream);
+        return;
+    }
+
+    fputc('"', stream);
+    for (long i = 0; i < len; i++) {
+        write_escaped_char(stream, buf[i]);
+    }
+    fputc('"', stream);
+}

frontier-cli/cli_json_output.h

@@ -0,0 +1,35 @@
+/*
+ * Frontier CLI - Shared JSON Output Utilities
+ *
+ * cli_json_output.h - JSON string escaping and output helpers
+ *
+ * Copyright (C) 1992-2004 UserLand Software, Inc.
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef CLI_JSON_OUTPUT_H
+#define CLI_JSON_OUTPUT_H
+
+#include <stdio.h>
+
+/*
+ * Write a JSON-escaped string (with surrounding quotes) to the given stream.
+ *
+ * Handles: \", \\, \b, \f, \n, \r, \t, and \uXXXX for control chars < 32.
+ * Passes through printable ASCII and valid UTF-8 multi-byte sequences.
+ *
+ * If str is NULL, writes the JSON literal "null" (without quotes).
+ */
+void cli_json_write_escaped_string(FILE *stream, const char *str);
+
+/*
+ * Write a JSON-escaped string from a buffer with explicit length.
+ * Same escaping rules as cli_json_write_escaped_string but does not
+ * rely on NUL termination — uses len bytes starting at buf.
+ */
+void cli_json_write_escaped_buffer(FILE *stream, const char *buf, long len);
+
+#endif /* CLI_JSON_OUTPUT_H */

frontier-cli/cli_parser.c

@@ -99,6 +99,14 @@ boolean cli_validate_options(const cli_options_t* options) {
         }
     }
 
+    /* --protocol mode: incompatible with script/inline execution */
+    if (options->protocol_mode) {
+        if (options->script_file != NULL || options->inline_script != NULL) {
+            log_error(LOG_COMP_GENERAL, "Error: --protocol cannot be combined with script execution");
+            return false;
+        }
+    }
+
     // Note: Conflict validation for positional .root argument is handled in cli_parse_arguments()
     // when we detect a .root file and system_root is already set.
 
@@ -142,13 +150,14 @@ boolean cli_parse_arguments(int argc, char* argv[], cli_options_t* options) {
         {"debug", no_argument, 0, 'D'},
         {"log", required_argument, 0, 'L'},
         {"skip-startup", no_argument, 0, 'S'},
+        {"protocol", no_argument, 0, 'P'},
         {"help", no_argument, 0, 'h'},
         {"version", no_argument, 0, 'V'},
         {0, 0, 0, 0}
     };
 
     // Parse command line arguments
-    while ((opt = getopt_long(argc, argv, "e:R:m:o:fbHJvDShV", long_options, &option_index)) != -1) {
+    while ((opt = getopt_long(argc, argv, "e:R:m:o:fbHJvDPShV", long_options, &option_index)) != -1) {
         switch (opt) {
             case 'e':
                 // Inline script execution
@@ -260,6 +269,11 @@ boolean cli_parse_arguments(int argc, char* argv[], cli_options_t* options) {
                 options->skip_startup = true;
                 break;
 
+            case 'P':
+                // NDJSON protocol mode (structured JSON over stdin/stdout)
+                options->protocol_mode = true;
+                break;
+
             case 'h':
                 // Help
                 options->show_help = true;
@@ -383,6 +397,7 @@ void cli_print_options(const cli_options_t* options) {
     printf("  Output Path: %s\n", options->output_path ? options->output_path : "(none)");
     printf("  Force Overwrite: %s\n", options->force_overwrite ? "yes" : "no");
     printf("  Skip Startup: %s\n", options->skip_startup ? "yes" : "no");
+    printf("  Protocol Mode: %s\n", options->protocol_mode ? "yes" : "no");
     printf("  Verbose: %s\n", options->verbose ? "yes" : "no");
     printf("  Debug: %s\n", options->debug ? "yes" : "no");
     printf("  Output JSON: %s\n", options->output_json ? "yes" : "no");

frontier-cli/cli_parser.h

@@ -35,6 +35,7 @@ typedef struct {
     boolean hydrate_system_root;// Hydrate system root tables flag
     boolean force_overwrite;    // Force overwrite existing output file (-f/--force)
     boolean skip_startup;       // Skip startup scripts (--skip-startup)
+    boolean protocol_mode;      // NDJSON protocol mode (--protocol)
     boolean show_help;          // Show help flag
     boolean show_version;       // Show version flag
 } cli_options_t;

frontier-cli/main.c

@@ -64,6 +64,7 @@
 #include "cli_executor.h"
 #include "cli_utils.h"
 #include "repl.h"
+#include "protocol_handler.h"
 
 extern long grabthreadglobals(void);
 extern long releasethreadglobals(void);
@@ -378,7 +379,10 @@ int main(int argc, char* argv[]) {
     boolean success = false;
     int exit_code = 0;
 
-    if (g_cli_options.script_file != NULL || g_cli_options.inline_script != NULL) {
+    if (g_cli_options.protocol_mode) {
+        // NDJSON protocol mode - structured JSON over stdin/stdout
+        exit_code = protocol_main(&g_cli_options);
+    } else if (g_cli_options.script_file != NULL || g_cli_options.inline_script != NULL) {
         // Batch mode - execute script and exit
         success = execute_script_mode();
         exit_code = success ? 0 : 1;