Read from stdin and write to stdout

[andy-portmen]

Is there a plan to support reading from stdin and writing to stdout? For instance in BunJS, I can do streaming:

for await (let chunk of Bun.stdin.stream()) {}

const writer = Bun.stdout.writer();

[richarddavison]

Yes, what you have there is an async iterator reading from stdin stream! These APIs will eventually be implemented in Rust. It's on the Roadmap but the streams API is rather complex so will take a while before we can ship it

[guest271314]

For Bun I would suggest using Bun.file("/dev/stdin").stream() because Bun.stdin.stream() consistently hangs on https://github.com/oven-sh/bun/issues/11553 bytes oven-sh/bun#11553, oven-sh/bun#11712.

This works to write to standard output

import { readFileSync, writeFileSync } from "node:fs";
const output = new Uint8Array([..."test"].map((s) => s.codePointAt()));;
writeFileSync("/proc/self/fd/1", output);

This does not work for reading standard input stream

const content = readFileSync("/proc/self/fd/0");
console.log({ content });

[guest271314]

@richarddavison How is standard input intended to be read at all right now?

Particularly from a non TTY?

[guest271314]

@andy-portmen This is what I came up with for reading standard input stream as a async iterable. I'm skeptical about WHATWG ReadableStream and WritableStream implementation, see #522.

async function* getMessage([command, argv]) {
  const message = await new Promise((resolve, reject) => {
    const res = [];
    const subprocess = spawn(command, argv, { stdio: "pipe" }); 
    subprocess.stdout.on("data", (data) => {
      res.push(...data);
    });
    subprocess.stdout.on("close", (code) => {
      resolve(new Uint8Array(res));
    });
    subprocess.stdout.on("exit", (code) => {
      reject(encodeMessage({ code }));
    });
  }).catch((e) => e);
  const cmd = command.split(/[/-]/).pop();
  if (cmd === "bash") {
    yield message;
  }
  if (cmd === "qjs") {
    const view = new DataView(message.subarray(0, 4).buffer);
    const length = view.getUint32(0, true);
    // sendMessage(encodeMessage({ length }));
    yield message.subarray(4, 4 + length);
  }
}

  while (true) {
    for await (const message of getMessage(qjs)) {
      try {
        //const message = await getMessage(bash);
        sendMessage(message);
      } catch (e) {
        sendMessage(encodeMessage(e.message));
        break;
      }
    }
  }

This is what I do in Bun world to write to standard output https://github.com/guest271314/NativeMessagingHosts/blob/main/nm_host.js#L34-L43

if (runtime.startsWith("Bun")) {
  readable = Bun.file("/dev/stdin").stream();
  writable = new WritableStream({
    async write(value) {
      await Bun.write(Bun.stdout, value);
    },
  }, new CountQueuingStrategy({ highWaterMark: Infinity }));
  ({ exit } = process);
  ({ argv: args } = Bun);
}

https://github.com/guest271314/NativeMessagingHosts/blob/main/nm_host.js#L75-L82

async function sendMessage(message) {
  await new Blob([
    new Uint8Array(new Uint32Array([message.length]).buffer),
    message,
  ])
    .stream()
    .pipeTo(writable, { preventClose: true });
}

[andy-portmen]

@guest271314 Nice, thanks for sharing.