Append CSV file line by line

[florianbepunkt]

I have a streaming data pipeline for processing CSV files. I would like to use archiver to split large CSV files and combine the chunks as one zip archive for user convenience.

Is it possible to add an entry stream line by line? Currently I only methods to add a file, a string or a file stream. But adding a stream or adding a csv file line by line are rather similiar

export const makeCsvStreamWrite = (header: Record<string, string>, rowLimit = 1000) =>
  async function* (input: AsyncIterable<Record<string, string | number>>) {
    let fileCounter = 1;
    let rowCounter = 0;
    let headerWritten = false;

    const archive = archiver("zip");
    const stringifier = stringify();
    const stringifiedInput = Readable.from(input).pipe(stringifier);

    for await (const row of stringifiedInput) {
      rowCounter++;

      if (!headerWritten) {
        const headerRow = stringifySync([header]);
        archive.append(headerRow, { name: `file-${fileCounter}.csv` });
        headerWritten = false;
      }

      // I would like to append the file line by line....
      archive.append(row, { name: `file-${fileCounter}.csv` });

      if (rowCounter >= rowLimit) {
        // Now I would like to start a new file.

        rowCounter = 0;
        fileCounter++;
        headerWritten = false;
      }

    archive.finalize()
    }
  };