V3.6.0 rc

History.md

@@ -1,3 +1,11 @@
+# v3.6.0
+
+* [ADDED] `maxRows` option to limit the number of rows parsed. [#275](https://github.com/C2FO/fast-csv/issues/275) [#277](https://github.com/C2FO/fast-csv/pull/277) - [@cbrittingham](https://github.com/cbrittingham)
+* [ADDED] `skipRows` to allow skipping parsed rows see [parsing.md](./docs/parsing.md)
+* [ADDED] `skipLines` to allow skipping entire lines of a csv [parsing.md](./docs/parsing.md) [#267](https://github.com/C2FO/fast-csv/issues/267)
+* Exported formatting and parsing types.
+* Removed `.npmignore` in favor of `package.json` files
+
 # v3.5.0
 
 * Upgraded dependencies

benchmark/.eslintrc.js

@@ -1,6 +1,8 @@
 module.exports = {
+    parserOptions: {
+        project: null,
+    },
     rules: {
         "no-console": 0,
-        "@typescript-eslint/no-var-requires": 0
     },
 };

benchmark/index.js

@@ -2,7 +2,6 @@ const path = require('path');
 const fs = require('fs');
 const fastCsv = require('..');
 
-
 function camelize(str) {
     return str.replace(/_(.)/g, (a, b) => b.toUpperCase());
 }
@@ -11,7 +10,7 @@ const promisfyStream = (stream, expectedRows) => {
     let count = 0;
     return new Promise((res, rej) => {
         stream
-            .on('data', (row) => {
+            .on('data', row => {
                 count += 1;
             })
             .on('end', () => {
@@ -25,13 +24,14 @@ const promisfyStream = (stream, expectedRows) => {
     });
 };
 
-const benchmarkFastCsv = type => (num) => {
+const benchmarkFastCsv = type => num => {
     const file = path.resolve(__dirname, `./assets/${num}.${type}.csv`);
-    const stream = fs.createReadStream(file)
-        .pipe(fastCsv.parse({ headers: true }))
-        .transform((data) => {
+    const stream = fs
+        .createReadStream(file)
+        .pipe(fastCsv.parse({ headers: true, maxRows: 10 }))
+        .transform(data => {
             const ret = {};
-            [ 'first_name', 'last_name', 'email_address' ].forEach((prop) => {
+            ['first_name', 'last_name', 'email_address'].forEach(prop => {
                 ret[camelize(prop)] = data[prop];
             });
             ret.address = data.address;
@@ -47,15 +47,15 @@ async function benchmarkRun(title, num, m) {
     for (let i = 0; i < howMany; i += 1) {
         // eslint-disable-next-line no-await-in-loop
         await m(num);
-        console.log('%s: RUN(%d lines) 1 %dms', title, num, (new Date() - runStart));
+        console.log('%s: RUN(%d lines) 1 %dms', title, num, new Date() - runStart);
         runStart = new Date();
     }
     console.log('%s: 3xAVG for %d lines %dms', title, num, (new Date() - start) / howMany);
 }
 
 function runBenchmarks(num, type) {
     console.log(`\nRUNNING ${num}.${type}.csv benchmarks`, num);
-    return benchmarkRun('fast-csv', num, benchmarkFastCsv(type))
+    return benchmarkRun('fast-csv', num, benchmarkFastCsv(type));
 }
 
 function benchmarks(type) {
@@ -67,7 +67,7 @@ function benchmarks(type) {
 benchmarks('nonquoted')
     .then(() => benchmarks('quoted'))
     .then(() => process.exit())
-    .catch((e) => {
+    .catch(e => {
         console.error(e.stack);
         return process.exit(1);
     });

docs/parsing.md

@@ -17,6 +17,9 @@
     * [Ignoring Empty Rows](#csv-parse-ignoring-empty-rows)
     * [Transforming Rows](#csv-parse-transforming)
     * [Validating Rows](#csv-parse-validation)
+    * [Max Rows](#max-rows)
+    * [Skip Rows](#skip-rows)
+    * [Skip Lines](#skip-lines)
 
 <a name="parsing-options"></a>
 ## Options
@@ -45,6 +48,9 @@
 * `rtrim: {boolean} = false`: Set to `true` to right trim all fields.
 * `ltrim: {boolean} = false`: Set to `true` to left trim all fields.
 * `encoding: {string} = 'utf8'`: Passed to [StringDecoder](https://nodejs.org/api/string_decoder.html#string_decoder_new_stringdecoder_encoding) when decoding incoming buffers. Change if incoming content is not 'utf8' encoded.
+* `maxRows: {number} = 0`: If number is `> 0` the specified number of rows will be parsed.(e.g. `100` would return the first 100 rows of data).
+* `skipRows: {number} = 0`: If number is `> 0` the specified number of **parsed** rows will be skipped.
+* `skipLines: {number} = 0`: If number is `> 0` the specified number of lines will be skipped.
 
 <a name="parsing-events"></a>
 ## Events
@@ -585,3 +591,123 @@ Valid [row={"firstName":"timmy","lastName":"yukon"}]
 Parsed 2 rows
 ```
 
+<a name="max-rows"></a>
+[`examples/parsing/max_rows.example.example.js`](../examples/parsing/max_rows.example.js)
+
+In the following example there are 10 rows, but only 5 will be parsed because of the `maxRows` option.
+
+```javascript
+const rows = [
+    'header1,header2\n',
+    'col1,col1\n',
+    'col2,col2\n',
+    'col3,col3\n',
+    'col4,col4\n',
+    'col5,col5\n',
+    'col6,col6\n',
+    'col7,col7\n',
+    'col8,col8\n',
+    'col9,col9\n',
+    'col10,col10',
+];
+
+const stream = csv
+    .parse({ headers: true, maxRows: 5 })
+    .on('error', error => console.error(error))
+    .on('data', row => console.log(row))
+    .on('end', rowCount => console.log(`Parsed ${rowCount} rows`));
+
+rows.forEach(row => stream.write(row));
+stream.end();
+```
+
+Expected output
+
+```
+{ header1: 'col1', header2: 'col1' }
+{ header1: 'col2', header2: 'col2' }
+{ header1: 'col3', header2: 'col3' }
+{ header1: 'col4', header2: 'col4' }
+{ header1: 'col5', header2: 'col5' }
+Parsed 5 rows
+```
+
+<a name="skip-rows"></a>
+[`examples/parsing/skip_rows.example.example.js`](../examples/parsing/skip_rows.example.js)
+
+In the following example the first 2 rows are skipped.
+
+**NOTE** Notice how the header row is not skipped, only the row.
+
+```javascript
+const rows = [
+    'header1,header2\n',
+    'col1,col1\n',
+    'col2,col2\n',
+    'col3,col3\n',
+    'col4,col4\n',
+    'col5,col5\n',
+    'col6,col6\n',
+];
+
+const stream = csv
+    .parse({ headers: true, skipRows: 2 })
+    .on('error', error => console.error(error))
+    .on('data', row => console.log(row))
+    .on('end', rowCount => console.log(`Parsed ${rowCount} rows`));
+
+rows.forEach(row => stream.write(row));
+stream.end();
+```
+
+Expected output
+
+```
+{ header1: 'col3', header2: 'col3' }
+{ header1: 'col4', header2: 'col4' }
+{ header1: 'col5', header2: 'col5' }
+{ header1: 'col6', header2: 'col6' }
+Parsed 4 rows
+```
+
+<a name="skip-lines"></a>
+[`examples/parsing/skip_lines.example.example.js`](../examples/parsing/skip_lines.example.js)
+
+In the following example the first 2 lines are skipped.
+
+**NOTE** Notice how the headers come from the third line because the first two are skipped.
+
+```javascript
+const csv = require('../../');
+
+const rows = [
+    'skip1_header1,skip1_header2\n',
+    'skip2_header1,skip2_header2\n',
+    'header1,header2\n',
+    'col1,col1\n',
+    'col2,col2\n',
+    'col3,col3\n',
+    'col4,col4\n',
+];
+
+const stream = csv
+    .parse({ headers: true, skipLines: 2 })
+    .on('error', error => console.error(error))
+    .on('data', row => console.log(row))
+    .on('end', rowCount => console.log(`Parsed ${rowCount} rows`));
+
+rows.forEach(row => stream.write(row));
+stream.end();
+```
+
+Expected output
+
+```
+{ header1: 'col1', header2: 'col1' }
+{ header1: 'col2', header2: 'col2' }
+{ header1: 'col3', header2: 'col3' }
+{ header1: 'col4', header2: 'col4' }
+Parsed 4 rows
+```
+
+

examples/parsing/max_rows.example.js

@@ -0,0 +1,24 @@
+const csv = require('../../');
+
+const rows = [
+    'header1,header2\n',
+    'col1,col1\n',
+    'col2,col2\n',
+    'col3,col3\n',
+    'col4,col4\n',
+    'col5,col5\n',
+    'col6,col6\n',
+    'col7,col7\n',
+    'col8,col8\n',
+    'col9,col9\n',
+    'col10,col10',
+];
+
+const stream = csv
+    .parse({ headers: true, maxRows: 5 })
+    .on('error', error => console.error(error))
+    .on('data', row => console.log(row))
+    .on('end', rowCount => console.log(`Parsed ${rowCount} rows`));
+
+rows.forEach(row => stream.write(row));
+stream.end();

examples/parsing/skip_lines.example.js

@@ -0,0 +1,20 @@
+const csv = require('../../');
+
+const rows = [
+    'skip1_header1,skip1_header2\n',
+    'skip2_header1,skip2_header2\n',
+    'header1,header2\n',
+    'col1,col1\n',
+    'col2,col2\n',
+    'col3,col3\n',
+    'col4,col4\n',
+];
+
+const stream = csv
+    .parse({ headers: true, skipLines: 2 })
+    .on('error', error => console.error(error))
+    .on('data', row => console.log(row))
+    .on('end', rowCount => console.log(`Parsed ${rowCount} rows`));
+
+rows.forEach(row => stream.write(row));
+stream.end();

examples/parsing/skip_rows.example.js

@@ -0,0 +1,20 @@
+const csv = require('../../');
+
+const rows = [
+    'header1,header2\n',
+    'col1,col1\n',
+    'col2,col2\n',
+    'col3,col3\n',
+    'col4,col4\n',
+    'col5,col5\n',
+    'col6,col6\n',
+];
+
+const stream = csv
+    .parse({ headers: true, skipRows: 2 })
+    .on('error', error => console.error(error))
+    .on('data', row => console.log(row))
+    .on('end', rowCount => console.log(`Parsed ${rowCount} rows`));
+
+rows.forEach(row => stream.write(row));
+stream.end();

package.json

@@ -1,11 +1,11 @@
 {
   "name": "fast-csv",
-  "version": "3.5.0",
+  "version": "3.6.0",
   "description": "CSV parser and writer",
   "main": "./build/src/index.js",
   "types": "./build/src/index.d.ts",
   "scripts": {
-    "prepublish": "npm run build",
+    "prepare": "npm run build",
     "build": "tsc",
     "mocha": "nyc mocha",
     "test": "npm run lint && npm run mocha",
@@ -14,6 +14,7 @@
     "benchmark": "node ./benchmark",
     "coverage": "nyc report --reporter=text-lcov | coveralls"
   },
+  "files": ["build/src/**"],
   "repository": {
     "type": "git",
     "url": "git@github.com:C2FO/fast-csv.git"

src/formatter/formatter/index.ts

@@ -1,5 +1,2 @@
-import RowFormatter from './RowFormatter';
-
-export default {
-    RowFormatter,
-};
+export { default as RowFormatter } from './RowFormatter';
+export { default as FieldFormatter } from './FieldFormatter';

src/formatter/index.ts

@@ -8,6 +8,7 @@ import CsvFormatterStream from './CsvFormatterStream';
 export { default as CsvFormatterStream } from './CsvFormatterStream';
 export * from './types';
 export * from './FormatterOptions';
+export * from './formatter';
 
 export const format = (options?: FormatterOptionsArgs): CsvFormatterStream =>
     new CsvFormatterStream(new FormatterOptions(options));

src/index.ts

@@ -6,10 +6,41 @@
  */
 
 import { deprecate } from 'util';
-import { parseStream, parseString, parseFile } from './parser';
+import { parseStream, parseString, parseFile, RowValidateCallback } from './parser';
 
-export { format, write, writeToStream, writeToBuffer, writeToString, writeToPath } from './formatter';
-export { parse, parseString, parseStream, parseFile } from './parser';
+export {
+    format,
+    write,
+    writeToStream,
+    writeToBuffer,
+    writeToString,
+    writeToPath,
+    FormatterOptionsArgs,
+    Row as FormatterRow,
+    RowMap as FormatterRowMap,
+    RowArray as FormatterRowArray,
+    RowHashArray as FormatterRowHashArray,
+    RowTransformCallback as FormatterRowTransformCallback,
+    RowTransformFunction as FormatterRowTransformFunction,
+} from './formatter';
+export {
+    parse,
+    parseString,
+    parseStream,
+    parseFile,
+    ParserOptionsArgs,
+    Row as ParserRow,
+    RowMap as ParserRowMap,
+    RowArray as ParserRowArray,
+    RowValidateCallback as ParserRowValidateCallback,
+    SyncRowValidate as ParserSyncRowValidate,
+    AsyncRowValidate as ParserAsyncRowValidate,
+    RowValidate as ParserRowValidate,
+    RowTransformCallback as ParserRowTransformCallback,
+    SyncRowTransform as ParserSyncRowTransform,
+    AsyncRowTransform as ParserAsyncRowTransform,
+    RowTransformFunction as ParserRowTransformFunction,
+} from './parser';
 
 export const fromString = deprecate(parseString, 'csv.fromString has been deprecated in favor of csv.parseString');
 export const fromStream = deprecate(parseStream, 'csv.fromStream has been deprecated in favor of csv.parseStream');