diff --git a/CHANGELOG.md b/CHANGELOG.md index ecc58f0..5c0b484 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ -- [csvtk v0.28.1](https://github.com/shenwei356/csvtk/releases/tag/v0.28.1) -[![Github Releases (by Release)](https://img.shields.io/github/downloads/shenwei356/csvtk/v0.28.1/total.svg)](https://github.com/shenwei356/csvtk/releases/tag/v0.28.1) +- [csvtk v0.29.0](https://github.com/shenwei356/csvtk/releases/tag/v0.29.0) +[![Github Releases (by Release)](https://img.shields.io/github/downloads/shenwei356/csvtk/v0.29.0/total.svg)](https://github.com/shenwei356/csvtk/releases/tag/v0.29.0) + - new commands: + - [`fix-quotes`](https://bioinf.shenwei.me/csvtk/usage/#fix-quotes): fix malformed CSV/TSV caused by double-quotes. [#260](https://github.com/shenwei356/csvtk/issues/260) + - [`del-quotes`](https://bioinf.shenwei.me/csvtk/usage/#del-quotes): remove extra double-quotes added by `fix-quotes`. + - `csvtk del-header`: + - fix deleting headers of 2nd and later files. [#257](https://github.com/shenwei356/csvtk/issues/257) + - `csvtk concat`: + - fix panic when no data found. - `csvtk sort`: - support column name containing colons. [#254](https://github.com/shenwei356/csvtk/issues/254) - `csvtk filter2`: @@ -7,10 +14,8 @@ - fix specifying the position for the new column containing only a constant string. [#252](https://github.com/shenwei356/csvtk/issues/252) - `csvtk plot`: - add a new flag `--tick-label-size`. - - `csvtk del-header`: - - fix deleting headers of 2nd and later files. [#257](https://github.com/shenwei356/csvtk/issues/257) - - `csvtk concat`: - - fix panic when no data found. + - `csvtk pretty`: + - replace tabs with spaces. - [csvtk v0.28.0](https://github.com/shenwei356/csvtk/releases/tag/v0.28.0) [![Github Releases (by Release)](https://img.shields.io/github/downloads/shenwei356/csvtk/v0.28.0/total.svg)](https://github.com/shenwei356/csvtk/releases/tag/v0.28.0) - `csvtk`: diff --git a/README.md b/README.md index ac941f7..33ccced 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # csvtk - a cross-platform, efficient and practical CSV/TSV toolkit - **Documents:** [http://bioinf.shenwei.me/csvtk](http://bioinf.shenwei.me/csvtk/) -( [**Usage**](http://bioinf.shenwei.me/csvtk/usage/) and [**Tutorial**](http://bioinf.shenwei.me/csvtk/tutorial/)). [中文介绍](http://bioinf.shenwei.me/csvtk/chinese) +( [**Usage**](http://bioinf.shenwei.me/csvtk/usage/), [**Tutorial**](http://bioinf.shenwei.me/csvtk/tutorial/) and [**FAQs**](http://bioinf.shenwei.me/csvtk/faq/)). +[中文介绍](http://bioinf.shenwei.me/csvtk/chinese) - **Source code:** [https://github.com/shenwei356/csvtk](https://github.com/shenwei356/csvtk) [![GitHub stars](https://img.shields.io/github/stars/shenwei356/csvtk.svg?style=social&label=Star&?maxAge=2592000)](https://github.com/shenwei356/csvtk) [![license](https://img.shields.io/github/license/shenwei356/csvtk.svg?maxAge=2592000)](https://github.com/shenwei356/csvtk/blob/master/LICENSE) - **Latest version:** [![Latest Stable Version](https://img.shields.io/github/release/shenwei356/csvtk.svg?style=flat)](https://github.com/shenwei356/csvtk/releases) @@ -63,7 +64,7 @@ It could save you lots of time in (not) writing Python/R scripts. ## Subcommands -51 subcommands in total. +53 subcommands in total. **Information** @@ -108,6 +109,8 @@ It could save you lots of time in (not) writing Python/R scripts. **Edit** - [`fix`](https://bioinf.shenwei.me/csvtk/usage/#fix): fix CSV/TSV with different numbers of columns in rows +- [`fix-quotes`](https://bioinf.shenwei.me/csvtk/usage/#fix-quotes): fix malformed CSV/TSV caused by double-quotes +- [`del-quotes`](https://bioinf.shenwei.me/csvtk/usage/#del-quotes): remove extra double-quotes added by `fix-quotes` - [`add-header`](https://bioinf.shenwei.me/csvtk/usage/#add-header): add column names - [`del-header`](https://bioinf.shenwei.me/csvtk/usage/#del-header): delete column names - [`rename`](https://bioinf.shenwei.me/csvtk/usage/#rename): renames column names with new names diff --git a/csvtk/cmd/del-quotes.go b/csvtk/cmd/del-quotes.go new file mode 100644 index 0000000..d14b5ca --- /dev/null +++ b/csvtk/cmd/del-quotes.go @@ -0,0 +1,125 @@ +// Copyright © 2016-2023 Wei Shen +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + "fmt" + "runtime" + "strings" + "unicode" + "unicode/utf8" + + "github.com/shenwei356/xopen" + "github.com/spf13/cobra" +) + +// delQuotesCmd represents the csv2tab command +var delQuotesCmd = &cobra.Command{ + Use: "del-quotes", + Short: "remove extra double quotes added by 'fix-quotes'", + Long: `remove extra double quotes added by 'fix-quotes' + +Limitation: + 1. Values containing line breaks are not supported. + +`, + Run: func(cmd *cobra.Command, args []string) { + config := getConfigs(cmd) + files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true) + if len(files) > 1 { + checkError(fmt.Errorf("no more than one file should be given")) + } + runtime.GOMAXPROCS(config.NumCPUs) + + outfh, err := xopen.Wopen(config.OutFile) + checkError(err) + defer outfh.Close() + + if config.Tabs { + config.Delimiter = '\t' + } + + file := files[0] + csvReader, err := newCSVReaderByConfig(config, file) + if err != nil { + if err == xopen.ErrNoContent { + log.Warningf("csvtk csv2tab: skipping empty input file: %s", file) + return + } + checkError(err) + } + + csvReader.Read(ReadOption{ + FieldStr: "1-", + ShowRowNumber: config.ShowRowNumber, + }) + + d := string(config.Delimiter) + var i int + var v string + for record := range csvReader.Ch { + if record.Err != nil { + checkError(record.Err) + } + for i, v = range record.Selected { + // if fieldNeedsQuotes(v, config.Delimiter) { + if strings.Contains(v, d) { + record.Selected[i] = `"` + v + `"` + } + } + outfh.WriteString(strings.Join(record.Selected, d)) + outfh.WriteByte('\n') + } + + readerReport(&config, csvReader, file) + }, +} + +func init() { + RootCmd.AddCommand(delQuotesCmd) +} + +// copy from https://cs.opensource.google/go/go/+/refs/tags/go1.21.4:src/encoding/csv/writer.go;l=157 +func fieldNeedsQuotes(field string, comma rune) bool { + if field == "" { + return false + } + + if field == `\.` { + return true + } + + if comma < utf8.RuneSelf { + for i := 0; i < len(field); i++ { + c := field[i] + if c == '\n' || c == '\r' || c == '"' || c == byte(comma) { + return true + } + } + } else { + if strings.ContainsRune(field, comma) || strings.ContainsAny(field, "\"\r\n") { + return true + } + } + + r1, _ := utf8.DecodeRuneInString(field) + return unicode.IsSpace(r1) +} diff --git a/csvtk/cmd/fix-quotes.go b/csvtk/cmd/fix-quotes.go new file mode 100644 index 0000000..2a5c532 --- /dev/null +++ b/csvtk/cmd/fix-quotes.go @@ -0,0 +1,229 @@ +// Copyright © 2016-2023 Wei Shen +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + "bufio" + "bytes" + "fmt" + "regexp" + "runtime" + + "github.com/shenwei356/xopen" + "github.com/spf13/cobra" +) + +// fixquotesCmd represents the pretty command +var fixquotesCmd = &cobra.Command{ + Use: "fix-quotes", + Short: "fix malformed CSV/TSV caused by double-quotes", + Long: `fix malformed CSV/TSV caused by double-quotes + +This command fixes fields not appropriately enclosed by double-quotes +to meet the RFC4180 specification (https://rfc-editor.org/rfc/rfc4180.html). + +When and how to: + 1. Values containing bare double quotes. e.g., + a,abc" xyz,d + Error information: bare " in non-quoted-field. + Fix: adding the flag -l/--lazy-quotes. + Using this command: + a,abc" xyz,d -> a,"abc"" xyz",d + 2. Values with double quotes in the begining but not in the end. e.g., + a,"abc" xyz,d + Error information: extraneous or missing " in quoted-field. + Using this command: + a,"abc" xyz,d -> a,"""abc"" xyz",d + +Next: + 1. You can process the data without the flag -l/--lazy-quotes. + 2. Use 'csvtk del-quotes' if you want to restore the original format. + +Limitation: + 1. Values containing line breaks are not supported. + +`, + Run: func(cmd *cobra.Command, args []string) { + config := getConfigs(cmd) + files := getFileListFromArgsAndFile(cmd, args, true, "infile-list", true) + if len(files) > 1 { + checkError(fmt.Errorf("no more than one file should be given")) + } + runtime.GOMAXPROCS(config.NumCPUs) + + if config.Tabs { + config.Delimiter = '\t' + } + + outfh, err := xopen.Wopen(config.OutFile) + checkError(err) + defer outfh.Close() + + fh, err := xopen.Ropen(files[0]) + checkError(err) + defer func() { + checkError(fh.Close()) + }() + + var buf bytes.Buffer + + scanner := bufio.NewScanner(fh) + var line string + var i, s int + var r, p rune + var firstField, firstChar bool + var hasLeftQuotes, hasRightQuotes bool + var nInnerQuotes int // number of inner quotes, might including the right quotes + d := config.Delimiter + re := regexp.MustCompile(`"`) + var field string + var n, ncols int + ncols = -1 + var iLine int + var reQuotedDelimiter = regexp.MustCompile(fmt.Sprintf(`(^|%c)".*%c.*"($|%c)`, d, d, d)) + var hasQuotedDelimiter bool + for scanner.Scan() { + iLine++ + line = scanner.Text() + hasQuotedDelimiter = reQuotedDelimiter.MatchString(line) + + if len(line) == 0 || line[0] == byte(config.CommentChar) { + outfh.WriteString(line) + outfh.WriteByte('\n') + + continue + } + + n = 0 + firstField = true + + firstChar = true + nInnerQuotes, hasLeftQuotes, hasRightQuotes = 0, false, false + buf.Reset() + + s = 0 + + for i, r = range line { + if r == d { + if p == '"' { + hasRightQuotes = true + nInnerQuotes-- + } + + // might be a comma within a field + if hasLeftQuotes && !hasRightQuotes && hasQuotedDelimiter { + continue + } + + if firstField { + field = line[s:i] + } else { + field = line[s+1 : i] + } + + if nInnerQuotes > 0 || + (hasLeftQuotes && !hasRightQuotes) || + (!hasLeftQuotes && hasRightQuotes) { + field = re.ReplaceAllString(field, `""`) + field = `"` + field + `"` + } + + if !firstField { + buf.WriteRune(d) + } + buf.WriteString(field) + + s = i + + firstField = false + n++ + + firstChar = true + nInnerQuotes, hasLeftQuotes, hasRightQuotes = 0, false, false + + continue + } + + if firstChar { + if r == '"' { + hasLeftQuotes = true + } + firstChar = false + } else if r == '"' { + nInnerQuotes++ + } + p = r + + } + + i = len(line) + // the last record + + if p == '"' { + hasRightQuotes = true + nInnerQuotes-- + } + + if firstField { + field = line[s:i] + } else { + field = line[s+1 : i] + } + + if nInnerQuotes > 0 || + (hasLeftQuotes && !hasRightQuotes) || + (!hasLeftQuotes && hasRightQuotes) { + field = re.ReplaceAllString(field, `""`) + field = `"` + field + `"` + } + + if !firstField { + buf.WriteRune(d) + } + buf.WriteString(field) + + // the last record + + n++ + + buf.WriteByte('\n') + + outfh.Write(buf.Bytes()) + + // check ncols + if ncols < 0 { + ncols = n + } else if n != ncols { + checkError(fmt.Errorf("failed to fix (unequal number of fields: %d (line %d) != %d (line %d), does exist quoted delimiter?): %s", + n, iLine, ncols, iLine-1, line)) + + } + + } + if err := scanner.Err(); err != nil { + checkError(err) + } + }, +} + +func init() { + RootCmd.AddCommand(fixquotesCmd) +} diff --git a/csvtk/cmd/version.go b/csvtk/cmd/version.go index c0c7185..69b66d2 100644 --- a/csvtk/cmd/version.go +++ b/csvtk/cmd/version.go @@ -29,7 +29,7 @@ import ( ) // VERSION of csvtk -const VERSION = "0.28.1" +const VERSION = "0.29.0" // versionCmd represents the version command var versionCmd = &cobra.Command{ diff --git a/doc/docs/faq.md b/doc/docs/faq.md new file mode 100644 index 0000000..711252c --- /dev/null +++ b/doc/docs/faq.md @@ -0,0 +1,78 @@ +# Frequently Asked Questions + +## The specification of CSV format + +The CSV parser used by csvtk follows the [RFC4180](https://rfc-editor.org/rfc/rfc4180.html) specification. + +## bare " in non-quoted-field + +``` + 5. Each field may or may not be enclosed in double quotes (however + some programs, such as Microsoft Excel, do not use double quotes + at all). If fields are not enclosed with double quotes, then + double quotes may not appear inside the fields. For example: + + "aaa","bbb","ccc" CRLF + zzz,yyy,xxx + + 6. Fields containing line breaks (CRLF), double quotes, and commas + should be enclosed in double-quotes. For example: + + "aaa","b CRLF + bb","ccc" CRLF + zzz,yyy,xxx + + 7. If double-quotes are used to enclose fields, then a double-quote + appearing inside a field must be escaped by preceding it with + another double quote. For example: + + "aaa","b""bb","ccc" +``` + +If a single double-quote exists in one non-quoted-field, an error will be reported. e.g, + + $ echo 'a,abc" xyz,d' + a,abc" xyz,d + + $ echo 'a,abc" xyz,d' | csvtk cut -f 1- + [ERRO] parse error on line 1, column 6: bare " in non-quoted-field + +You can add the flag `-l/--lazy-quotes` to fix this. + + $ echo 'a,abc" xyz,d' | csvtk cut -f 1- -l + a,"abc"" xyz",d + +## extraneous or missing " in quoted-field + +But for the situation below, `-l/--lazy-quotes` won't help: + + $ echo 'a,"abc" xyz,d' + a,"abc" xyz,d + + $ echo 'a,"abc" xyz,d' | csvtk cut -f 1- + [ERRO] parse error on line 1, column 7: extraneous or missing " in quoted-field + + $ echo 'a,"abc" xyz,d' | csvtk cut -f 1- -l + a,"abc"" xyz,d + " + + $ echo 'a,"abc" xyz,d' | csvtk cut -f 1- -l | csvtk dim + file num_cols num_rows + - 2 0 + +**You need to use [csvtk fix-quotes](https://bioinf.shenwei.me/csvtk/usage/#fix-quotes) (available in v0.29.0 or later versions)**: + + $ echo 'a,"abc" xyz,d' | csvtk fix-quotes + a,"""abc"" xyz",d + + $ echo 'a,"abc" xyz,d' | csvtk fix-quotes | csvtk cut -f 1- + a,"""abc"" xyz",d + + $ echo 'a,"abc" xyz,d' | csvtk fix-quotes | csvtk cut -f 1- | csvtk dim + file num_cols num_rows + - 3 0 + +Use [del-quotes](https://bioinf.shenwei.me/csvtk/usage/#del-quotes) if you need the original format after some operations. + + $ echo 'a,"abc" xyz,d' | csvtk fix-quotes | csvtk cut -f 1- | csvtk del-quotes + a,"abc" xyz,d diff --git a/doc/docs/usage.md b/doc/docs/usage.md index 1896915..a6add00 100644 --- a/doc/docs/usage.md +++ b/doc/docs/usage.md @@ -2703,6 +2703,139 @@ $ cat testdata/unequal_ncols.csv | csvtk fix | csvtk pretty -S grid ``` +## fix-quotes + +Usage + +```text +fix malformed CSV/TSV caused by double-quotes + +This command fixes fields not appropriately enclosed by double-quotes +to meet the RFC4180 standard (https://rfc-editor.org/rfc/rfc4180.html). + +When and how to: + 1. Values containing bare double quotes. e.g., + a,abc" xyz,d + Error information: bare " in non-quoted-field. + Fix: adding the flag -l/--lazy-quotes. + Using this command: + a,abc" xyz,d -> a,"abc"" xyz",d + 2. Values with double quotes in the begining but not in the end. e.g., + a,"abc" xyz,d + Error information: extraneous or missing " in quoted-field. + Using this command: + a,"abc" xyz,d -> a,"""abc"" xyz",d + +Next: + 1. You can process the data without the flag -l/--lazy-quotes. + 2. Use 'csvtk del-quotes' if you want to restore the original format. + +Limitation: + 1. Values containing line breaks are not supported. + +Usage: + csvtk fix-quotes [flags] + +Flags: + -h, --help help for fix-quotes +``` + +Examples: + +1. Test data, in which there are five cases with values containing double quotes. + + $ cat testdata/malformed.tsv + 1 Cellvibrio no quotes & not tab + 2 "Cellvibrio gilvus" quotes can be removed + 3 "quotes required" quotes needed (with a tab in the cell) + 4 fake" record bare double-quote in non-quoted-field + 5 "Cellvibrio" Winogradsky only with doub-quote in the beginning + 6 fake record2" "only with doub-quote in the end" + + $ cat testdata/malformed.tsv | csvtk cut -f 1- + [ERRO] parse error on line 2, column 3: bare " in non-quoted-field + + # -l does not work, and it's messed up. + $ cat testdata/malformed.tsv | csvtk cut -f 1- -l + 1 Cellvibrio no quotes & not tab + "2 ""Cellvibrio gilvus"" quotes can be removed" + "3 ""quotes required"" quotes needed (with a tab in the cell)" + "4 fake"" record bare double-quote in non-quoted-field" + "5 ""Cellvibrio"" Winogradsky only with doub-quote in the beginning" + "6 fake record2"" ""only with doub-quote in the end""" + +1. Fix it!!! + + $ cat testdata/malformed.tsv | csvtk fix-quotes -t + 1 Cellvibrio no quotes & not tab + 2 "Cellvibrio gilvus" quotes can be removed + 3 "quotes required" quotes needed (with a tab in the cell) + 4 "fake"" record" bare double-quote in non-quoted-field + 5 """Cellvibrio"" Winogradsky" only with doub-quote in the beginning + 6 "fake record2""" "only with doub-quote in the end" + + # pretty + $ cat testdata/malformed.tsv | csvtk fix-quotes -t | csvtk pretty -Ht -S grid + +---+--------------------------+----------------------------------------+ + | 1 | Cellvibrio | no quotes & not tab | + +---+--------------------------+----------------------------------------+ + | 2 | Cellvibrio gilvus | quotes can be removed | + +---+--------------------------+----------------------------------------+ + | 3 | quotes required | quotes needed (with a tab in the cell) | + +---+--------------------------+----------------------------------------+ + | 4 | fake" record | bare double-quote in non-quoted-field | + +---+--------------------------+----------------------------------------+ + | 5 | "Cellvibrio" Winogradsky | only with doub-quote in the beginning | + +---+--------------------------+----------------------------------------+ + | 6 | fake record2" | only with doub-quote in the end | + +---+--------------------------+----------------------------------------+ + + # do something, like searching rows containing double-quotes. + # since the command-line argument parser csvtk uses parse the value of flag -p + # as CSV data, we have to use -p '""""' to represents one double-quotes, + # where the outter two double quotes are used to quote the value, + # and the two inner double-quotes actually means an escaped double-quote + # + $ cat testdata/malformed.tsv \ + | csvtk fix-quotes -t \ + | csvtk grep -Ht -f 2 -r -p '""""' + 4 "fake"" record" bare double-quote in non-quoted-field + 5 """Cellvibrio"" Winogradsky" only with doub-quote in the beginning + 6 "fake record2""" only with doub-quote in the end + +1. Note that fixed rows are different from the orginal ones, you can use `csvtk del-quotes` to reset them. + + $ cat testdata/malformed.tsv \ + | csvtk fix-quotes -t \ + | csvtk filter2 -t -f '$1 > 0' \ + | csvtk del-quotes -t + 1 Cellvibrio no quotes & not tab + 2 Cellvibrio gilvus quotes can be removed + 3 "quotes required" quotes needed (with a tab in the cell) + 4 fake" record bare double-quote in non-quoted-field + 5 "Cellvibrio" Winogradsky only with doub-quote in the beginning + 6 fake record2" only with doub-quote in the end + + +## del-quotes + +Usage + +```text +remove extra double quotes added by 'fix-quotes' + +Limitation: + 1. Values containing line breaks are not supported. + +Usage: + csvtk del-quotes [flags] + +Flags: + -h, --help help for del-quotes +``` + +Examples: see eamples of [fix-quotes](#fix-quotes) + ## add-header Usage diff --git a/doc/mkdocs.yml b/doc/mkdocs.yml index 8b3c856..e2561d1 100644 --- a/doc/mkdocs.yml +++ b/doc/mkdocs.yml @@ -3,6 +3,7 @@ nav: - Home: index.md - Download: download.md - Usage: usage.md +- FAQs: faq.md - Tutorial: tutorial.md - 中文介绍: chinese.md - More tools: https://github.com/shenwei356 diff --git a/go.mod b/go.mod index e9a849f..08855c4 100644 --- a/go.mod +++ b/go.mod @@ -16,9 +16,9 @@ require ( github.com/shenwei356/breader v0.3.2 github.com/shenwei356/go-logging v0.0.0-20171012171522-c6b9702d88ba github.com/shenwei356/natsort v0.0.0-20220117010048-580176ad49fb - github.com/shenwei356/stable v0.1.4 + github.com/shenwei356/stable v0.1.5 github.com/shenwei356/util v0.5.2 - github.com/shenwei356/xopen v0.2.2 + github.com/shenwei356/xopen v0.3.1 github.com/spf13/cobra v1.7.0 github.com/tatsushid/go-prettytable v0.0.0-20141013043238-ed2d14c29939 github.com/twotwotwo/sorts v0.0.0-20160814051341-bf5c1f2b8553 diff --git a/go.sum b/go.sum index 2657e38..d224414 100644 --- a/go.sum +++ b/go.sum @@ -132,8 +132,8 @@ github.com/shenwei356/go-logging v0.0.0-20171012171522-c6b9702d88ba/go.mod h1:Li github.com/shenwei356/natsort v0.0.0-20190418160752-600d539c017d/go.mod h1:SiiGiRFyRtV7S9RamOrmQR5gpGIRhWJM1w0EtmuQ1io= github.com/shenwei356/natsort v0.0.0-20220117010048-580176ad49fb h1:pb0RhpaADsFrKNLST9oogHPlZJec7vT4Gvny5FFhaxU= github.com/shenwei356/natsort v0.0.0-20220117010048-580176ad49fb/go.mod h1:SiiGiRFyRtV7S9RamOrmQR5gpGIRhWJM1w0EtmuQ1io= -github.com/shenwei356/stable v0.1.4 h1:cQL/I2pBpLEH8UWNo0SE3/q/e3XwzFN2NUnpECzwDoU= -github.com/shenwei356/stable v0.1.4/go.mod h1:KghgqlviHPiKn9AuSTpadb7ep74n42VsNtPLoZZ/JIc= +github.com/shenwei356/stable v0.1.5 h1:d6VivPq2YOzQkTve6tW9FxHeiTrZtqV4rcGBC39aCIM= +github.com/shenwei356/stable v0.1.5/go.mod h1:KghgqlviHPiKn9AuSTpadb7ep74n42VsNtPLoZZ/JIc= github.com/shenwei356/util v0.5.2 h1:kU9bnkE3RRUAlya+hbfwy83iTMOJqIHOlYgejYPb7mU= github.com/shenwei356/util v0.5.2/go.mod h1:3tRAOfreWdgl/Zh1gE008h2lWocf5/YAxVSjgLKvd4k= github.com/shenwei356/xopen v0.2.2 h1:g1v3YjiIky9k6oN4qmnU1bDciAHnSrmOn2sMTE5pChY= diff --git a/testdata/malformed.tsv b/testdata/malformed.tsv new file mode 100644 index 0000000..3409cf4 --- /dev/null +++ b/testdata/malformed.tsv @@ -0,0 +1,6 @@ +1 Cellvibrio no quotes & not tab +2 "Cellvibrio gilvus" quotes can be removed +3 "quotes required" quotes needed (with a tab in the cell) +4 fake" record bare double-quote in non-quoted-field +5 "Cellvibrio" Winogradsky only with doub-quote in the beginning +6 fake record2" "only with doub-quote in the end"