Added yaml import

cmd/cmd.go

@@ -122,6 +122,7 @@ func (cli Cli) Run(args []string) int {
 	flags.BoolVar(&inFlag.CSV, "icsv", false, "CSV format for input.")
 	flags.BoolVar(&inFlag.LTSV, "iltsv", false, "LTSV format for input.")
 	flags.BoolVar(&inFlag.JSON, "ijson", false, "JSON format for input.")
+	flags.BoolVar(&inFlag.YAML, "iyaml", false, "YAML format for input.")
 	flags.BoolVar(&inFlag.TBLN, "itbln", false, "TBLN format for input.")
 	flags.BoolVar(&inFlag.WIDTH, "iwidth", false, "width specification format for input.")
 
@@ -478,6 +479,7 @@ type inputFlag struct {
 	CSV   bool
 	LTSV  bool
 	JSON  bool
+	YAML  bool
 	TBLN  bool
 	WIDTH bool
 }
@@ -491,6 +493,8 @@ func inputFormat(i inputFlag) trdsql.Format {
 		return trdsql.LTSV
 	case i.JSON:
 		return trdsql.JSON
+	case i.YAML:
+		return trdsql.YAML
 	case i.TBLN:
 		return trdsql.TBLN
 	case i.WIDTH:
@@ -502,7 +506,7 @@ func inputFormat(i inputFlag) trdsql.Format {
 
 func isInFormat(name string) bool {
 	switch name {
-	case "ig", "icsv", "iltsv", "ijson", "itbln", "iwidth":
+	case "ig", "icsv", "iltsv", "ijson", "iyaml", "itbln", "iwidth":
 		return true
 	}
 	return false

importer.go

@@ -37,6 +37,8 @@ var (
 	ErrNoMatchFound = errors.New("no match found")
 	// ErrNonDefinition is returned when there is no definition.
 	ErrNonDefinition = errors.New("no definition")
+	// ErrInvalidYAML is returned when the YAML is invalid.
+	ErrInvalidYAML = errors.New("invalid YAML")
 )
 
 // Importer is the interface import data into the database.
@@ -276,11 +278,6 @@ func GuessOpts(readOpts *ReadOpts, fileName string) (*ReadOpts, string) {
 		}
 	}
 
-	// If the option -ijq is specified, it is assumed to be json(only for guess).
-	if readOpts.InJQuery != "" && readOpts.InFormat == GUESS {
-		readOpts.InFormat = JSON
-	}
-
 	if readOpts.InFormat != GUESS {
 		readOpts.realFormat = readOpts.InFormat
 		return readOpts, fileName
@@ -315,6 +312,8 @@ func guessFormat(fileName string) Format {
 			return LTSV
 		case "JSON", "JSONL":
 			return JSON
+		case "YAML", "YML":
+			return YAML
 		case "TBLN":
 			return TBLN
 		case "WIDTH":

input_yaml.go

@@ -0,0 +1,254 @@
+package trdsql
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"log"
+
+	"github.com/goccy/go-yaml"
+	"github.com/itchyny/gojq"
+)
+
+// YAMLReader provides methods of the Reader interface.
+type YAMLReader struct {
+	reader    *yaml.Decoder
+	query     *gojq.Query
+	already   map[string]bool
+	inNULL    string
+	preRead   []map[string]interface{}
+	names     []string
+	types     []string
+	limitRead bool
+	needNULL  bool
+}
+
+// NewYAMLReader returns YAMLReader and error.
+func NewYAMLReader(reader io.Reader, opts *ReadOpts) (*YAMLReader, error) {
+	r := &YAMLReader{}
+	r.reader = yaml.NewDecoder(reader)
+	r.already = make(map[string]bool)
+	var top interface{}
+
+	if opts.InJQuery != "" {
+		str := trimQuoteAll(opts.InJQuery)
+		query, err := gojq.Parse(str)
+		if err != nil {
+			return nil, fmt.Errorf("%w gojq:(%s)", err, opts.InJQuery)
+		}
+		r.query = query
+	}
+	r.limitRead = opts.InLimitRead
+	r.needNULL = opts.InNeedNULL
+	r.inNULL = opts.InNULL
+
+	for i := 0; i < opts.InPreRead; i++ {
+		if err := r.reader.Decode(&top); err != nil {
+			if !errors.Is(err, io.EOF) {
+				return r, err
+			}
+			debug.Printf(err.Error())
+			return r, nil
+		}
+
+		if r.query != nil {
+			if err := r.jquery(top); err != nil {
+				return nil, err
+			}
+			return r, nil
+		}
+
+		if err := r.readAhead(top); err != nil {
+			return nil, err
+		}
+	}
+
+	return r, nil
+}
+
+func (r *YAMLReader) jquery(top interface{}) error {
+	iter := r.query.Run(top)
+	for {
+		v, ok := iter.Next()
+		if !ok {
+			break
+		}
+		if err, ok := v.(error); ok {
+			return fmt.Errorf("%w gojq:(%s) ", err, r.query)
+		}
+
+		if err := r.readAhead(v); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Names returns column names.
+func (r *YAMLReader) Names() ([]string, error) {
+	return r.names, nil
+}
+
+// Types returns column types.
+// All YAML types return the DefaultDBType.
+func (r *YAMLReader) Types() ([]string, error) {
+	r.types = make([]string, len(r.names))
+	for i := 0; i < len(r.names); i++ {
+		r.types[i] = DefaultDBType
+	}
+	return r.types, nil
+}
+
+func (r *YAMLReader) readAhead(top interface{}) error {
+	switch m := top.(type) {
+	case []interface{}:
+		for _, v := range m {
+			pre, names, err := r.topLevel(v)
+			if err != nil {
+				return err
+			}
+			r.appendNames(names)
+			r.preRead = append(r.preRead, pre)
+		}
+	case map[string]interface{}:
+		pre, names, err := r.topLevel(m)
+		if err != nil {
+			return err
+		}
+		r.appendNames(names)
+		r.preRead = append(r.preRead, pre)
+	case yaml.MapSlice:
+		pre, names, err := r.objectMapRow(m)
+		if err != nil {
+			return err
+		}
+		r.appendNames(names)
+		r.preRead = append(r.preRead, pre)
+	default:
+		return ErrInvalidYAML
+	}
+	return nil
+}
+
+// appendNames adds multiple names for the argument to be unique.
+func (r *YAMLReader) appendNames(names []string) {
+	for _, name := range names {
+		if !r.already[name] {
+			r.already[name] = true
+			r.names = append(r.names, name)
+		}
+	}
+}
+
+func (r *YAMLReader) topLevel(top interface{}) (map[string]interface{}, []string, error) {
+	switch obj := top.(type) {
+	case map[string]interface{}:
+		return r.objectRow(obj)
+	case yaml.MapSlice:
+		return r.objectMapRow(obj)
+	default:
+		return r.etcRow(obj)
+	}
+}
+
+// PreReadRow is returns only columns that store preRead rows.
+// One YAML (not YAMLl) returns all rows with preRead.
+func (r *YAMLReader) PreReadRow() [][]interface{} {
+	rows := make([][]interface{}, len(r.preRead))
+	for n, v := range r.preRead {
+		rows[n] = make([]interface{}, len(r.names))
+		for i := range r.names {
+			rows[n][i] = v[r.names[i]]
+		}
+
+	}
+	return rows
+}
+
+// ReadRow is read the rest of the row.
+// Only YAMLl requires ReadRow in YAML.
+func (r *YAMLReader) ReadRow(row []interface{}) ([]interface{}, error) {
+	if r.limitRead {
+		return nil, io.EOF
+	}
+
+	var data interface{}
+	if err := r.reader.Decode(&data); err != nil {
+		return nil, err
+	}
+	v := r.rowParse(row, data)
+	return v, nil
+}
+
+func (r *YAMLReader) rowParse(row []interface{}, YAMLRow interface{}) []interface{} {
+	switch m := YAMLRow.(type) {
+	case map[string]interface{}:
+		for i := range r.names {
+			row[i] = r.YAMLString(m[r.names[i]])
+		}
+	default:
+		for i := range r.names {
+			row[i] = nil
+		}
+		row[0] = r.YAMLString(YAMLRow)
+	}
+	return row
+}
+
+func (r *YAMLReader) objectMapRow(obj yaml.MapSlice) (map[string]interface{}, []string, error) {
+	names := make([]string, 0, len(obj))
+	row := make(map[string]interface{})
+	for _, item := range obj {
+		key := item.Key.(string)
+		names = append(names, key)
+		if item.Value == nil {
+			row[key] = nil
+		} else {
+			row[key] = r.YAMLString(item.Value)
+		}
+	}
+	return row, names, nil
+}
+
+func (r *YAMLReader) objectRow(obj map[string]interface{}) (map[string]interface{}, []string, error) {
+	names := make([]string, 0, len(obj))
+	row := make(map[string]interface{})
+	for k, v := range obj {
+		names = append(names, k)
+		if v == nil {
+			row[k] = nil
+		} else {
+			row[k] = r.YAMLString(v)
+		}
+	}
+	return row, names, nil
+}
+
+func (r *YAMLReader) etcRow(val interface{}) (map[string]interface{}, []string, error) {
+	var names []string
+	k := "c1"
+	names = append(names, k)
+	row := make(map[string]interface{})
+	row[k] = r.YAMLString(val)
+	return row, names, nil
+}
+
+func (r *YAMLReader) YAMLString(val interface{}) interface{} {
+	var str string
+	switch val.(type) {
+	case nil:
+		return nil
+	case map[string]interface{}, []interface{}:
+		b, err := yaml.Marshal(val)
+		if err != nil {
+			log.Printf("ERROR: YAMLString:%s", err)
+		}
+		str = ValString(b)
+	default:
+		str = ValString(val)
+	}
+	if r.needNULL {
+		return replaceNULL(r.inNULL, str)
+	}
+	return str
+}

output_yaml.go

@@ -1,7 +1,7 @@
 package trdsql
 
 import (
-	yaml "github.com/goccy/go-yaml"
+	"github.com/goccy/go-yaml"
 )
 
 // YAMLWriter provides methods of the Writer interface.

reader.go

@@ -170,6 +170,8 @@ func NewReader(reader io.Reader, readOpts *ReadOpts) (Reader, error) {
 		return NewLTSVReader(reader, readOpts)
 	case JSON:
 		return NewJSONReader(reader, readOpts)
+	case YAML:
+		return NewYAMLReader(reader, readOpts)
 	case TBLN:
 		return NewTBLNReader(reader, readOpts)
 	case WIDTH:

trdsql.go

@@ -87,7 +87,7 @@ const (
 	// JSON Lines format(http://jsonlines.org/).
 	JSONL
 
-	// export
+	// import/export
 	// YAML format.
 	YAML