regen test data

cloudquery · May 9, 2024 · 51c9c6f · 51c9c6f
1 parent 7fe7b10
commit 51c9c6f
Show file tree

Hide file tree

Showing 6 changed files with 25 additions and 21 deletions.
diff --git a/csv/read.go b/csv/read.go
@@ -38,7 +38,7 @@ func castFromString(rec arrow.Record, arrowSchema *arrow.Schema) (arrow.Record,
 	cols := make([]arrow.Array, rec.NumCols())
 	for c, f := range arrowSchema.Fields() {
 		col := rec.Column(c)
-		if isTypeSupported(f.Type) {
+		if typeSupported(f.Type) {
 			cols[c] = col
 			continue
 		}

diff --git a/csv/testdata/TestWriteRead-default.csv b/csv/testdata/TestWriteRead-default.csv
diff --git a/csv/testdata/TestWriteRead-with_delimiter.csv b/csv/testdata/TestWriteRead-with_delimiter.csv
diff --git a/csv/testdata/TestWriteRead-with_delimiter_headers.csv b/csv/testdata/TestWriteRead-with_delimiter_headers.csv
diff --git a/csv/testdata/TestWriteRead-with_headers.csv b/csv/testdata/TestWriteRead-with_headers.csv
diff --git a/csv/write.go b/csv/write.go
@@ -57,7 +57,7 @@ func convertSchema(sch *arrow.Schema) *arrow.Schema {
 	fields := make([]arrow.Field, len(oldFields))
 	copy(fields, oldFields)
 	for i, f := range fields {
-		if !isTypeSupported(f.Type) {
+		if !typeSupported(f.Type) {
 			fields[i].Type = arrow.BinaryTypes.String
 		}
 		fields[i].Metadata = stripCQExtensionMetadata(fields[i].Metadata)
@@ -68,24 +68,28 @@ func convertSchema(sch *arrow.Schema) *arrow.Schema {
 	return newSchema
 }
 
-func isTypeSupported(t arrow.DataType) bool {
-	// list from arrow/csv/common.go
-	switch t.(type) {
+// typeSupported copied from arrow/csv/common.go
+func typeSupported(dt arrow.DataType) bool {
+	switch dt := dt.(type) {
 	case *arrow.BooleanType:
 	case *arrow.Int8Type, *arrow.Int16Type, *arrow.Int32Type, *arrow.Int64Type:
 	case *arrow.Uint8Type, *arrow.Uint16Type, *arrow.Uint32Type, *arrow.Uint64Type:
-	case *arrow.Float32Type, *arrow.Float64Type:
-	case *arrow.StringType:
+	case *arrow.Float16Type, *arrow.Float32Type, *arrow.Float64Type:
+	case *arrow.StringType, *arrow.LargeStringType:
 	case *arrow.TimestampType:
 	case *arrow.Date32Type, *arrow.Date64Type:
 	case *arrow.Decimal128Type, *arrow.Decimal256Type:
-	case *arrow.ListType:
-	case *arrow.BinaryType:
+	case *arrow.MapType:
+		return false
+	case arrow.ListLikeType:
+		return typeSupported(dt.Elem())
+	case *arrow.BinaryType, *arrow.LargeBinaryType, *arrow.FixedSizeBinaryType:
 	case arrow.ExtensionType:
-		return true
+	case *arrow.NullType:
+	default:
+		return false
 	}
-
-	return false
+	return true
 }
 
 // castToString casts extension columns or unsupported columns to string. It does not release the original record.
@@ -94,7 +98,7 @@ func castToString(rec arrow.Record) arrow.Record {
 	cols := make([]arrow.Array, rec.NumCols())
 	for c := 0; c < int(rec.NumCols()); c++ {
 		col := rec.Column(c)
-		if isTypeSupported(col.DataType()) {
+		if typeSupported(col.DataType()) {
 			cols[c] = col
 			continue
 		}