feat(bigquery): support null marker for csv in external data config (#5287)

Author: shollyman

Diff for: bigquery/external.go

@@ -219,6 +219,10 @@ type CSVOptions struct {
 	// The number of rows at the top of a CSV file that BigQuery will skip when
 	// reading data.
 	SkipLeadingRows int64
+
+	// An optional custom string that will represent a NULL
+	// value in CSV import data.
+	NullMarker string
 }
 
 func (o *CSVOptions) populateExternalDataConfig(c *bq.ExternalDataConfiguration) {
@@ -229,6 +233,7 @@ func (o *CSVOptions) populateExternalDataConfig(c *bq.ExternalDataConfiguration)
 		FieldDelimiter:      o.FieldDelimiter,
 		Quote:               o.quote(),
 		SkipLeadingRows:     o.SkipLeadingRows,
+		NullMarker:          o.NullMarker,
 	}
 }
 
@@ -260,6 +265,7 @@ func bqToCSVOptions(q *bq.CsvOptions) *CSVOptions {
 		Encoding:            Encoding(q.Encoding),
 		FieldDelimiter:      q.FieldDelimiter,
 		SkipLeadingRows:     q.SkipLeadingRows,
+		NullMarker:          q.NullMarker,
 	}
 	o.setQuote(q.Quote)
 	return o

Diff for: bigquery/external_test.go

@@ -39,6 +39,7 @@ func TestExternalDataConfig(t *testing.T) {
 				FieldDelimiter:      "f",
 				Quote:               "q",
 				SkipLeadingRows:     3,
+				NullMarker:          "marker",
 			},
 		},
 		{

Diff for: bigquery/file.go

@@ -92,6 +92,7 @@ func (fc *FileConfig) populateLoadConfig(conf *bq.JobConfigurationLoad) {
 	conf.FieldDelimiter = fc.FieldDelimiter
 	conf.IgnoreUnknownValues = fc.IgnoreUnknownValues
 	conf.MaxBadRecords = fc.MaxBadRecords
+	conf.NullMarker = fc.NullMarker
 	if fc.Schema != nil {
 		conf.Schema = fc.Schema.toBQ()
 	}
@@ -118,6 +119,7 @@ func bqPopulateFileConfig(conf *bq.JobConfigurationLoad, fc *FileConfig) {
 	fc.AllowQuotedNewlines = conf.AllowQuotedNewlines
 	fc.Encoding = Encoding(conf.Encoding)
 	fc.FieldDelimiter = conf.FieldDelimiter
+	fc.CSVOptions.NullMarker = conf.NullMarker
 	fc.CSVOptions.setQuote(conf.Quote)
 }
 

Diff for: bigquery/file_test.go

@@ -39,6 +39,7 @@ var (
 			AllowJaggedRows:     true,
 			AllowQuotedNewlines: true,
 			Encoding:            UTF_8,
+			NullMarker:          "marker",
 		},
 	}
 )
@@ -71,6 +72,7 @@ func TestFileConfigPopulateLoadConfig(t *testing.T) {
 				Encoding:            "UTF-8",
 				MaxBadRecords:       7,
 				IgnoreUnknownValues: true,
+				NullMarker:          "marker",
 				Schema: &bq.TableSchema{
 					Fields: []*bq.TableFieldSchema{
 						bqStringFieldSchema(),
@@ -154,6 +156,7 @@ func TestFileConfigPopulateExternalDataConfig(t *testing.T) {
 					FieldDelimiter:      "\t",
 					Quote:               &hyphen,
 					SkipLeadingRows:     8,
+					NullMarker:          "marker",
 				},
 			},
 		},