Skip to content

Commit

Permalink
feat: add support for JSON data type in Spanner (#6390)
Browse files Browse the repository at this point in the history
  • Loading branch information
@olavloite
olavloite committed Aug 23, 2021
1 parent 29077ad commit 7c6a6f1
Show file tree
Hide file tree
Showing 11 changed files with 179 additions and 42 deletions.
66 changes: 47 additions & 19 deletions ...gle.Cloud.Spanner.Data/Google.Cloud.Spanner.Data.IntegrationTests/AllTypesTableFixture.cs
View file
Expand Up @@ -30,25 +30,53 @@ public AllTypesTableFixture() : base("TypesTable")

protected override void CreateTable()
{
ExecuteDdl($@"CREATE TABLE {TableName}(
K STRING(MAX) NOT NULL,
BoolValue BOOL,
Int64Value INT64,
Float64Value FLOAT64,
NumericValue NUMERIC,
StringValue STRING(MAX),
BytesValue BYTES(MAX),
TimestampValue TIMESTAMP,
DateValue DATE,
BoolArrayValue ARRAY<BOOL>,
Int64ArrayValue ARRAY<INT64>,
Float64ArrayValue ARRAY<FLOAT64>,
NumericArrayValue ARRAY<NUMERIC>,
StringArrayValue ARRAY < STRING(MAX) >,
BytesArrayValue ARRAY < BYTES(MAX) >,
TimestampArrayValue ARRAY<TIMESTAMP>,
DateArrayValue ARRAY<DATE>,
) PRIMARY KEY(K)");
// The emulator doesn't yet support the JSON type.
if (RunningOnEmulator)
{
ExecuteDdl($@"CREATE TABLE {TableName}(
K STRING(MAX) NOT NULL,
BoolValue BOOL,
Int64Value INT64,
Float64Value FLOAT64,
StringValue STRING(MAX),
NumericValue NUMERIC,
BytesValue BYTES(MAX),
TimestampValue TIMESTAMP,
DateValue DATE,
BoolArrayValue ARRAY<BOOL>,
Int64ArrayValue ARRAY<INT64>,
Float64ArrayValue ARRAY<FLOAT64>,
NumericArrayValue ARRAY<NUMERIC>,
StringArrayValue ARRAY<STRING(MAX)>,
BytesArrayValue ARRAY<BYTES(MAX)>,
TimestampArrayValue ARRAY<TIMESTAMP>,
DateArrayValue ARRAY<DATE>,
) PRIMARY KEY(K)");
}
else
{
ExecuteDdl($@"CREATE TABLE {TableName}(
K STRING(MAX) NOT NULL,
BoolValue BOOL,
Int64Value INT64,
Float64Value FLOAT64,
NumericValue NUMERIC,
StringValue STRING(MAX),
BytesValue BYTES(MAX),
TimestampValue TIMESTAMP,
DateValue DATE,
JsonValue JSON,
BoolArrayValue ARRAY<BOOL>,
Int64ArrayValue ARRAY<INT64>,
Float64ArrayValue ARRAY<FLOAT64>,
NumericArrayValue ARRAY<NUMERIC>,
StringArrayValue ARRAY<STRING(MAX)>,
BytesArrayValue ARRAY<BYTES(MAX)>,
TimestampArrayValue ARRAY<TIMESTAMP>,
DateArrayValue ARRAY<DATE>,
JsonArrayValue ARRAY<JSON>,
) PRIMARY KEY(K)");
}
}
}
}
28 changes: 27 additions & 1 deletion apis/Google.Cloud.Spanner.Data/Google.Cloud.Spanner.Data.IntegrationTests/BindingTests.cs
View file
Expand Up @@ -39,6 +39,7 @@ public class BindingTests
SpannerDbType.Numeric,
SpannerDbType.Date,
SpannerDbType.Bytes,
SpannerDbType.Json,
SpannerDbType.ArrayOf(SpannerDbType.Bool),
SpannerDbType.ArrayOf(SpannerDbType.String),
SpannerDbType.ArrayOf(SpannerDbType.Int64),
Expand All @@ -47,12 +48,14 @@ public class BindingTests
SpannerDbType.ArrayOf(SpannerDbType.Numeric),
SpannerDbType.ArrayOf(SpannerDbType.Date),
SpannerDbType.ArrayOf(SpannerDbType.Bytes),
SpannerDbType.ArrayOf(SpannerDbType.Json),
};

[Theory]
[SkippableTheory]
[MemberData(nameof(BindNullData))]
public async Task BindNull(SpannerDbType parameterType)
{
Skip.If(_fixture.RunningOnEmulator && (SpannerDbType.Json.Equals(parameterType) || SpannerDbType.ArrayOf(SpannerDbType.Json).Equals(parameterType)), "The emulator does not support the JSON type");
using (var connection = _fixture.GetConnection())
{
var cmd = connection.CreateSelectCommand("SELECT @v");
Expand Down Expand Up @@ -235,5 +238,28 @@ public async Task BindNumericEmptyArray()
public Task BindTimestampEmptyArray() => TestBindNonNull(
SpannerDbType.ArrayOf(SpannerDbType.Timestamp),
new DateTime?[] { });

[SkippableFact]
public async Task BindJson()
{
Skip.If(_fixture.RunningOnEmulator, "The emulator doesn't yet support the JSON type");
await TestBindNonNull(SpannerDbType.Json, "{\"key\":\"value\"}", r => r.GetString(0));
}

[SkippableFact]
public async Task BindJsonArray()
{
Skip.If(_fixture.RunningOnEmulator, "The emulator doesn't yet support the JSON type");
await TestBindNonNull(
SpannerDbType.ArrayOf(SpannerDbType.Json),
new string[] { "{\"key\":\"value\"}", null, "{\"other-key\":\"other-value\"}" });
}

[SkippableFact]
public async Task BindJsonEmptyArray()
{
Skip.If(_fixture.RunningOnEmulator, "The emulator doesn't yet support the JSON type");
await TestBindNonNull(SpannerDbType.ArrayOf(SpannerDbType.Json), new string[] { });
}
}
}
10 changes: 7 additions & 3 deletions ...ogle.Cloud.Spanner.Data/Google.Cloud.Spanner.Data.IntegrationTests/GetSchemaTableTests.cs
View file
Expand Up @@ -43,9 +43,10 @@ public async Task GetSchemaTable_Default_ReturnsNull()
}

[MemberData(nameof(SchemaTestData))]
[Theory]
[SkippableTheory]
public async Task GetSchemaTable_WithFlagEnabled_ReturnsSchema(string columnName, System.Type type, SpannerDbType spannerDbType)
{
Skip.If(_fixture.RunningOnEmulator && (SpannerDbType.Json.Equals(spannerDbType) || SpannerDbType.ArrayOf(SpannerDbType.Json).Equals(spannerDbType)), "The emulator does not support the JSON type");
using (var connection = new SpannerConnection($"{_fixture.ConnectionString};EnableGetSchemaTable=true"))
{
var command = connection.CreateSelectCommand($"SELECT {columnName} FROM {_fixture.TableName}");
Expand Down Expand Up @@ -80,6 +81,7 @@ public async Task GetSchemaTable_WithFlagEnabled_ReturnsSchema(string columnName
{ "BytesValue", typeof(byte[]), SpannerDbType.Bytes },
{ "TimestampValue", typeof(DateTime), SpannerDbType.Timestamp },
{ "DateValue", typeof(DateTime), SpannerDbType.Date },
{ "JsonValue", typeof(string), SpannerDbType.Json },
// Array types.
{ "BoolArrayValue", typeof(List<bool>), SpannerDbType.ArrayOf(SpannerDbType.Bool) },
{ "Int64ArrayValue", typeof(List<long>), SpannerDbType.ArrayOf(SpannerDbType.Int64) },
Expand All @@ -89,6 +91,7 @@ public async Task GetSchemaTable_WithFlagEnabled_ReturnsSchema(string columnName
{ "BytesArrayValue", typeof(List<byte[]>), SpannerDbType.ArrayOf(SpannerDbType.Bytes) },
{ "TimestampArrayValue", typeof(List<DateTime>), SpannerDbType.ArrayOf(SpannerDbType.Timestamp) },
{ "DateArrayValue", typeof(List<DateTime>), SpannerDbType.ArrayOf(SpannerDbType.Date) },
{ "JsonArrayValue", typeof(List<string>), SpannerDbType.ArrayOf(SpannerDbType.Json) },
};

[Fact]
Expand All @@ -101,8 +104,9 @@ public async Task GetSchemaTable_WithFlagEnabled_ReturnsColumnOrdinals()
{
var table = reader.GetSchemaTable();
// The table also contains the `K` column that is the primary key.
Assert.Equal(SchemaTestData.Count() + 1, table.Rows.Count);
for (var ordinal = 1; ordinal <= SchemaTestData.Count(); ordinal++)
var expectedRowCount = _fixture.RunningOnEmulator ? SchemaTestData.Count() - 1 : SchemaTestData.Count() + 1;
Assert.Equal(expectedRowCount, table.Rows.Count);
for (var ordinal = 1; ordinal < expectedRowCount; ordinal++)
{
var row = table.Rows[ordinal];
Assert.Equal(ordinal, (int)row["ColumnOrdinal"]);
Expand Down
36 changes: 36 additions & 0 deletions apis/Google.Cloud.Spanner.Data/Google.Cloud.Spanner.Data.IntegrationTests/WriteTests.cs
View file
Expand Up @@ -79,6 +79,7 @@ public async Task WriteValues()
long?[] lArray = { 0, null, 1 };
double?[] dArray = { 0.0, null, 2.0 };
SpannerNumeric?[] nArray = { SpannerNumeric.Parse("0.0"), null, SpannerNumeric.Parse("2.0") };
string[] jsonArray = { "{\"f1\":\"v1\"}", "{}", "[]", null };
string[] sArray = { "abc", null, "123" };
string[] bArrayArray =
{
Expand Down Expand Up @@ -109,6 +110,13 @@ public async Task WriteValues()
{ "NumericArrayValue", SpannerDbType.ArrayOf(SpannerDbType.Numeric), nArray }
};

// The emulator doesn't yet support the JSON type.
if (!_fixture.RunningOnEmulator)
{
parameters.Add("JsonValue", SpannerDbType.Json, "{\"f1\":\"v1\"}");
parameters.Add("JsonArrayValue", SpannerDbType.ArrayOf(SpannerDbType.Json), jsonArray);
}

Assert.Equal(1, await InsertAsync(parameters));
await WithLastRowAsync(reader =>
{
Expand All @@ -130,6 +138,11 @@ public async Task WriteValues()
Assert.Equal(tmArray, reader.GetFieldValue<DateTime?[]>(reader.GetOrdinal("TimestampArrayValue")));
Assert.Equal(dtArray, reader.GetFieldValue<DateTime?[]>(reader.GetOrdinal("DateArrayValue")));
Assert.Equal(nArray, reader.GetFieldValue<SpannerNumeric?[]>(reader.GetOrdinal("NumericArrayValue")));
if (!_fixture.RunningOnEmulator)
{
Assert.Equal("{\"f1\":\"v1\"}", reader.GetFieldValue<string>(reader.GetOrdinal("JsonValue")));
Assert.Equal(jsonArray, reader.GetFieldValue<string[]>(reader.GetOrdinal("JsonArrayValue")));
}
});
}

Expand Down Expand Up @@ -190,6 +203,12 @@ public async Task WriteEmpties()
{ "NumericArrayValue", SpannerDbType.ArrayOf(SpannerDbType.Numeric), new SpannerNumeric[0] }
};

// The emulator doesn't yet support the JSON type.
if (!_fixture.RunningOnEmulator)
{
parameters.Add("JsonArrayValue", SpannerDbType.ArrayOf(SpannerDbType.Json), new string[0]);
}

Assert.Equal(1, await InsertAsync(parameters));
await WithLastRowAsync(reader =>
{
Expand All @@ -202,6 +221,11 @@ public async Task WriteEmpties()
new DateTime[] { }, reader.GetFieldValue<DateTime[]>(reader.GetOrdinal("TimestampArrayValue")));
Assert.Equal(new DateTime[] { }, reader.GetFieldValue<DateTime[]>(reader.GetOrdinal("DateArrayValue")));
Assert.Equal(new SpannerNumeric[] { }, reader.GetFieldValue<SpannerNumeric[]>(reader.GetOrdinal("NumericArrayValue")));
if (!_fixture.RunningOnEmulator)
{
Assert.Equal(new SpannerNumeric[] { }, reader.GetFieldValue<SpannerNumeric[]>(reader.GetOrdinal("NumericArrayValue")));
Assert.Equal(new string[] { }, reader.GetFieldValue<string[]>(reader.GetOrdinal("JsonArrayValue")));
}
});
}

Expand Down Expand Up @@ -249,6 +273,13 @@ public async Task WriteNulls()
{ "NumericArrayValue", SpannerDbType.ArrayOf(SpannerDbType.Numeric), null }
};

// The emulator doesn't yet support the JSON type.
if (!_fixture.RunningOnEmulator)
{
parameters.Add("JsonValue", SpannerDbType.Json, null);
parameters.Add("JsonArrayValue", SpannerDbType.ArrayOf(SpannerDbType.Json), null);
}

Assert.Equal(1, await InsertAsync(parameters));
await WithLastRowAsync(reader =>
{
Expand All @@ -268,6 +299,11 @@ public async Task WriteNulls()
Assert.True(reader.IsDBNull(reader.GetOrdinal("TimestampArrayValue")));
Assert.True(reader.IsDBNull(reader.GetOrdinal("DateArrayValue")));
Assert.True(reader.IsDBNull(reader.GetOrdinal("NumericArrayValue")));
if (!_fixture.RunningOnEmulator)
{
Assert.True(reader.IsDBNull(reader.GetOrdinal("JsonValue")));
Assert.True(reader.IsDBNull(reader.GetOrdinal("JsonArrayValue")));
}
});
}

Expand Down
18 changes: 16 additions & 2 deletions ...Cloud.Spanner.Data/Google.Cloud.Spanner.Data.Tests/SpannerDbTypeTests.ValueConversions.cs
View file
Expand Up @@ -54,13 +54,14 @@ public enum TestType
{ "BoolField", SpannerDbType.Bool, true },
{ "DateField", SpannerDbType.Date, new DateTime(2017, 1, 31) },
{ "TimestampField", SpannerDbType.Timestamp, new DateTime(2017, 1, 31, 3, 15, 30) },
{ "NumericField", SpannerDbType.Numeric, SpannerNumeric.MaxValue }
{ "NumericField", SpannerDbType.Numeric, SpannerNumeric.MaxValue },
{ "JsonField", SpannerDbType.Json, "{\"field\": \"value\"}" }
};

// Structs are serialized as lists of their values. The field names aren't present, as they're
// specified in the type.
private static readonly string s_sampleStructSerialized =
"[ \"stringValue\", \"2\", \"NaN\", true, \"2017-01-31\", \"2017-01-31T03:15:30Z\", \"99999999999999999999999999999.999999999\" ]";
"[ \"stringValue\", \"2\", \"NaN\", true, \"2017-01-31\", \"2017-01-31T03:15:30Z\", \"99999999999999999999999999999.999999999\", \"{\\\"field\\\": \\\"value\\\"}\" ]";

private static string Quote(string s) => $"\"{s}\"";

Expand Down Expand Up @@ -113,6 +114,13 @@ private static IEnumerable<SpannerNumeric> GetSpannerNumericsForArray()
yield return SpannerNumeric.MaxValue;
}

private static IEnumerable<string> GetJsonStringsForArray()
{
yield return "";
yield return "{\"field1\": \"value1\"}";
yield return "[]";
}

private static void WithCulture(CultureInfo culture, Action action)
{
var originalCulture = CultureInfo.CurrentCulture;
Expand Down Expand Up @@ -313,6 +321,12 @@ public static IEnumerable<object[]> GetValidValueConversions()
new List<SpannerNumeric>(GetSpannerNumericsForArray()), SpannerDbType.ArrayOf(SpannerDbType.Numeric),
"[ \"-99999999999999999999999999999.999999999\", \"0.000000001\", \"99999999999999999999999999999.999999999\" ]"
};
// JSON can not be converted from Value to Clr, as there is no unique Clr type for JSON.
yield return new object[]
{
new List<string>(GetJsonStringsForArray()), SpannerDbType.ArrayOf(SpannerDbType.Json),
"[ \"\", \"{\\\"field1\\\": \\\"value1\\\"}\", \"[]\" ]", TestType.ClrToValue
};

// List test cases (various source/target list types)
yield return new object[]
Expand Down
10 changes: 8 additions & 2 deletions apis/Google.Cloud.Spanner.Data/Google.Cloud.Spanner.Data.Tests/SpannerDbTypeTests.cs
View file
Expand Up @@ -33,13 +33,15 @@ public static IEnumerable<object[]> GetSpannerDbTypes()
{
{ "StringValue", SpannerDbType.String, null },
{ "StringValue2", SpannerDbType.String, null },
{ "JsonValue", SpannerDbType.Json, null },
{ "FloatValue", SpannerDbType.Float64, null },
{ "BoolArrayValue", SpannerDbType.ArrayOf(SpannerDbType.Bool), null},
}.GetSpannerDbType(),
new SpannerStruct
{
{ "StringValue", SpannerDbType.String, null },
{ "StringValue2", SpannerDbType.String, null },
{ "JsonValue", SpannerDbType.Json, null },
{ "FloatValue", SpannerDbType.Float64, null },
{ "BoolArrayValue", SpannerDbType.ArrayOf(SpannerDbType.Bool), null},
}.GetSpannerDbType()
Expand All @@ -60,6 +62,7 @@ public static IEnumerable<object[]> GetSpannerStringConversions()
yield return new object[] { "STRING(MAX)", SpannerDbType.String };
yield return new object[] { "BOOL", SpannerDbType.Bool };
yield return new object[] { "BYTES", SpannerDbType.Bytes };
yield return new object[] { "JSON", SpannerDbType.Json };
yield return new object[] { "DATE", SpannerDbType.Date };
yield return new object[] { "FLOAT64", SpannerDbType.Float64 };
yield return new object[] { "INT64", SpannerDbType.Int64 };
Expand All @@ -69,6 +72,7 @@ public static IEnumerable<object[]> GetSpannerStringConversions()
yield return new object[] { " STRING ", SpannerDbType.String };
yield return new object[] { " BOOL ", SpannerDbType.Bool };
yield return new object[] { " BYTES ", SpannerDbType.Bytes };
yield return new object[] { " JSON ", SpannerDbType.Json };
yield return new object[] { " DATE ", SpannerDbType.Date };
yield return new object[] { " FLOAT64 ", SpannerDbType.Float64 };
yield return new object[] { " INT64 ", SpannerDbType.Int64 };
Expand All @@ -92,6 +96,7 @@ public static IEnumerable<object[]> GetSpannerStringConversions()
yield return new object[] { "ARRAY<BOOL>", SpannerDbType.ArrayOf(SpannerDbType.Bool) };
yield return new object[] { "ARRAY<BYTES>", SpannerDbType.ArrayOf(SpannerDbType.Bytes) };
yield return new object[] { "ARRAY<BYTES(100)>", SpannerDbType.ArrayOf(SpannerDbType.Bytes.WithSize(100)) };
yield return new object[] { "ARRAY<JSON>", SpannerDbType.ArrayOf(SpannerDbType.Json) };
yield return new object[] { "ARRAY<DATE>", SpannerDbType.ArrayOf(SpannerDbType.Date) };
yield return new object[] { "ARRAY<FLOAT64>", SpannerDbType.ArrayOf(SpannerDbType.Float64) };
yield return new object[] { "ARRAY<INT64>", SpannerDbType.ArrayOf(SpannerDbType.Int64) };
Expand Down Expand Up @@ -144,9 +149,10 @@ public static IEnumerable<object[]> GetSpannerStringConversions()
{ "F5", SpannerDbType.Date, null },
{ "F6", SpannerDbType.Float64, null },
{ "F7", SpannerDbType.Timestamp, null },
{ "F8", SpannerDbType.Numeric, null }
{ "F8", SpannerDbType.Numeric, null },
{ "F9", SpannerDbType.Json, null },
};
yield return new object[] { "STRUCT<F1:STRING,F2:INT64,F3:BOOL,F4:BYTES,F5:DATE,F6:FLOAT64,F7:TIMESTAMP,F8:NUMERIC>", sampleStruct.GetSpannerDbType() };
yield return new object[] { "STRUCT<F1:STRING,F2:INT64,F3:BOOL,F4:BYTES,F5:DATE,F6:FLOAT64,F7:TIMESTAMP,F8:NUMERIC,F9:JSON>", sampleStruct.GetSpannerDbType() };

sampleStruct = new SpannerStruct
{
Expand Down

0 comments on commit 7c6a6f1

Please sign in to comment.