From e6c5d69db44682af8f37865f4af6dd1e7d659a84 Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Sun, 30 Oct 2022 15:01:39 -0300 Subject: [PATCH 01/25] First basic impl for runtime of cosmosDB. Insert and select all items query --- dotnet/DotNetStandardClasses.sln | 15 +- .../DynService.Cosmos/CosmosDBConnection.cs | 247 +++++++++++++++ .../DynService.Cosmos/CosmosDBDataReader.cs | 292 ++++++++++++++++++ .../CosmosDBDatastoreHelper.cs | 59 ++++ .../DynService.Cosmos/CosmosDBHelper.cs | 104 +++++++ .../DynService.Cosmos/CosmosDBMaps.cs | 39 +++ .../CosmosDBRequestWrapper.cs | 63 ++++ .../CosmosDBResponseWrapper.cs | 32 ++ .../DynService.CosmosDB.csproj | 32 ++ .../DynService.Dynamo/DynamoDBConnection.cs | 1 + 10 files changed, 881 insertions(+), 3 deletions(-) create mode 100644 dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs create mode 100644 dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs create mode 100644 dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDatastoreHelper.cs create mode 100644 dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs create mode 100644 dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBMaps.cs create mode 100644 dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBRequestWrapper.cs create mode 100644 dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBResponseWrapper.cs create mode 100644 dotnet/src/dotnetcommon/DynService.Cosmos/DynService.CosmosDB.csproj diff --git a/dotnet/DotNetStandardClasses.sln b/dotnet/DotNetStandardClasses.sln index a2a2c74f3..fd1dc6287 100644 --- a/dotnet/DotNetStandardClasses.sln +++ b/dotnet/DotNetStandardClasses.sln @@ -208,11 +208,15 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestMockDBAccess", "src\ext EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GeneXus.Programs.Common", "src\extensions\Azure\test\GeneXus.Programs.Common\GeneXus.Programs.Common.csproj", "{DCEC0B38-93B6-4003-81E6-9FBC2BB4F163}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GXAmazonSQS", "src\dotnetcore\Providers\Messaging\GXAmazonSQS\GXAmazonSQS.csproj", "{F8BA0D65-267D-491F-BFAB-33F5E5B61AD7}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GXAmazonSQS", "src\dotnetcore\Providers\Messaging\GXAmazonSQS\GXAmazonSQS.csproj", "{F8BA0D65-267D-491F-BFAB-33F5E5B61AD7}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "apiattractions", "src\extensions\Azure\test\apiattractions\apiattractions.csproj", "{E85FDB0F-FA81-4CDD-8BF3-865269CE2DB3}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProjectHealthTest", "test\ProjectHealthTest\ProjectHealthTest.csproj", "{65048104-212A-4819-AECF-89CA9C08C83F}" -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DotNetRedisTest", "test\DotNetRedisTest\DotNetRedisTest.csproj", "{48430E50-043A-47A2-8278-B86A4420758A}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ProjectHealthTest", "test\ProjectHealthTest\ProjectHealthTest.csproj", "{65048104-212A-4819-AECF-89CA9C08C83F}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotNetRedisTest", "test\DotNetRedisTest\DotNetRedisTest.csproj", "{48430E50-043A-47A2-8278-B86A4420758A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DynService.CosmosDB", "src\dotnetcommon\DynService.Cosmos\DynService.CosmosDB.csproj", "{19625C3D-CEE1-44B7-A11E-1AA27E6A5439}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -520,6 +524,10 @@ Global {48430E50-043A-47A2-8278-B86A4420758A}.Debug|Any CPU.Build.0 = Debug|Any CPU {48430E50-043A-47A2-8278-B86A4420758A}.Release|Any CPU.ActiveCfg = Release|Any CPU {48430E50-043A-47A2-8278-B86A4420758A}.Release|Any CPU.Build.0 = Release|Any CPU + {19625C3D-CEE1-44B7-A11E-1AA27E6A5439}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {19625C3D-CEE1-44B7-A11E-1AA27E6A5439}.Debug|Any CPU.Build.0 = Debug|Any CPU + {19625C3D-CEE1-44B7-A11E-1AA27E6A5439}.Release|Any CPU.ActiveCfg = Release|Any CPU + {19625C3D-CEE1-44B7-A11E-1AA27E6A5439}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -624,6 +632,7 @@ Global {E85FDB0F-FA81-4CDD-8BF3-865269CE2DB3} = {7BA5A2CE-7992-4F87-9D84-91AE4D046F5A} {65048104-212A-4819-AECF-89CA9C08C83F} = {1D6F1776-FF4B-46C2-9B3D-BC46CCF049DC} {48430E50-043A-47A2-8278-B86A4420758A} = {1D6F1776-FF4B-46C2-9B3D-BC46CCF049DC} + {19625C3D-CEE1-44B7-A11E-1AA27E6A5439} = {C264F34E-2CE0-4DCA-B22D-4155821BE611} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {E18684C9-7D76-45CD-BF24-E3944B7F174C} diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs new file mode 100644 index 000000000..ca79cacd0 --- /dev/null +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs @@ -0,0 +1,247 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.Common; +using System.IO; +using System.Text; +using System.Threading.Tasks; +using GeneXus.Data.Cosmos; +using GeneXus.Data.NTier.CosmosDB; +using Microsoft.Azure.Cosmos; + +namespace GeneXus.Data.NTier +{ + + public class CosmosDBService : GxService + { + public CosmosDBService(string id, string providerId) : base(id, providerId, typeof(CosmosDBConnection)) + { + + } + /*public override IDataReader GetCacheDataReader(CacheItem item, bool computeSize, string keyCache) + { + return new GxDynamoDBCacheDataReader(item, computeSize, keyCache); + }*/ + } + + public class CosmosDBConnection : ServiceConnection + { + private const string REGION = "ApplicationRegion"; + private const string DATABASE = "database"; + private const string SERVICE_URI = "serviceURI"; + private const string ACCOUNT_KEY = "AccountKey"; + private static CosmosClient cosmosClient; + private static Database cosmosDatabase; + private static string mapplicationRegion; + private static string mdatabase; + private static string mAccountKey; + private static string mserviceURI; + private static string mConnectionString; + + //https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/query/select + private const string SELECT_CMD = "SELECT"; + private const string FROM = "FROM"; + private const string TABLE_ALIAS = "t"; + //private const string WHERE = "WHERE"; + //private const string DISTINCT = "DISTINCT"; + + //LOG? + + //TODO: Usar un Hashset para guardar los containers + + private static readonly object _lock = new object(); + public override string ConnectionString + { + get + { + return mConnectionString; + } + + set + { + mConnectionString = value; + State = ConnectionState.Executing; + InitializeDBConnection(); + } + } + private static void InitializeDBConnection() + { + + DbConnectionStringBuilder builder = new DbConnectionStringBuilder(false); + builder.ConnectionString = mConnectionString; + + if (builder.TryGetValue(SERVICE_URI, out object serviceURI)) + { + mserviceURI = serviceURI.ToString(); + } + if (builder.TryGetValue(REGION, out object region)) + { + mapplicationRegion = region.ToString(); + } + if (builder.TryGetValue(ACCOUNT_KEY, out object accountKey)) + { + mAccountKey = accountKey.ToString(); + mserviceURI = $"{mserviceURI};AccountKey={mAccountKey}"; + } + if (builder.TryGetValue(DATABASE, out object database)) + { + mdatabase = database.ToString(); + } + //TODO: check Mandatory parameters + //TODO: Connect using connection string + connection key + } + + private static void Initialize() + { + if (cosmosClient == null) + { + lock (_lock) + { + if (cosmosClient == null) + { + if (!string.IsNullOrEmpty(mserviceURI) && !string.IsNullOrEmpty(mapplicationRegion)) + cosmosClient = new CosmosClient(mserviceURI, new CosmosClientOptions() { ApplicationRegion = mapplicationRegion }); + + if (!string.IsNullOrEmpty(mdatabase)) + cosmosDatabase = cosmosClient.GetDatabase(mdatabase); + } + } + } + } + + private Container GetContainer(string containerName) + { + if (cosmosDatabase != null && !string.IsNullOrEmpty(containerName)) + return cosmosClient.GetContainer(cosmosDatabase.Id, containerName); + return null; + } + + public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCollection parms, CommandBehavior behavior) + { + Initialize(); + CosmosDBQuery query = cursorDef.Query as CosmosDBQuery; + bool isInsert = query.CursorType == ServiceCursorDef.CursorType.Insert; + + Dictionary values = new Dictionary(); + string jsonData = string.Empty; + + string partitionKey = query.PartitionKey; + string partitionKeyValue = string.Empty; + foreach (KeyValuePair asg in query.AssignAtts) + { + string name = asg.Key; + string parmName = asg.Value.Substring(1); + CosmosDBHelper.AddItemValue(isInsert ? name : $":{name}", parmName, values, parms, query.Vars, ref jsonData); + if (name == partitionKey) + partitionKeyValue = values[name].ToString(); + } + jsonData = "{" + jsonData + "}"; + + switch (query.CursorType) + { + case ServiceCursorDef.CursorType.Select: + throw new NotImplementedException(); + + case ServiceCursorDef.CursorType.Delete: + + //TODO + throw new NotImplementedException(); + + case ServiceCursorDef.CursorType.Insert: + //TODO: Get container from HashSet for performance + Container container = GetContainer(query.TableName); + if (container != null) + { + try + { + using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(jsonData))) + { + + Task task = Task.Run(async () => await container.CreateItemStreamAsync(stream, new PartitionKey(partitionKeyValue)).ConfigureAwait(false)); + if (task.Result.IsSuccessStatusCode) + { + return 1; + } + else + { + throw new Exception($"Create item from stream failed. Status code: {task.Result.StatusCode} Message: {task.Result.ErrorMessage}"); + + } + + } + } + catch (Exception ex) // TODO check CosmosDB Exception https://learn.microsoft.com/en-us/dotnet/api/microsoft.azure.cosmos.cosmosexception?view=azure-dotnet + { + throw ex; + } + } + else + { + throw new Exception("Execution failed. Container not found."); + } + case ServiceCursorDef.CursorType.Update: + throw new NotImplementedException(); + } + + return 0; + } + public override IDataReader ExecuteReader(ServiceCursorDef cursorDef, IDataParameterCollection parms, CommandBehavior behavior) + { + + Initialize(); + CosmosDBQuery query = cursorDef.Query as CosmosDBQuery; + //Get container from hashset for performance + Container container = GetContainer(query.TableName); + try + { + CreateCosmosQuery(query,cursorDef, GetQueryValues(query, parms), container, out CosmosDBDataReader dataReader, out RequestWrapper requestWrapper); + return dataReader; + } + catch (CosmosException cosmosException) + { + //TODO: Handle cases + throw cosmosException; + } + + catch (Exception e) { throw e; } + } + private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef,Dictionary queryValues, Container container, out CosmosDBDataReader cosmosDBDataReader,out RequestWrapper requestWrapper) + { + //Create the query + + string tableName = query.TableName; + IEnumerable projection = query.Projection; + + string element; + string projectionList = string.Empty; + foreach (string key in projection) + { + element = $"{TABLE_ALIAS}.{key}"; + if (!string.IsNullOrEmpty(projectionList)) + projectionList = $"{element},{projectionList}"; + else + projectionList = $"{element}"; + + } + + string sqlQuery = $"{SELECT_CMD} {projectionList} {FROM} {tableName} {TABLE_ALIAS}"; + + QueryDefinition queryDefinition = new QueryDefinition(sqlQuery); + requestWrapper = new RequestWrapper(cosmosClient, container, queryDefinition); + cosmosDBDataReader = new CosmosDBDataReader(cursorDef, requestWrapper); + } + private Dictionary GetQueryValues(CosmosDBQuery query, IDataParameterCollection parms) + { + Dictionary values = new Dictionary(); + //TODO: Handle IDataParameterCollection params + + return values; + } + internal static IOServiceContext NewServiceContext() => null; + } + public class CosmosDBErrors + { + public const string ValidationException = "ValidationException"; + public const string ValidationExceptionMessageKey = "The AttributeValue for a key attribute cannot contain an empty string value."; + } +} diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs new file mode 100644 index 000000000..560661754 --- /dev/null +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs @@ -0,0 +1,292 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using GeneXus.Data.NTier; +using GeneXus.Data.NTier.CosmosDB; +using Microsoft.Azure.Cosmos; +using Newtonsoft.Json; + +namespace GeneXus.Data.Cosmos +{ + public class CosmosDBDataReader : IDataReader + { + private readonly RequestWrapper mRequest; + private ResponseWrapper mResponse; + private readonly IODataMap2[] selectList; + private FeedIterator feedIterator; + private CosmosDBRecordEntry currentEntry; + private int mCurrentPosition; + + private int ItemCount; + private List> Items = null; + + private void CheckCurrentPosition() + { + if (currentEntry == null) + throw new ServiceException(ServiceError.RecordNotFound); + } + public CosmosDBDataReader(ServiceCursorDef cursorDef, RequestWrapper request) + { + Query query = cursorDef.Query as Query; + selectList = query.SelectList.ToArray(); + mRequest = request; + mResponse = mRequest.Read(); + feedIterator = mResponse.feedIterator; + //mCurrentPosition = -1; + } + public object this[string name] + { + get + { + throw new NotImplementedException(); + } + } + + public object this[int i] + { + get + { + throw new NotImplementedException(); + } + } + public int Depth + { + get + { + return 0; + } + } + + public int FieldCount + { + get + { + return selectList.Length; + } + } + public bool IsClosed + { + get + { + return false; + } + } + + public int RecordsAffected + { + get + { + return -1; + } + } + + public void Close() + { + if (mRequest != null) + { + mRequest.Close(); + } + } + public void Dispose() + { + } + public long getLong(int i) + { + return Convert.ToInt64(GetAttValue(i)); + } + + private object GetAttValue(int i) + { + return (selectList[i].GetValue(CosmosDBConnection.NewServiceContext(), currentEntry)); + } + public bool GetBoolean(int i) + { + if (GetAttValue(i) is bool value) + { + return value; + } + return false; + } + + public byte GetByte(int i) + { + return Convert.ToByte(GetAttValue(i)); + } + + public long GetBytes(int i, long fieldOffset, byte[] buffer, int bufferoffset, int length) + { + MemoryStream ms = (MemoryStream)GetAttValue(i); + if (ms == null) + return 0; + ms.Seek(fieldOffset, SeekOrigin.Begin); + return ms.Read(buffer, bufferoffset, length); + } + public char GetChar(int i) + { + return Convert.ToChar(GetAttValue(i)); + } + public long GetChars(int i, long fieldoffset, char[] buffer, int bufferoffset, int length) + { + throw new NotImplementedException(); + } + + public IDataReader GetData(int i) + { + throw new NotImplementedException(); + } + + public string GetDataTypeName(int i) + { + throw new NotImplementedException(); + } + + public DateTime GetDateTime(int i) + { + if (GetAttValue(i) is DateTime value) + return value; + return default(DateTime); + // DateTime.TryParse(GetAttValue(i).ToString(), null, DateTimeStyles.AdjustToUniversal, out DateTime dt); + // return dt; + } + public decimal GetDecimal(int i) + { + return Convert.ToDecimal(GetAttValue(i)); + } + public double GetDouble(int i) + { + return Convert.ToDouble(GetAttValue(i)); + } + public Type GetFieldType(int i) + { + return selectList[i].GetValue(CosmosDBConnection.NewServiceContext(), currentEntry).GetType(); + } + + public float GetFloat(int i) + { + return Convert.ToSingle(GetAttValue(i)); + } + public Guid GetGuid(int i) + { + return new Guid(GetAttValue(i).ToString()); + } + + public short GetInt16(int i) + { + return Convert.ToInt16(GetAttValue(i)); + } + + public int GetInt32(int i) + { + return Convert.ToInt32(GetAttValue(i)); + } + + public long GetInt64(int i) + { + return Convert.ToInt64(GetAttValue(i)); ; + } + + public string GetName(int i) + { + return selectList[i].GetName(null); + } + + public int GetOrdinal(string name) + { + CheckCurrentPosition(); + int ordinal = currentEntry.CurrentRow.ToList().FindIndex(col => col.Key.ToLower() == name.ToLower()); + if (ordinal == -1) + throw new ArgumentOutOfRangeException(nameof(name)); + else return ordinal; + } + public DataTable GetSchemaTable() + { + throw new NotImplementedException(); + } + public string GetString(int i) + { + return GetAttValue(i).ToString(); + } + public int GetValues(object[] values) + { + System.Diagnostics.Debug.Assert(selectList.Length == values.Length, "Values mismatch"); + for (int i = 0; i < selectList.Length && i < values.Length; i++) + { + values[i] = GetAttValue(i); + } + return selectList.Length; + } + public bool IsDBNull(int i) + { + return GetAttValue(i) == null; + } + + public bool NextResult() + { + mCurrentPosition++; + currentEntry = (mCurrentPosition < ItemCount) ? new CosmosDBRecordEntry(Items[mCurrentPosition]) : null; + return currentEntry != null; + } + + private async Task GetPage() + { + while (feedIterator.HasMoreResults) + { + using (ResponseMessage response = await feedIterator.ReadNextAsync().ConfigureAwait(false)) + { + if (response.Diagnostics != null) + { + //LOG + //Console.WriteLine($"ItemStreamFeed Diagnostics: {response.Diagnostics.ToString()}"); + } + + response.EnsureSuccessStatusCode(); + using (StreamReader sr = new StreamReader(response.Content)) + using (JsonTextReader jtr = new JsonTextReader(sr)) + { + Newtonsoft.Json.JsonSerializer jsonSerializer = new Newtonsoft.Json.JsonSerializer(); + object array = jsonSerializer.Deserialize(jtr); + + string json = ((Newtonsoft.Json.Linq.JToken)array).Root.ToString(); + var jsonDocument = JsonDocument.Parse(json); + var jsonDoc = jsonDocument.RootElement; + foreach (var jsonProperty in jsonDoc.EnumerateObject()) + { + if (jsonProperty.Name == "Documents") + { + Items = JsonConvert.DeserializeObject>>(jsonProperty.Value.ToString()); + break; + } + } + ItemCount = Items.Count; + } + } + return true; + } + return false; + } + public bool Read() + { + Task task; + if (NextResult()) + return true; + else + task = Task.Run(async () => await GetPage().ConfigureAwait(false)); + if (task.Result) + { + mCurrentPosition = -1; + return NextResult(); + } + return false; + } + + public object GetValue(int i) + { + return GetAttValue(i); + } + } + +} diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDatastoreHelper.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDatastoreHelper.cs new file mode 100644 index 000000000..1099e6c39 --- /dev/null +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDatastoreHelper.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using GeneXus.Data.Cosmos; + +namespace GeneXus.Data.NTier +{ + public class CosmosDBDatastoreHelper : DynServiceDataStoreHelperBase + { + + //TODO Esto no aplica a CosmosDB + public CosmosDBQuery NewQuery() => new CosmosDBQuery(this); + + public CosmosDBQuery NewScan() => new CosmosDBQuery(this); + + public CosmosDBMap Map(string name) + { + return new CosmosDBMap(name); + } + + } + + public static class CosmosFluentExtensions + { + + public static Query SetPartitionKey(this Query cosmosQuery, string partitionKey) + { + return (cosmosQuery as CosmosDBQuery)?.SetKey(partitionKey); + } + + } + public class CosmosDBQuery : Query + { + public string Index { get; set; } + + public CosmosDBQuery OrderBy(string index) + { + ////TO DO/// + return this; + } + + public string PartitionKey { get; private set; } + public CosmosDBQuery SetKey(string partitionKey) + { + PartitionKey = partitionKey; + return this; + } + internal IEnumerable KeyFilters { get; set; } = Array.Empty(); + public CosmosDBQuery KeyFilter(string[] filters) + { + KeyFilters = filters; + return this; + } + + public CosmosDBQuery(CosmosDBDatastoreHelper dataStoreHelper) : base(dataStoreHelper) { } + } +} diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs new file mode 100644 index 000000000..301ec483b --- /dev/null +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs @@ -0,0 +1,104 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using GeneXus.Data.NTier; + +namespace GeneXus.Data.Cosmos +{ + internal class CosmosDBHelper + { + internal static bool AddItemValue(string parmName, string fromName, Dictionary values, IDataParameterCollection parms, IEnumerable queryVars, ref string jsonData) + { + + if (!AddItemValue(parmName, values, parms[fromName] as ServiceParameter, out string data)) + { + //TODO + return false; + } + StringBuilder stringBuilder = new StringBuilder(jsonData); + string concatData; + if (!string.IsNullOrEmpty(jsonData)) + { + concatData = $",{data}"; + stringBuilder.Append(concatData); + jsonData = stringBuilder.ToString(); + } + else + jsonData = data; + + return true; + } + internal static bool AddItemValue(string parmName, Dictionary dynParm, ServiceParameter parm, out string jsonData) + { + jsonData = string.Empty; + if (parm == null) + return false; + object value = ToItemValue(parm.DbType, parm.Value); + if (value != null) + { + dynParm[parmName] = value; + string valueItem; + if (parm.DbType == DbType.String) + { + //if (!string.IsNullOrEmpty(jsonData)) + // { + valueItem = string.Format("\"{0}\"", value); + jsonData = string.Format("\"{0}\": {1}", parmName, string.Join(",", valueItem)); + //} + } + else + jsonData = string.Format("\"{0}\": {1}", parmName, string.Join(",", value)); + return true; + } + return false; + } + + internal static object ToItemValue(DbType dbType, Object value) + { + object attValue; + switch (dbType) + { + case DbType.Binary: + if (value is byte[] valueArr) + { + attValue = new MemoryStream(valueArr); + break; + } + else throw new ArgumentException("Required value not found"); + case DbType.Boolean: + case DbType.Byte: + attValue = Convert.ToByte(value); + break; + case DbType.Time: + case DbType.Date: + attValue = DateTime.SpecifyKind((DateTime)value, DateTimeKind.Utc).ToString("yyyy-MM-dd"); + break; + case DbType.DateTime2: + case DbType.DateTime: + attValue = DateTime.SpecifyKind((DateTime)value, DateTimeKind.Utc).ToString("yyyy-MM-ddTHH:mm:ssK"); + break; + + case DbType.UInt16: + case DbType.UInt32: + case DbType.UInt64: + case DbType.VarNumeric: + case DbType.Decimal: + case DbType.Double: + case DbType.Int16: + case DbType.Int32: + case DbType.Int64: + attValue = value.ToString(); + break; + default: + string valueDefault = value.ToString().Replace("%", string.Empty); + attValue = valueDefault; + break; + } + return attValue; + } + } +} diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBMaps.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBMaps.cs new file mode 100644 index 000000000..de8b5311d --- /dev/null +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBMaps.cs @@ -0,0 +1,39 @@ +using GeneXus.Data.NTier; +using System; +using System.Collections.Generic; + +namespace GeneXus.Data.Cosmos +{ + public class CosmosDBMap : Map + { + internal bool NeedsAttributeMap { get; } + + //TODO + public CosmosDBMap(string name): base(name) + { + + } + public override object GetValue(IOServiceContext context, RecordEntryRow currentEntry) + { + Dictionary values = ((CosmosDBRecordEntry)currentEntry).CurrentRow; + + values.TryGetValue(GetName(context), out object val); + return val; + } + + public override void SetValue(RecordEntryRow currentEntry, object value) + { + throw new NotImplementedException(); + } + + } + public class CosmosDBRecordEntry : RecordEntryRow + { + public Dictionary CurrentRow { get; } + + public CosmosDBRecordEntry(Dictionary cRow) + { + CurrentRow = cRow; + } + } +} diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBRequestWrapper.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBRequestWrapper.cs new file mode 100644 index 000000000..46b258b68 --- /dev/null +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBRequestWrapper.cs @@ -0,0 +1,63 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Microsoft.Azure.Cosmos; +using Newtonsoft.Json; + +namespace GeneXus.Data.NTier.CosmosDB +{ + public class RequestWrapper + { + private readonly Container mcontainer; + private readonly CosmosClient mcosmosClient; + private readonly QueryDefinition mqueryDefinition; + //wrapper string idValue, string partitionKeyValue + public RequestWrapper(CosmosClient cosmosClient, Container container, QueryDefinition queryDefinition) + { + mcontainer = container; + mcosmosClient = cosmosClient; + mqueryDefinition = queryDefinition; + } + + //TODO: Por PK usar este metodo que es mas performante + /*private async Task ReadItemAsyncByPK(string idValue, string partitionKeyValue) + { + using (ResponseMessage responseMessage = await mcontainer.ReadItemStreamAsync( + partitionKey: new PartitionKey(partitionKeyValue), + id: idValue).ConfigureAwait(false)) + { + + if (responseMessage.IsSuccessStatusCode) + { + return null; + // Log the diagnostics + //Console.WriteLine($"\n1.2.2 - Item Read Diagnostics: {responseMessage.Diagnostics.ToString()}"); + } + else + { + return new ResponseWrapper(responseMessage); + //Console.WriteLine($"Read item from stream failed. Status code: {responseMessage.StatusCode} Message: {responseMessage.ErrorMessage}"); + } + } + }*/ + public ResponseWrapper Read() + { + /* if (queryByPK) + { + Task task = Task.Run(async () => await ReadItemAsyncByPK(idValue, partitionKeyValue).ConfigureAwait(false)); + return task.Result; + }*/ + //GetItemQueryStreamIterator + QueryRequestOptions requestOptions = new QueryRequestOptions() { MaxBufferedItemCount = 100 }; + //options.MaxConcurrency = 1; + //TODO Cancelation Token + request options + using (FeedIterator feedIterator = mcontainer.GetItemQueryStreamIterator(mqueryDefinition, null, requestOptions)) + + return new ResponseWrapper(feedIterator); + } + internal void Close() + { + mcosmosClient.Dispose(); + } + } +} diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBResponseWrapper.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBResponseWrapper.cs new file mode 100644 index 000000000..67114e09c --- /dev/null +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBResponseWrapper.cs @@ -0,0 +1,32 @@ +using Microsoft.Azure.Cosmos; +using System.Collections.Generic; +using System.IO; + +namespace GeneXus.Data.NTier.CosmosDB +{ + public class ResponseWrapper + { + + public FeedIterator feedIterator; + Stream stream; + + public ResponseWrapper(ResponseMessage responseMessage, FeedIterator feedIter) + { + feedIterator = feedIter; + stream = responseMessage.Content; + //Items = queryResponse.Items; + //ItemCount = queryResponse.Items.Count; + } + public ResponseWrapper(FeedIterator feedIter) + { + feedIterator = feedIter; + //stream = responseMessage.Content; + //Items = queryResponse.Items; + //ItemCount = queryResponse.Items.Count; + } + + public List> Items { get; set; } + public int ItemCount { get; set; } + + } +} diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/DynService.CosmosDB.csproj b/dotnet/src/dotnetcommon/DynService.Cosmos/DynService.CosmosDB.csproj new file mode 100644 index 000000000..b461f15f4 --- /dev/null +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/DynService.CosmosDB.csproj @@ -0,0 +1,32 @@ + + + net462;net6.0 + GeneXus.Data.NTier + GeneXus.Data.DynService.CosmosDB + false + CosmosDB + GeneXus.DynService.CosmosDB + + + + NETCORE + + + + + + + + + + + + + + + + + + + + diff --git a/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBConnection.cs b/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBConnection.cs index f3b2acf9d..88182e51c 100644 --- a/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBConnection.cs +++ b/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBConnection.cs @@ -44,6 +44,7 @@ public class DynamoDBConnection : ServiceConnection private void InitializeDBConnection() { + System.Diagnostics.Debugger.Launch(); DbConnectionStringBuilder builder = new DbConnectionStringBuilder(false); builder.ConnectionString = this.ConnectionString; mConfig = new AmazonDynamoDBConfig(); From db703208c03b3d6f3634d5f44580363e1f6e03eb Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Sun, 30 Oct 2022 15:08:36 -0300 Subject: [PATCH 02/25] remove debugger line --- dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBConnection.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBConnection.cs b/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBConnection.cs index 88182e51c..f3b2acf9d 100644 --- a/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBConnection.cs +++ b/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBConnection.cs @@ -44,7 +44,6 @@ public class DynamoDBConnection : ServiceConnection private void InitializeDBConnection() { - System.Diagnostics.Debugger.Launch(); DbConnectionStringBuilder builder = new DbConnectionStringBuilder(false); builder.ConnectionString = this.ConnectionString; mConfig = new AmazonDynamoDBConfig(); From 44aebce7405892ef2f3333bfef46f226db43e526 Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Thu, 24 Nov 2022 15:51:28 -0300 Subject: [PATCH 03/25] select by filters --- .../DynService.Cosmos/CosmosDBConnection.cs | 80 +++++++++++++++++-- .../CosmosDBDatastoreHelper.cs | 12 ++- .../DynService.Dynamo/DynamoDBConnection.cs | 1 - 3 files changed, 84 insertions(+), 9 deletions(-) diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs index ca79cacd0..8bc6dfe3d 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs @@ -3,7 +3,9 @@ using System.Data; using System.Data.Common; using System.IO; +using System.Linq; using System.Text; +using System.Text.RegularExpressions; using System.Threading.Tasks; using GeneXus.Data.Cosmos; using GeneXus.Data.NTier.CosmosDB; @@ -42,8 +44,9 @@ public class CosmosDBConnection : ServiceConnection private const string SELECT_CMD = "SELECT"; private const string FROM = "FROM"; private const string TABLE_ALIAS = "t"; - //private const string WHERE = "WHERE"; + private const string WHERE = "WHERE"; //private const string DISTINCT = "DISTINCT"; + //private const string ORDER_BY = "ORDER BY"; //LOG? @@ -66,7 +69,6 @@ public override string ConnectionString } private static void InitializeDBConnection() { - DbConnectionStringBuilder builder = new DbConnectionStringBuilder(false); builder.ConnectionString = mConnectionString; @@ -208,7 +210,6 @@ public override IDataReader ExecuteReader(ServiceCursorDef cursorDef, IDataParam private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef,Dictionary queryValues, Container container, out CosmosDBDataReader cosmosDBDataReader,out RequestWrapper requestWrapper) { //Create the query - string tableName = query.TableName; IEnumerable projection = query.Projection; @@ -221,10 +222,75 @@ private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef,Di projectionList = $"{element},{projectionList}"; else projectionList = $"{element}"; + } + + IEnumerable allFilters = query.KeyFilters.Concat(query.Filters); + IEnumerable allFiltersQuery = Array.Empty(); + + string regex1 = @"\(([^)]+)\)"; + string regex2 = @"(.*)\s*([=|!=|<|>|<=|>=|<>])\s*(:.*)"; + + string keyFilterS; + string condition = string.Empty; + IEnumerable keyFilterQ = Array.Empty(); + + foreach (string keyFilter in allFilters) + { + keyFilterS = keyFilter; + condition = keyFilter; + if (keyFilter.StartsWith("((")) + { + keyFilterS = keyFilter.Remove(0, 1); + keyFilterS = keyFilterS.Remove(keyFilterS.Length - 1, 1); + } + MatchCollection matchCollection = Regex.Matches(keyFilterS, regex1); + + foreach (Match match in matchCollection) + { + if (match.Groups.Count > 0) + { + string cond = match.Groups[1].Value; + Match match2 = Regex.Match(cond, regex2); + + //Get the value for this item + string varValuestr = string.Empty; + if (match2.Groups.Count > 0) + { + string column = match2.Groups[1].Value.Trim(); + column = $"{TABLE_ALIAS}.{column}"; + + string op = match2.Groups[2].Value.Trim(); + foreach (VarValue item in query.Vars) + { + if (item.Name == match2.Groups[3].Value) + { + if (item.Type == GXType.Char || item.Type == GXType.LongVarChar || item.Type == GXType.VarChar || item.Type == GXType.Text) + //Nvarchar, etc? + //Date? + { + varValuestr = '"' + $"{item.Value.ToString()}" + '"'; + } + else + varValuestr = item.Value.ToString(); + break; + } + } + condition = condition.Replace(cond, $"{column}{op}{varValuestr}"); + } + } + } + keyFilterQ = new string[] { condition }; + allFiltersQuery = allFiltersQuery.Concat(keyFilterQ); + } - - string sqlQuery = $"{SELECT_CMD} {projectionList} {FROM} {tableName} {TABLE_ALIAS}"; + string FilterExpression = allFiltersQuery.Any() ? String.Join(" AND ", allFiltersQuery) : null; + + string sqlQuery = string.Empty; + if (FilterExpression != null) + sqlQuery = $"{SELECT_CMD} {projectionList} {FROM} {tableName} {TABLE_ALIAS} {WHERE} {FilterExpression}"; + else + sqlQuery = $"{SELECT_CMD} {projectionList} {FROM} {tableName} {TABLE_ALIAS}"; QueryDefinition queryDefinition = new QueryDefinition(sqlQuery); requestWrapper = new RequestWrapper(cosmosClient, container, queryDefinition); @@ -235,7 +301,11 @@ private Dictionary GetQueryValues(CosmosDBQuery query, IDataPara Dictionary values = new Dictionary(); //TODO: Handle IDataParameterCollection params + // foreach (VarValue item in query.Vars) + // values.Add(item.Name, DynamoDBHelper.ToAttributeValue(item)); + return values; + } internal static IOServiceContext NewServiceContext() => null; } diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDatastoreHelper.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDatastoreHelper.cs index 1099e6c39..b0e518a65 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDatastoreHelper.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDatastoreHelper.cs @@ -21,15 +21,21 @@ public CosmosDBMap Map(string name) } } - public static class CosmosFluentExtensions { - public static Query SetPartitionKey(this Query cosmosQuery, string partitionKey) { return (cosmosQuery as CosmosDBQuery)?.SetKey(partitionKey); } - + public static Query KeyFilter(this Query cosmosQuery, string[] filters) + { + return (cosmosQuery as CosmosDBQuery)?.KeyFilter(filters); + } + public static Query SetKey(this Query cosmosQuery, string partitionKey) + { + return (cosmosQuery as CosmosDBQuery)?.SetKey(partitionKey); + } + } public class CosmosDBQuery : Query { diff --git a/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBConnection.cs b/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBConnection.cs index cb206c19e..f3b2acf9d 100644 --- a/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBConnection.cs +++ b/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBConnection.cs @@ -73,7 +73,6 @@ private void InitializeDBConnection() private void Initialize() { - System.Diagnostics.Debugger.Launch(); InitializeDBConnection(); State = ConnectionState.Executing; From 65192693334c89bc6e8806277cfc121e47dda118 Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Sun, 4 Dec 2022 16:31:10 -0300 Subject: [PATCH 04/25] Update and delete implemented. --- .../DynService.Core/DynServiceCommon.cs | 13 ++ .../DynService.Cosmos/CosmosDBConnection.cs | 183 ++++++++++++++---- .../CosmosDBDatastoreHelper.cs | 22 +-- .../DynService.Cosmos/CosmosDBHelper.cs | 64 ++++-- .../DynamoDBDataStoreHelper.cs | 35 ++-- 5 files changed, 230 insertions(+), 87 deletions(-) diff --git a/dotnet/src/dotnetcommon/DynService.Core/DynServiceCommon.cs b/dotnet/src/dotnetcommon/DynService.Core/DynServiceCommon.cs index 695113439..17965e3ab 100644 --- a/dotnet/src/dotnetcommon/DynService.Core/DynServiceCommon.cs +++ b/dotnet/src/dotnetcommon/DynService.Core/DynServiceCommon.cs @@ -79,6 +79,19 @@ public Query AddConst(GXType gxType, object parm) mVarValues.Add(new VarValue($":const{ mVarValues.Count + 1 }", gxType, parm)); return this; } + + public virtual Query SetKey(string key) + { + return null; + } + public virtual Query KeyFilter(string[] filters) + { + return null; + } + public virtual Query OrderBy(string index) + { + return null; + } } public class VarValue diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs index 8bc6dfe3d..57794644c 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs @@ -45,6 +45,9 @@ public class CosmosDBConnection : ServiceConnection private const string FROM = "FROM"; private const string TABLE_ALIAS = "t"; private const string WHERE = "WHERE"; + + private const string FILTER_PATTERN = @"\((.*) = :(.*)\)"; + //private const string DISTINCT = "DISTINCT"; //private const string ORDER_BY = "ORDER BY"; @@ -52,7 +55,6 @@ public class CosmosDBConnection : ServiceConnection //TODO: Usar un Hashset para guardar los containers - private static readonly object _lock = new object(); public override string ConnectionString { get @@ -95,20 +97,11 @@ private static void InitializeDBConnection() private static void Initialize() { - if (cosmosClient == null) - { - lock (_lock) - { - if (cosmosClient == null) - { - if (!string.IsNullOrEmpty(mserviceURI) && !string.IsNullOrEmpty(mapplicationRegion)) - cosmosClient = new CosmosClient(mserviceURI, new CosmosClientOptions() { ApplicationRegion = mapplicationRegion }); + if (!string.IsNullOrEmpty(mserviceURI) && !string.IsNullOrEmpty(mapplicationRegion)) + cosmosClient = new CosmosClient(mserviceURI, new CosmosClientOptions() { ApplicationRegion = mapplicationRegion }); - if (!string.IsNullOrEmpty(mdatabase)) - cosmosDatabase = cosmosClient.GetDatabase(mdatabase); - } - } - } + if (!string.IsNullOrEmpty(mdatabase)) + cosmosDatabase = cosmosClient.GetDatabase(mdatabase); } private Container GetContainer(string containerName) @@ -123,6 +116,7 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo Initialize(); CosmosDBQuery query = cursorDef.Query as CosmosDBQuery; bool isInsert = query.CursorType == ServiceCursorDef.CursorType.Insert; + bool isUpdate = query.CursorType == ServiceCursorDef.CursorType.Update; Dictionary values = new Dictionary(); string jsonData = string.Empty; @@ -133,25 +127,100 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo { string name = asg.Key; string parmName = asg.Value.Substring(1); - CosmosDBHelper.AddItemValue(isInsert ? name : $":{name}", parmName, values, parms, query.Vars, ref jsonData); + CosmosDBHelper.AddItemValue(name, parmName, values, parms, query.Vars, ref jsonData); if (name == partitionKey) partitionKeyValue = values[name].ToString(); } - jsonData = "{" + jsonData + "}"; + Dictionary keyCondition = new Dictionary(); + + foreach (string keyFilter in query.KeyFilters.Concat(query.Filters)) + { + Match match = Regex.Match(keyFilter, FILTER_PATTERN); + if (match.Groups.Count > 1) + { + string varName = match.Groups[2].Value; + string name = match.Groups[1].Value; + VarValue varValue = query.Vars.FirstOrDefault(v => v.Name == $":{varName}"); + + string jsonDataKey = String.Empty; + if (varValue != null) + { + keyCondition[name] = varValue.Value; + //keyCondition[name] = GeneXus.Data.Cosmos.CosmosDBHelper.ToItemValue(varValue.Type, varValue.Value); + + if (isUpdate && name == "id") + jsonDataKey = GeneXus.Data.Cosmos.CosmosDBHelper.AddToJsonStream(varValue.Type, name, varValue.Value); + } + else + { + if (parms[varName] is ServiceParameter serviceParm) + { + keyCondition[name] = serviceParm.Value; + //keyCondition[name] = GeneXus.Data.Cosmos.CosmosDBHelper.ToItemValue(serviceParm.DbType, serviceParm.Value); + + if (isUpdate && name == "id") + jsonDataKey = GeneXus.Data.Cosmos.CosmosDBHelper.AddToJsonStream(serviceParm.DbType, name, serviceParm.Value); + } + } + if (!string.IsNullOrEmpty(jsonData)) + { + jsonData = $"{jsonData},{jsonDataKey}"; + } + else + jsonData = jsonDataKey; + + } + } + + jsonData = "{" + jsonData + "}"; + + //TODO: Get container from HashSet for performance + Container container = GetContainer(query.TableName); switch (query.CursorType) { + case ServiceCursorDef.CursorType.Select: throw new NotImplementedException(); case ServiceCursorDef.CursorType.Delete: - //TODO - throw new NotImplementedException(); - + if (container != null) + { + try + { + if (keyCondition != null && keyCondition["id"] != null) + { + if (string.IsNullOrEmpty(partitionKeyValue)) + partitionKeyValue = keyCondition["id"].ToString(); // partitionKeyValue = id + Task task = Task.Run(async () => await container.DeleteItemStreamAsync(keyCondition["id"].ToString(), new PartitionKey(partitionKeyValue)).ConfigureAwait(false)); + if (task.Result.IsSuccessStatusCode) + { + //ResponseMessage wrapps the delete record + return 1; + } + else + { + if (task.Result.ErrorMessage.Contains("404")) + throw new ServiceException(ServiceError.RecordNotFound, null); + else + throw new Exception($"Delete item from stream failed. Status code: {task.Result.StatusCode} Message: {task.Result.ErrorMessage}"); + } + } + else + throw new Exception($"Delete item failed: error parsing the query."); + + } + catch (Exception ex) + { throw ex; } + } + + else + { + throw new Exception("Execution failed. Container not found."); + } + case ServiceCursorDef.CursorType.Insert: - //TODO: Get container from HashSet for performance - Container container = GetContainer(query.TableName); if (container != null) { try @@ -166,6 +235,10 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo } else { + if (task.Result.ErrorMessage.Contains("Conflict (409)")) + throw new ServiceException(ServiceError.RecordAlreadyExists, null); + else + throw new Exception($"Create item from stream failed. Status code: {task.Result.StatusCode} Message: {task.Result.ErrorMessage}"); } @@ -182,7 +255,38 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo throw new Exception("Execution failed. Container not found."); } case ServiceCursorDef.CursorType.Update: - throw new NotImplementedException(); + if (container != null) + { + try + { + if (string.IsNullOrEmpty(partitionKeyValue)) + partitionKeyValue = keyCondition["id"].ToString(); // partitionKeyValue = id + using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(jsonData))) + { + + Task task = Task.Run(async () => await container.UpsertItemStreamAsync(stream, new PartitionKey(partitionKeyValue)).ConfigureAwait(false)); + if (task.Result.IsSuccessStatusCode) + { + return 1; + } + else + { + + throw new Exception($"Update item from stream failed. Status code: {task.Result.StatusCode} Message: {task.Result.ErrorMessage}"); + + } + + } + } + catch (Exception ex) // TODO check CosmosDB Exception https://learn.microsoft.com/en-us/dotnet/api/microsoft.azure.cosmos.cosmosexception?view=azure-dotnet + { + throw ex; + } + } + else + { + throw new Exception("Execution failed. Container not found."); + } } return 0; @@ -196,7 +300,7 @@ public override IDataReader ExecuteReader(ServiceCursorDef cursorDef, IDataParam Container container = GetContainer(query.TableName); try { - CreateCosmosQuery(query,cursorDef, GetQueryValues(query, parms), container, out CosmosDBDataReader dataReader, out RequestWrapper requestWrapper); + CreateCosmosQuery(query,cursorDef, parms, container, out CosmosDBDataReader dataReader, out RequestWrapper requestWrapper); return dataReader; } catch (CosmosException cosmosException) @@ -207,7 +311,7 @@ public override IDataReader ExecuteReader(ServiceCursorDef cursorDef, IDataParam catch (Exception e) { throw e; } } - private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef,Dictionary queryValues, Container container, out CosmosDBDataReader cosmosDBDataReader,out RequestWrapper requestWrapper) + private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, IDataParameterCollection parms, Container container, out CosmosDBDataReader cosmosDBDataReader,out RequestWrapper requestWrapper) { //Create the query string tableName = query.TableName; @@ -259,13 +363,31 @@ private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef,Di string column = match2.Groups[1].Value.Trim(); column = $"{TABLE_ALIAS}.{column}"; + string attName = match2.Groups[3].Value.Trim(); + if (attName.StartsWith(":")) + attName = attName.Substring(1); + string op = match2.Groups[2].Value.Trim(); + //look at IDataParameterCollection parms + if (parms[attName] is ServiceParameter serviceParm) + if (GeneXus.Data.Cosmos.CosmosDBHelper.FormattedAsStringDbType(serviceParm.DbType)) + //Nvarchar, etc? + //Date? + { + varValuestr = '"' + $"{serviceParm.Value.ToString()}" + '"'; + } + else + varValuestr = serviceParm.Value.ToString(); + + + //look at query.vars foreach (VarValue item in query.Vars) { if (item.Name == match2.Groups[3].Value) { - if (item.Type == GXType.Char || item.Type == GXType.LongVarChar || item.Type == GXType.VarChar || item.Type == GXType.Text) + //if (item.Type == GXType.Char || item.Type == GXType.LongVarChar || item.Type == GXType.VarChar || item.Type == GXType.Text) + if (GeneXus.Data.Cosmos.CosmosDBHelper.FormattedAsStringGXType(item.Type)) //Nvarchar, etc? //Date? { @@ -296,17 +418,6 @@ private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef,Di requestWrapper = new RequestWrapper(cosmosClient, container, queryDefinition); cosmosDBDataReader = new CosmosDBDataReader(cursorDef, requestWrapper); } - private Dictionary GetQueryValues(CosmosDBQuery query, IDataParameterCollection parms) - { - Dictionary values = new Dictionary(); - //TODO: Handle IDataParameterCollection params - - // foreach (VarValue item in query.Vars) - // values.Add(item.Name, DynamoDBHelper.ToAttributeValue(item)); - - return values; - - } internal static IOServiceContext NewServiceContext() => null; } public class CosmosDBErrors diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDatastoreHelper.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDatastoreHelper.cs index b0e518a65..e2643f9e4 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDatastoreHelper.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDatastoreHelper.cs @@ -20,41 +20,25 @@ public CosmosDBMap Map(string name) return new CosmosDBMap(name); } - } - public static class CosmosFluentExtensions - { - public static Query SetPartitionKey(this Query cosmosQuery, string partitionKey) - { - return (cosmosQuery as CosmosDBQuery)?.SetKey(partitionKey); - } - public static Query KeyFilter(this Query cosmosQuery, string[] filters) - { - return (cosmosQuery as CosmosDBQuery)?.KeyFilter(filters); - } - public static Query SetKey(this Query cosmosQuery, string partitionKey) - { - return (cosmosQuery as CosmosDBQuery)?.SetKey(partitionKey); - } - } public class CosmosDBQuery : Query { public string Index { get; set; } - public CosmosDBQuery OrderBy(string index) + public override Query OrderBy(string index) { ////TO DO/// return this; } public string PartitionKey { get; private set; } - public CosmosDBQuery SetKey(string partitionKey) + public override Query SetKey(string partitionKey) { PartitionKey = partitionKey; return this; } internal IEnumerable KeyFilters { get; set; } = Array.Empty(); - public CosmosDBQuery KeyFilter(string[] filters) + public override Query KeyFilter(string[] filters) { KeyFilters = filters; return this; diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs index 301ec483b..b4f56b2cb 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs @@ -16,8 +16,13 @@ internal static bool AddItemValue(string parmName, string fromName, Dictionary v.Name == $":{fromName}"); + if (varValue != null) + { + values[parmName] = varValue; + jsonData = AddToJsonStream(varValue.Type, parmName, varValue); + } + return varValue != null; } StringBuilder stringBuilder = new StringBuilder(jsonData); string concatData; @@ -32,6 +37,47 @@ internal static bool AddItemValue(string parmName, string fromName, Dictionary dynParm, ServiceParameter parm, out string jsonData) { jsonData = string.Empty; @@ -41,17 +87,7 @@ internal static bool AddItemValue(string parmName, Dictionary dy if (value != null) { dynParm[parmName] = value; - string valueItem; - if (parm.DbType == DbType.String) - { - //if (!string.IsNullOrEmpty(jsonData)) - // { - valueItem = string.Format("\"{0}\"", value); - jsonData = string.Format("\"{0}\": {1}", parmName, string.Join(",", valueItem)); - //} - } - else - jsonData = string.Format("\"{0}\": {1}", parmName, string.Join(",", value)); + jsonData = AddToJsonStream(parm.DbType, parmName, value); return true; } return false; @@ -71,7 +107,7 @@ internal static object ToItemValue(DbType dbType, Object value) else throw new ArgumentException("Required value not found"); case DbType.Boolean: case DbType.Byte: - attValue = Convert.ToByte(value); + attValue = Convert.ToByte(value) == 1 ? true : false; break; case DbType.Time: case DbType.Date: diff --git a/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBDataStoreHelper.cs b/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBDataStoreHelper.cs index 0c3cb20b8..86c8c21c3 100644 --- a/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBDataStoreHelper.cs +++ b/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBDataStoreHelper.cs @@ -56,21 +56,21 @@ public object empty(GXType gxtype) } } - public static class DynamoFluentExtensions - { - public static Query OrderBy(this Query dynamoQuery, string index) - { - return (dynamoQuery as DynamoQuery)?.OrderBy(index); - } - public static Query SetKey(this Query dynamoQuery, string partitionKey) - { - return (dynamoQuery as DynamoQuery)?.SetKey(partitionKey); - } - public static Query KeyFilter(this Query dynamoQuery, string[] filters) + //public static class DynamoFluentExtensions + //{ + //public static Query OrderBy(this Query dynamoQuery, string index) + //{ + // return (dynamoQuery as DynamoQuery)?.OrderBy(index); + //} + /*public static Query SetKey(this Query dynamoQuery, string partitionKey) { - return (dynamoQuery as DynamoQuery)?.KeyFilter(filters); - } - } + return (dynamoQuery.SetKey(partitionKey)); + }*/ + //public static Query KeyFilter(this Query dynamoQuery, string[] filters) + //{ + //return (dynamoQuery as DynamoQuery)?.KeyFilter(filters); + //} + //} public class DynamoQuery : Query { @@ -79,7 +79,7 @@ public class DynamoQuery : Query private const string RANGE_KEY_INDEX = "RangeKey"; private static readonly char[] indexTrimChars = new char[] { '(', ')' }; - public DynamoQuery OrderBy(string index) + public override Query OrderBy(string index) { if(index.StartsWith("(", StringComparison.InvariantCulture)) { @@ -92,18 +92,17 @@ public DynamoQuery OrderBy(string index) } public string PartitionKey { get; private set; } - public DynamoQuery SetKey(string partitionKey) + public override Query SetKey(string partitionKey) { PartitionKey = partitionKey; return this; } internal IEnumerable KeyFilters { get; set; } = Array.Empty(); - public DynamoQuery KeyFilter(string[] filters) + public override Query KeyFilter(string[] filters) { KeyFilters = filters; return this; } - public DynamoQuery(DynamoDBDataStoreHelper dataStoreHelper) : base(dataStoreHelper) { } } } From 5944014a26ce841c9f27084ce4de4ccab5a06da6 Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Fri, 9 Dec 2022 09:24:02 -0300 Subject: [PATCH 05/25] fix some type of sql queries --- .../DynService.Cosmos/CosmosDBConnection.cs | 63 ++++++++++--------- .../DynService.Cosmos/CosmosDBDataReader.cs | 61 +++++++++++------- .../DynService.Cosmos/CosmosDBHelper.cs | 4 ++ 3 files changed, 73 insertions(+), 55 deletions(-) diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs index 57794644c..e705640a7 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs @@ -9,6 +9,7 @@ using System.Threading.Tasks; using GeneXus.Data.Cosmos; using GeneXus.Data.NTier.CosmosDB; +using log4net; using Microsoft.Azure.Cosmos; namespace GeneXus.Data.NTier @@ -51,7 +52,7 @@ public class CosmosDBConnection : ServiceConnection //private const string DISTINCT = "DISTINCT"; //private const string ORDER_BY = "ORDER BY"; - //LOG? + //static readonly ILog logger = log4net.LogManager.GetLogger(typeof(CosmosDBConnection)); //TODO: Usar un Hashset para guardar los containers @@ -193,7 +194,7 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo { if (string.IsNullOrEmpty(partitionKeyValue)) partitionKeyValue = keyCondition["id"].ToString(); // partitionKeyValue = id - Task task = Task.Run(async () => await container.DeleteItemStreamAsync(keyCondition["id"].ToString(), new PartitionKey(partitionKeyValue)).ConfigureAwait(false)); + Task task = Task.Run(async () => await container.DeleteItemStreamAsync(keyCondition["id"].ToString(), new PartitionKey(partitionKeyValue)).ConfigureAwait(false)); if (task.Result.IsSuccessStatusCode) { //ResponseMessage wrapps the delete record @@ -204,7 +205,7 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo if (task.Result.ErrorMessage.Contains("404")) throw new ServiceException(ServiceError.RecordNotFound, null); else - throw new Exception($"Delete item from stream failed. Status code: {task.Result.StatusCode} Message: {task.Result.ErrorMessage}"); + throw new Exception($"Delete item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); } } else @@ -217,7 +218,7 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo else { - throw new Exception("Execution failed. Container not found."); + throw new Exception("CosmosDB Execution failed. Container not found."); } case ServiceCursorDef.CursorType.Insert: @@ -238,21 +239,19 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo if (task.Result.ErrorMessage.Contains("Conflict (409)")) throw new ServiceException(ServiceError.RecordAlreadyExists, null); else - - throw new Exception($"Create item from stream failed. Status code: {task.Result.StatusCode} Message: {task.Result.ErrorMessage}"); + throw new Exception($"Create item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); } - } } - catch (Exception ex) // TODO check CosmosDB Exception https://learn.microsoft.com/en-us/dotnet/api/microsoft.azure.cosmos.cosmosexception?view=azure-dotnet + catch (Exception ex) { throw ex; } } else { - throw new Exception("Execution failed. Container not found."); + throw new Exception("CosmosDB Execution failed. Container not found."); } case ServiceCursorDef.CursorType.Update: if (container != null) @@ -261,31 +260,29 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo { if (string.IsNullOrEmpty(partitionKeyValue)) partitionKeyValue = keyCondition["id"].ToString(); // partitionKeyValue = id - using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(jsonData))) - { - - Task task = Task.Run(async () => await container.UpsertItemStreamAsync(stream, new PartitionKey(partitionKeyValue)).ConfigureAwait(false)); - if (task.Result.IsSuccessStatusCode) - { - return 1; - } - else + using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(jsonData))) { + Task task = Task.Run(async () => await container.UpsertItemStreamAsync(stream, new PartitionKey(partitionKeyValue)).ConfigureAwait(false)); + if (task.Result.IsSuccessStatusCode) + { + return 1; + } + else + { - throw new Exception($"Update item from stream failed. Status code: {task.Result.StatusCode} Message: {task.Result.ErrorMessage}"); + throw new Exception($"Update item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); + } } - - } } - catch (Exception ex) // TODO check CosmosDB Exception https://learn.microsoft.com/en-us/dotnet/api/microsoft.azure.cosmos.cosmosexception?view=azure-dotnet + catch (Exception ex) { throw ex; } } else { - throw new Exception("Execution failed. Container not found."); + throw new Exception("CosmosDB Execution failed. Container not found."); } } @@ -331,9 +328,9 @@ private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, I IEnumerable allFilters = query.KeyFilters.Concat(query.Filters); IEnumerable allFiltersQuery = Array.Empty(); - string regex1 = @"\(([^)]+)\)"; - string regex2 = @"(.*)\s*([=|!=|<|>|<=|>=|<>])\s*(:.*)"; - + string regex1 = @"\(([^\)\(]+)\)"; + string regex2 = @"(.*)[^<>!=]\s*(=|!=|<|>|<=|>=|<>)\s*(:.*)"; + string keyFilterS; string condition = string.Empty; IEnumerable keyFilterQ = Array.Empty(); @@ -342,11 +339,7 @@ private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, I { keyFilterS = keyFilter; condition = keyFilter; - if (keyFilter.StartsWith("((")) - { - keyFilterS = keyFilter.Remove(0, 1); - keyFilterS = keyFilterS.Remove(keyFilterS.Length - 1, 1); - } + MatchCollection matchCollection = Regex.Matches(keyFilterS, regex1); foreach (Match match in matchCollection) @@ -398,8 +391,16 @@ private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, I break; } } + + //Controlar antes de mandar la sentencia a ejecutar + condition = condition.Replace(cond, $"{column}{op}{varValuestr}"); } + else + { + //Check that cond is a valid attribute - boolean + + } } } keyFilterQ = new string[] { condition }; diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs index 560661754..c827fd650 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using GeneXus.Data.NTier; using GeneXus.Data.NTier.CosmosDB; +using log4net; using Microsoft.Azure.Cosmos; using Newtonsoft.Json; @@ -24,6 +25,7 @@ public class CosmosDBDataReader : IDataReader private int ItemCount; private List> Items = null; + static readonly ILog logger = log4net.LogManager.GetLogger(typeof(CosmosDBDataReader)); private void CheckCurrentPosition() { if (currentEntry == null) @@ -235,36 +237,47 @@ private async Task GetPage() { while (feedIterator.HasMoreResults) { - using (ResponseMessage response = await feedIterator.ReadNextAsync().ConfigureAwait(false)) + try { - if (response.Diagnostics != null) + using (ResponseMessage response = await feedIterator.ReadNextAsync().ConfigureAwait(false)) { - //LOG - //Console.WriteLine($"ItemStreamFeed Diagnostics: {response.Diagnostics.ToString()}"); - } - - response.EnsureSuccessStatusCode(); - using (StreamReader sr = new StreamReader(response.Content)) - using (JsonTextReader jtr = new JsonTextReader(sr)) - { - Newtonsoft.Json.JsonSerializer jsonSerializer = new Newtonsoft.Json.JsonSerializer(); - object array = jsonSerializer.Deserialize(jtr); - - string json = ((Newtonsoft.Json.Linq.JToken)array).Root.ToString(); - var jsonDocument = JsonDocument.Parse(json); - var jsonDoc = jsonDocument.RootElement; - foreach (var jsonProperty in jsonDoc.EnumerateObject()) + + if (!response.IsSuccessStatusCode) { - if (jsonProperty.Name == "Documents") - { - Items = JsonConvert.DeserializeObject>>(jsonProperty.Value.ToString()); - break; + if (response.Diagnostics != null) + GXLogging.Debug(logger, $"Read ItemStreamFeed Diagnostics: {response.Diagnostics.ToString()}"); + throw new Exception(GeneXus.Data.Cosmos.CosmosDBHelper.FormatExceptionMessage(response.StatusCode.ToString(),response.ErrorMessage)); + } + else + { + using (StreamReader sr = new StreamReader(response.Content)) + using (JsonTextReader jtr = new JsonTextReader(sr)) + { + Newtonsoft.Json.JsonSerializer jsonSerializer = new Newtonsoft.Json.JsonSerializer(); + object array = jsonSerializer.Deserialize(jtr); + + string json = ((Newtonsoft.Json.Linq.JToken)array).Root.ToString(); + var jsonDocument = JsonDocument.Parse(json); + var jsonDoc = jsonDocument.RootElement; + foreach (var jsonProperty in jsonDoc.EnumerateObject()) + { + if (jsonProperty.Name == "Documents") + { + Items = JsonConvert.DeserializeObject>>(jsonProperty.Value.ToString()); + break; + } + } + ItemCount = Items.Count; } - } - ItemCount = Items.Count; + } } + return true; + } + catch (CosmosException ex) + { + GXLogging.Error(logger, ex); + throw ex; } - return true; } return false; } diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs index b4f56b2cb..42b271aa3 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs @@ -78,6 +78,10 @@ internal static string AddToJsonStream(DbType dbType, string parmName, object va } } + internal static string FormatExceptionMessage(string statusCode, string message) + { + return ($"CosmosDB Execution failed. Status code: {statusCode}. Message: {message}"); + } internal static bool AddItemValue(string parmName, Dictionary dynParm, ServiceParameter parm, out string jsonData) { jsonData = string.Empty; From d9c353b7d08f5ea8ca097241b39682470884ef15 Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Thu, 15 Dec 2022 13:52:52 -0300 Subject: [PATCH 06/25] Support tables with partition key different than the id --- .../DynService.Cosmos/CosmosDBConnection.cs | 122 +++++++++++------- .../DynService.Cosmos/CosmosDBDataReader.cs | 4 +- 2 files changed, 81 insertions(+), 45 deletions(-) diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs index e705640a7..4c58415d6 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs @@ -72,6 +72,7 @@ public override string ConnectionString } private static void InitializeDBConnection() { + DbConnectionStringBuilder builder = new DbConnectionStringBuilder(false); builder.ConnectionString = mConnectionString; @@ -145,6 +146,7 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo VarValue varValue = query.Vars.FirstOrDefault(v => v.Name == $":{varName}"); string jsonDataKey = String.Empty; + string jsonDataPartitionKey = string.Empty; if (varValue != null) { keyCondition[name] = varValue.Value; @@ -152,6 +154,12 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo if (isUpdate && name == "id") jsonDataKey = GeneXus.Data.Cosmos.CosmosDBHelper.AddToJsonStream(varValue.Type, name, varValue.Value); + if (isUpdate && name == partitionKey) + jsonDataPartitionKey = GeneXus.Data.Cosmos.CosmosDBHelper.AddToJsonStream(varValue.Type, name, varValue.Value); + + if (name == partitionKey) + //TODO Partition Key can be numeric + partitionKeyValue = varValue.Value.ToString(); } else { @@ -162,14 +170,31 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo if (isUpdate && name == "id") jsonDataKey = GeneXus.Data.Cosmos.CosmosDBHelper.AddToJsonStream(serviceParm.DbType, name, serviceParm.Value); + if (isUpdate && name == partitionKey) + jsonDataPartitionKey = GeneXus.Data.Cosmos.CosmosDBHelper.AddToJsonStream(serviceParm.DbType, name, serviceParm.Value); + + if (name == partitionKey) + //TODO Partition Key can be numeric + partitionKeyValue = serviceParm.Value.ToString(); } } - if (!string.IsNullOrEmpty(jsonData)) + if (!string.IsNullOrEmpty(jsonDataKey)) { - jsonData = $"{jsonData},{jsonDataKey}"; + if (!string.IsNullOrEmpty(jsonData)) + { + jsonData = $"{jsonData},{jsonDataKey}"; + } + else + jsonData = jsonDataKey; + } + + if (!string.IsNullOrEmpty(jsonDataPartitionKey)) + { + if (!string.IsNullOrEmpty(jsonData)) + jsonData = $"{jsonData},{jsonDataPartitionKey}"; + else + jsonData = jsonDataPartitionKey; } - else - jsonData = jsonDataKey; } } @@ -348,61 +373,70 @@ private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, I { string cond = match.Groups[1].Value; Match match2 = Regex.Match(cond, regex2); - - //Get the value for this item - string varValuestr = string.Empty; - if (match2.Groups.Count > 0) + if (match2.Success) { - string column = match2.Groups[1].Value.Trim(); - column = $"{TABLE_ALIAS}.{column}"; - - string attName = match2.Groups[3].Value.Trim(); - if (attName.StartsWith(":")) - attName = attName.Substring(1); - - string op = match2.Groups[2].Value.Trim(); + //Get the value for this item + string varValuestr = string.Empty; + if (match2.Groups.Count > 0) + { + string column = match2.Groups[1].Value.Trim(); + string attName = match2.Groups[3].Value.Trim(); + if (attName.StartsWith(":")) + attName = attName.Substring(1); - //look at IDataParameterCollection parms - if (parms[attName] is ServiceParameter serviceParm) - if (GeneXus.Data.Cosmos.CosmosDBHelper.FormattedAsStringDbType(serviceParm.DbType)) - //Nvarchar, etc? - //Date? - { - varValuestr = '"' + $"{serviceParm.Value.ToString()}" + '"'; - } - else - varValuestr = serviceParm.Value.ToString(); - + string op = match2.Groups[2].Value.Trim(); - //look at query.vars - foreach (VarValue item in query.Vars) - { - if (item.Name == match2.Groups[3].Value) - { - //if (item.Type == GXType.Char || item.Type == GXType.LongVarChar || item.Type == GXType.VarChar || item.Type == GXType.Text) - if (GeneXus.Data.Cosmos.CosmosDBHelper.FormattedAsStringGXType(item.Type)) + //look at IDataParameterCollection parms + if (parms[attName] is ServiceParameter serviceParm) + if (GeneXus.Data.Cosmos.CosmosDBHelper.FormattedAsStringDbType(serviceParm.DbType)) //Nvarchar, etc? //Date? { - varValuestr = '"' + $"{item.Value.ToString()}" + '"'; + varValuestr = '"' + $"{serviceParm.Value.ToString()}" + '"'; } else - varValuestr = item.Value.ToString(); - break; + varValuestr = serviceParm.Value.ToString(); + + + //look at query.vars + foreach (VarValue item in query.Vars) + { + if (item.Name == match2.Groups[3].Value) + { + //if (item.Type == GXType.Char || item.Type == GXType.LongVarChar || item.Type == GXType.VarChar || item.Type == GXType.Text) + if (GeneXus.Data.Cosmos.CosmosDBHelper.FormattedAsStringGXType(item.Type)) + //Nvarchar, etc? + //Date? + { + varValuestr = '"' + $"{item.Value.ToString()}" + '"'; + } + else + { + varValuestr = item.Value.ToString(); + varValuestr = varValuestr.Equals("True") ? "true" : varValuestr; + varValuestr = varValuestr.Equals("False") ? "false" : varValuestr; + } + break; + } } - } - //Controlar antes de mandar la sentencia a ejecutar + //Controlar antes de mandar la sentencia a ejecutar - condition = condition.Replace(cond, $"{column}{op}{varValuestr}"); - } - else - { - //Check that cond is a valid attribute - boolean + condition = condition.Replace(cond, $"{column}{op}{varValuestr}"); + } + else + { + //Check that cond is a valid attribute - boolean + } } } } + + foreach (string d in projection) + { + condition = condition.Replace(d, $"{TABLE_ALIAS}.{d}"); + } keyFilterQ = new string[] { condition }; allFiltersQuery = allFiltersQuery.Concat(keyFilterQ); diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs index c827fd650..c94723a42 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Text.Json; using System.Threading.Tasks; +using GeneXus.Configuration; using GeneXus.Data.NTier; using GeneXus.Data.NTier.CosmosDB; using log4net; @@ -234,7 +235,8 @@ public bool NextResult() } private async Task GetPage() - { + { + //Config.LoadConfiguration(); while (feedIterator.HasMoreResults) { try From 0a90ac8796636c219136fa62d8dc03cb8d556eaf Mon Sep 17 00:00:00 2001 From: cmurialdo Date: Fri, 16 Dec 2022 11:10:22 -0300 Subject: [PATCH 07/25] Add null reference checks and follow fortify rules. --- .../src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs | 5 ++++- .../src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs index 4c58415d6..2fb73a219 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs @@ -117,6 +117,9 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo { Initialize(); CosmosDBQuery query = cursorDef.Query as CosmosDBQuery; + if (query == null) + return 0; + bool isInsert = query.CursorType == ServiceCursorDef.CursorType.Insert; bool isUpdate = query.CursorType == ServiceCursorDef.CursorType.Update; @@ -319,7 +322,7 @@ public override IDataReader ExecuteReader(ServiceCursorDef cursorDef, IDataParam Initialize(); CosmosDBQuery query = cursorDef.Query as CosmosDBQuery; //Get container from hashset for performance - Container container = GetContainer(query.TableName); + Container container = GetContainer(query?.TableName); try { CreateCosmosQuery(query,cursorDef, parms, container, out CosmosDBDataReader dataReader, out RequestWrapper requestWrapper); diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs index c94723a42..b09295f4f 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs @@ -35,7 +35,8 @@ private void CheckCurrentPosition() public CosmosDBDataReader(ServiceCursorDef cursorDef, RequestWrapper request) { Query query = cursorDef.Query as Query; - selectList = query.SelectList.ToArray(); + if (query != null) + selectList = query.SelectList.ToArray(); mRequest = request; mResponse = mRequest.Read(); feedIterator = mResponse.feedIterator; From cc5bf4a2656bbfa24ab7c9361f62b68973b683c2 Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Sat, 17 Dec 2022 17:42:09 -0300 Subject: [PATCH 08/25] Support order --- .../DynService.Cosmos/CosmosDBConnection.cs | 64 ++++++++++++++++--- 1 file changed, 55 insertions(+), 9 deletions(-) diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs index 4c58415d6..b6608ac61 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs @@ -42,10 +42,11 @@ public class CosmosDBConnection : ServiceConnection private static string mConnectionString; //https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/query/select - private const string SELECT_CMD = "SELECT"; - private const string FROM = "FROM"; + //private const string SELECT_CMD = "SELECT"; + //private const string FROM = "FROM"; private const string TABLE_ALIAS = "t"; - private const string WHERE = "WHERE"; + //private const string WHERE = "WHERE"; + private const string FILTER_PATTERN = @"\((.*) = :(.*)\)"; @@ -99,6 +100,7 @@ private static void InitializeDBConnection() private static void Initialize() { + System.Diagnostics.Debugger.Launch(); if (!string.IsNullOrEmpty(mserviceURI) && !string.IsNullOrEmpty(mapplicationRegion)) cosmosClient = new CosmosClient(mserviceURI, new CosmosClientOptions() { ApplicationRegion = mapplicationRegion }); @@ -112,7 +114,38 @@ private Container GetContainer(string containerName) return cosmosClient.GetContainer(cosmosDatabase.Id, containerName); return null; } + private string SetupQuery(string projectionList, string filterExpression, string tableName, string orderbys) + { + string sqlSelect = string.Empty; + string sqlFrom = string.Empty; + string sqlWhere = string.Empty; + string sqlOrder = string.Empty; + + string SELECT_TEMPLATE = "select {0}"; + string FROM_TEMPLATE = "from {0} t"; + string WHERE_TEMPLATE = "where {0}"; + string ORDER_TEMPLATE = "order by {0}"; + + if (!string.IsNullOrEmpty(projectionList)) + sqlSelect = string.Format(SELECT_TEMPLATE, projectionList); + else + { //ERROR + + } + if (!string.IsNullOrEmpty(tableName)) + sqlFrom = string.Format(FROM_TEMPLATE, tableName); + else + { + //ERROR + } + if (!string.IsNullOrEmpty(filterExpression)) + sqlWhere = string.Format(WHERE_TEMPLATE, filterExpression); + if (!string.IsNullOrEmpty(orderbys)) + sqlOrder = string.Format(ORDER_TEMPLATE, orderbys); + + return $"{sqlSelect} {sqlFrom} {sqlWhere} {sqlOrder}"; + } public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCollection parms, CommandBehavior behavior) { Initialize(); @@ -441,13 +474,26 @@ private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, I allFiltersQuery = allFiltersQuery.Concat(keyFilterQ); } - string FilterExpression = allFiltersQuery.Any() ? String.Join(" AND ", allFiltersQuery) : null; + string filterExpression = allFiltersQuery.Any() ? String.Join(" AND ", allFiltersQuery) : null; - string sqlQuery = string.Empty; - if (FilterExpression != null) - sqlQuery = $"{SELECT_CMD} {projectionList} {FROM} {tableName} {TABLE_ALIAS} {WHERE} {FilterExpression}"; - else - sqlQuery = $"{SELECT_CMD} {projectionList} {FROM} {tableName} {TABLE_ALIAS}"; + IEnumerable orderExpressionList = Array.Empty(); + string expression = string.Empty; + + foreach (string orderAtt in query.OrderBys) + { + + expression = orderAtt.StartsWith("(") ? $"{TABLE_ALIAS}.{orderAtt} DESC" : $"{TABLE_ALIAS}.{orderAtt} ASC"; + orderExpressionList = orderExpressionList.Concat(new string[] { expression}); + } + + string orderExpression = String.Join(",", orderExpressionList); + + string sqlQuery = SetupQuery(projectionList, filterExpression, tableName, orderExpression); + + //if (FilterExpression != null) + //sqlQuery = $"{SELECT_CMD} {projectionList} {FROM} {tableName} {TABLE_ALIAS} {WHERE} {FilterExpression}"; + //else + //sqlQuery = $"{SELECT_CMD} {projectionList} {FROM} {tableName} {TABLE_ALIAS}"; QueryDefinition queryDefinition = new QueryDefinition(sqlQuery); requestWrapper = new RequestWrapper(cosmosClient, container, queryDefinition); From 8c4f4d0ed5b8495059724165800ae8a18176a7b5 Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Sat, 17 Dec 2022 17:45:07 -0300 Subject: [PATCH 09/25] remove line --- dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs index 6f11cc891..efdb51e23 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs @@ -100,7 +100,6 @@ private static void InitializeDBConnection() private static void Initialize() { - System.Diagnostics.Debugger.Launch(); if (!string.IsNullOrEmpty(mserviceURI) && !string.IsNullOrEmpty(mapplicationRegion)) cosmosClient = new CosmosClient(mserviceURI, new CosmosClientOptions() { ApplicationRegion = mapplicationRegion }); From 622a3e045cfef63a6617860fb183500d9552415a Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Sat, 17 Dec 2022 17:53:00 -0300 Subject: [PATCH 10/25] remove comments --- .../DynService.Dynamo/DynamoDBConnection.cs | 1 - .../DynService.Dynamo/DynamoDBDataStoreHelper.cs | 16 ---------------- 2 files changed, 17 deletions(-) diff --git a/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBConnection.cs b/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBConnection.cs index f3b2acf9d..6eeacb090 100644 --- a/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBConnection.cs +++ b/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBConnection.cs @@ -75,7 +75,6 @@ private void Initialize() { InitializeDBConnection(); State = ConnectionState.Executing; - mDynamoDB = new AmazonDynamoDBClient(mCredentials, mConfig); } diff --git a/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBDataStoreHelper.cs b/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBDataStoreHelper.cs index 86c8c21c3..d28a2ddd1 100644 --- a/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBDataStoreHelper.cs +++ b/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBDataStoreHelper.cs @@ -55,22 +55,6 @@ public object empty(GXType gxtype) } } } - - //public static class DynamoFluentExtensions - //{ - //public static Query OrderBy(this Query dynamoQuery, string index) - //{ - // return (dynamoQuery as DynamoQuery)?.OrderBy(index); - //} - /*public static Query SetKey(this Query dynamoQuery, string partitionKey) - { - return (dynamoQuery.SetKey(partitionKey)); - }*/ - //public static Query KeyFilter(this Query dynamoQuery, string[] filters) - //{ - //return (dynamoQuery as DynamoQuery)?.KeyFilter(filters); - //} - //} public class DynamoQuery : Query { From 7842fd89a8c88509111df1e59d87202b08218618 Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Sat, 17 Dec 2022 22:22:40 -0300 Subject: [PATCH 11/25] fix replacement of qualified atts in query --- .../src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs index efdb51e23..ad06f5a96 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs @@ -467,10 +467,11 @@ private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, I } } } - + foreach (string d in projection) { - condition = condition.Replace(d, $"{TABLE_ALIAS}.{d}"); + string wholeWordPattern = String.Format(@"\b{0}\b", d); + condition = Regex.Replace(condition, wholeWordPattern, $"{TABLE_ALIAS}.{d}"); } keyFilterQ = new string[] { condition }; allFiltersQuery = allFiltersQuery.Concat(keyFilterQ); From bba9e671432e40631214922fadc4b16550d5acc1 Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Mon, 19 Dec 2022 12:47:34 -0300 Subject: [PATCH 12/25] add empty method used by automatic prompts --- .../CosmosDBDatastoreHelper.cs | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDatastoreHelper.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDatastoreHelper.cs index e2643f9e4..d38766105 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDatastoreHelper.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDatastoreHelper.cs @@ -4,6 +4,7 @@ using System.Text; using System.Threading.Tasks; using GeneXus.Data.Cosmos; +using GeneXus.Utils; namespace GeneXus.Data.NTier { @@ -19,7 +20,44 @@ public CosmosDBMap Map(string name) { return new CosmosDBMap(name); } + public object empty(GXType gxtype) + { + switch (gxtype) + { + case GXType.Number: + case GXType.Int16: + case GXType.Int32: + case GXType.Int64: return 0; + case GXType.Date: + case GXType.DateTime: + case GXType.DateTime2: return DateTimeUtil.NullDate(); + case GXType.Byte: + case GXType.NChar: + case GXType.NClob: + case GXType.NVarChar: + case GXType.Char: + case GXType.LongVarChar: + case GXType.Clob: + case GXType.VarChar: + case GXType.Raw: + case GXType.Blob: return string.Empty; + case GXType.Boolean: return false; + case GXType.Decimal: return 0f; + case GXType.NText: + case GXType.Text: + case GXType.Image: + case GXType.UniqueIdentifier: + case GXType.Xml: return string.Empty; + case GXType.Geography: + case GXType.Geopoint: + case GXType.Geoline: + case GXType.Geopolygon: return new Geospatial(); + case GXType.DateAsChar: return string.Empty; + case GXType.Undefined: + default: return null; + } + } } public class CosmosDBQuery : Query { From fd809edd539b0dfa996224b63842f5db1a5ec098 Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Wed, 21 Dec 2022 21:22:01 -0300 Subject: [PATCH 13/25] Use more performant method when querying by PK --- .../DynService.Cosmos/CosmosDBConnection.cs | 102 ++++++++++++------ .../DynService.Cosmos/CosmosDBDataReader.cs | 92 ++++++++++------ .../CosmosDBDatastoreHelper.cs | 6 -- .../DynService.Cosmos/CosmosDBHelper.cs | 2 - .../CosmosDBRequestWrapper.cs | 58 +++++----- .../CosmosDBResponseWrapper.cs | 12 +-- 6 files changed, 163 insertions(+), 109 deletions(-) diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs index ad06f5a96..bb495cd02 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs @@ -42,17 +42,11 @@ public class CosmosDBConnection : ServiceConnection private static string mConnectionString; //https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/query/select - //private const string SELECT_CMD = "SELECT"; - //private const string FROM = "FROM"; private const string TABLE_ALIAS = "t"; - //private const string WHERE = "WHERE"; - - private const string FILTER_PATTERN = @"\((.*) = :(.*)\)"; //private const string DISTINCT = "DISTINCT"; - //private const string ORDER_BY = "ORDER BY"; - + //static readonly ILog logger = log4net.LogManager.GetLogger(typeof(CosmosDBConnection)); //TODO: Usar un Hashset para guardar los containers @@ -73,7 +67,6 @@ public override string ConnectionString } private static void InitializeDBConnection() { - DbConnectionStringBuilder builder = new DbConnectionStringBuilder(false); builder.ConnectionString = mConnectionString; @@ -368,12 +361,66 @@ public override IDataReader ExecuteReader(ServiceCursorDef cursorDef, IDataParam catch (Exception e) { throw e; } } + + private VarValue GetDataParameterfromQueryVars(string filter, IEnumerable values) + { + VarValue varValue = null; + Match match = Regex.Match(filter, FILTER_PATTERN); + if (match.Groups.Count > 1) + { + string varName = match.Groups[2].Value; + string name = match.Groups[1].Value; + varValue = values.FirstOrDefault(v => v.Name == $":{varName}"); + } + return varValue; + } + + private string GetDataParameterDataParameterCollection(string filter, IDataParameterCollection parms) + { + Match match = Regex.Match(filter, FILTER_PATTERN); + if (match.Groups.Count > 1) + { + string varName = match.Groups[2].Value; + + if (parms[varName] is ServiceParameter serviceParm) + { + return serviceParm.Value.ToString(); + } + } + return string.Empty; + } private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, IDataParameterCollection parms, Container container, out CosmosDBDataReader cosmosDBDataReader,out RequestWrapper requestWrapper) { + + if (query.KeyFilters.Any() && (!query.Filters.Any())) + { + //Key is just the id or + requestWrapper = new RequestWrapper(cosmosClient, container, null); + requestWrapper.queryByPK = true; + requestWrapper.idValue = GetDataParameterfromQueryVars(query.KeyFilters.First(), query.Vars)?.Value.ToString(); + requestWrapper.idValue = requestWrapper.idValue ?? GetDataParameterDataParameterCollection(query.KeyFilters.First(),parms); + + if (requestWrapper.idValue != null) + { + if (query.KeyFilters.Count() > 1) + { + requestWrapper.partitionKeyValue = GetDataParameterfromQueryVars(query.KeyFilters.Skip(1).Take(1).First(), query.Vars)?.Value.ToString(); + requestWrapper.partitionKeyValue = requestWrapper.partitionKeyValue ?? GetDataParameterDataParameterCollection(query.KeyFilters.Skip(1).Take(1).First(), parms); + } + else + requestWrapper.partitionKeyValue = requestWrapper.idValue; + + if (requestWrapper.partitionKeyValue != null) + { + cosmosDBDataReader = new CosmosDBDataReader(cursorDef, requestWrapper); + return; + } + } + } + //Create the query string tableName = query.TableName; IEnumerable projection = query.Projection; - string element; string projectionList = string.Empty; foreach (string key in projection) @@ -390,18 +437,18 @@ private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, I string regex1 = @"\(([^\)\(]+)\)"; string regex2 = @"(.*)[^<>!=]\s*(=|!=|<|>|<=|>=|<>)\s*(:.*)"; - + string keyFilterS; string condition = string.Empty; IEnumerable keyFilterQ = Array.Empty(); - foreach (string keyFilter in allFilters) + foreach (string keyFilter in allFilters) { keyFilterS = keyFilter; condition = keyFilter; - + MatchCollection matchCollection = Regex.Matches(keyFilterS, regex1); - + foreach (Match match in matchCollection) { if (match.Groups.Count > 0) @@ -424,8 +471,6 @@ private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, I //look at IDataParameterCollection parms if (parms[attName] is ServiceParameter serviceParm) if (GeneXus.Data.Cosmos.CosmosDBHelper.FormattedAsStringDbType(serviceParm.DbType)) - //Nvarchar, etc? - //Date? { varValuestr = '"' + $"{serviceParm.Value.ToString()}" + '"'; } @@ -438,15 +483,13 @@ private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, I { if (item.Name == match2.Groups[3].Value) { - //if (item.Type == GXType.Char || item.Type == GXType.LongVarChar || item.Type == GXType.VarChar || item.Type == GXType.Text) + if (GeneXus.Data.Cosmos.CosmosDBHelper.FormattedAsStringGXType(item.Type)) - //Nvarchar, etc? - //Date? { varValuestr = '"' + $"{item.Value.ToString()}" + '"'; } else - { + { varValuestr = item.Value.ToString(); varValuestr = varValuestr.Equals("True") ? "true" : varValuestr; varValuestr = varValuestr.Equals("False") ? "false" : varValuestr; @@ -462,12 +505,11 @@ private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, I else { //Check that cond is a valid attribute - boolean - } } } } - + foreach (string d in projection) { string wholeWordPattern = String.Format(@"\b{0}\b", d); @@ -475,31 +517,25 @@ private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, I } keyFilterQ = new string[] { condition }; allFiltersQuery = allFiltersQuery.Concat(keyFilterQ); - + } string filterExpression = allFiltersQuery.Any() ? String.Join(" AND ", allFiltersQuery) : null; IEnumerable orderExpressionList = Array.Empty(); string expression = string.Empty; - + foreach (string orderAtt in query.OrderBys) { - - expression = orderAtt.StartsWith("(") ? $"{TABLE_ALIAS}.{orderAtt} DESC" : $"{TABLE_ALIAS}.{orderAtt} ASC"; - orderExpressionList = orderExpressionList.Concat(new string[] { expression}); + expression = orderAtt.StartsWith("(") ? $"{TABLE_ALIAS}.{orderAtt.Remove(orderAtt.Length-1,1).Remove(0,1)} DESC" : $"{TABLE_ALIAS}.{orderAtt} ASC"; + orderExpressionList = orderExpressionList.Concat(new string[] { expression }); } string orderExpression = String.Join(",", orderExpressionList); - string sqlQuery = SetupQuery(projectionList, filterExpression, tableName, orderExpression); - - //if (FilterExpression != null) - //sqlQuery = $"{SELECT_CMD} {projectionList} {FROM} {tableName} {TABLE_ALIAS} {WHERE} {FilterExpression}"; - //else - //sqlQuery = $"{SELECT_CMD} {projectionList} {FROM} {tableName} {TABLE_ALIAS}"; - QueryDefinition queryDefinition = new QueryDefinition(sqlQuery); requestWrapper = new RequestWrapper(cosmosClient, container, queryDefinition); + requestWrapper.queryByPK = false; + cosmosDBDataReader = new CosmosDBDataReader(cursorDef, requestWrapper); } internal static IOServiceContext NewServiceContext() => null; diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs index b09295f4f..43d948eba 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs @@ -16,12 +16,12 @@ namespace GeneXus.Data.Cosmos { public class CosmosDBDataReader : IDataReader { - private readonly RequestWrapper mRequest; - private ResponseWrapper mResponse; - private readonly IODataMap2[] selectList; - private FeedIterator feedIterator; - private CosmosDBRecordEntry currentEntry; - private int mCurrentPosition; + private readonly RequestWrapper m_request; + private ResponseWrapper m_response; + private readonly IODataMap2[] m_selectList; + private FeedIterator m_feedIterator; + private CosmosDBRecordEntry m_currentEntry; + private int m_currentPosition; private int ItemCount; private List> Items = null; @@ -29,18 +29,44 @@ public class CosmosDBDataReader : IDataReader static readonly ILog logger = log4net.LogManager.GetLogger(typeof(CosmosDBDataReader)); private void CheckCurrentPosition() { - if (currentEntry == null) + if (m_currentEntry == null) throw new ServiceException(ServiceError.RecordNotFound); } + private void ProcessStream(Stream stream) + { + if (stream != null) + { + using (StreamReader sr = new StreamReader(stream)) + using (JsonTextReader jtr = new JsonTextReader(sr)) + { + Newtonsoft.Json.JsonSerializer jsonSerializer = new Newtonsoft.Json.JsonSerializer(); + object array = jsonSerializer.Deserialize(jtr); + + string json = ((Newtonsoft.Json.Linq.JToken)array).Root.ToString(); + var jsonDocument = JsonDocument.Parse(json); + var jsonDoc = jsonDocument.RootElement; + foreach (var jsonProperty in jsonDoc.EnumerateObject()) + { + if (jsonProperty.Name == "Documents") + { + Items = JsonConvert.DeserializeObject>>(jsonProperty.Value.ToString()); + break; + } + } + ItemCount = Items.Count; + } + } + } public CosmosDBDataReader(ServiceCursorDef cursorDef, RequestWrapper request) { Query query = cursorDef.Query as Query; if (query != null) - selectList = query.SelectList.ToArray(); - mRequest = request; - mResponse = mRequest.Read(); - feedIterator = mResponse.feedIterator; - //mCurrentPosition = -1; + m_selectList = query.SelectList.ToArray(); + m_request = request; + m_response = m_request.Read(); + m_feedIterator = m_response.feedIterator; + if (m_feedIterator == null) + ProcessStream(m_response.stream); } public object this[string name] { @@ -69,7 +95,7 @@ public int FieldCount { get { - return selectList.Length; + return m_selectList.Length; } } public bool IsClosed @@ -90,9 +116,9 @@ public int RecordsAffected public void Close() { - if (mRequest != null) + if (m_request != null) { - mRequest.Close(); + m_request.Close(); } } public void Dispose() @@ -105,7 +131,7 @@ public long getLong(int i) private object GetAttValue(int i) { - return (selectList[i].GetValue(CosmosDBConnection.NewServiceContext(), currentEntry)); + return (m_selectList[i].GetValue(CosmosDBConnection.NewServiceContext(), m_currentEntry)); } public bool GetBoolean(int i) { @@ -166,7 +192,7 @@ public double GetDouble(int i) } public Type GetFieldType(int i) { - return selectList[i].GetValue(CosmosDBConnection.NewServiceContext(), currentEntry).GetType(); + return m_selectList[i].GetValue(CosmosDBConnection.NewServiceContext(), m_currentEntry).GetType(); } public float GetFloat(int i) @@ -195,13 +221,13 @@ public long GetInt64(int i) public string GetName(int i) { - return selectList[i].GetName(null); + return m_selectList[i].GetName(null); } public int GetOrdinal(string name) { CheckCurrentPosition(); - int ordinal = currentEntry.CurrentRow.ToList().FindIndex(col => col.Key.ToLower() == name.ToLower()); + int ordinal = m_currentEntry.CurrentRow.ToList().FindIndex(col => col.Key.ToLower() == name.ToLower()); if (ordinal == -1) throw new ArgumentOutOfRangeException(nameof(name)); else return ordinal; @@ -216,12 +242,12 @@ public string GetString(int i) } public int GetValues(object[] values) { - System.Diagnostics.Debug.Assert(selectList.Length == values.Length, "Values mismatch"); - for (int i = 0; i < selectList.Length && i < values.Length; i++) + System.Diagnostics.Debug.Assert(m_selectList.Length == values.Length, "Values mismatch"); + for (int i = 0; i < m_selectList.Length && i < values.Length; i++) { values[i] = GetAttValue(i); } - return selectList.Length; + return m_selectList.Length; } public bool IsDBNull(int i) { @@ -230,26 +256,25 @@ public bool IsDBNull(int i) public bool NextResult() { - mCurrentPosition++; - currentEntry = (mCurrentPosition < ItemCount) ? new CosmosDBRecordEntry(Items[mCurrentPosition]) : null; - return currentEntry != null; + m_currentPosition++; + m_currentEntry = (m_currentPosition < ItemCount) ? new CosmosDBRecordEntry(Items[m_currentPosition]) : null; + return m_currentEntry != null; } private async Task GetPage() { - //Config.LoadConfiguration(); - while (feedIterator.HasMoreResults) + while (m_feedIterator.HasMoreResults) { try { - using (ResponseMessage response = await feedIterator.ReadNextAsync().ConfigureAwait(false)) + using (ResponseMessage response = await m_feedIterator.ReadNextAsync().ConfigureAwait(false)) { if (!response.IsSuccessStatusCode) { if (response.Diagnostics != null) GXLogging.Debug(logger, $"Read ItemStreamFeed Diagnostics: {response.Diagnostics.ToString()}"); - throw new Exception(GeneXus.Data.Cosmos.CosmosDBHelper.FormatExceptionMessage(response.StatusCode.ToString(),response.ErrorMessage)); + throw new Exception(GeneXus.Data.Cosmos.CosmosDBHelper.FormatExceptionMessage(response.StatusCode.ToString(),response.ErrorMessage)); } else { @@ -290,12 +315,15 @@ public bool Read() if (NextResult()) return true; else - task = Task.Run(async () => await GetPage().ConfigureAwait(false)); - if (task.Result) + if (m_feedIterator != null) + { + task = Task.Run(async () => await GetPage().ConfigureAwait(false)); + if (task.Result) { - mCurrentPosition = -1; + m_currentPosition = -1; return NextResult(); } + } return false; } diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDatastoreHelper.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDatastoreHelper.cs index d38766105..6430baf2b 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDatastoreHelper.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDatastoreHelper.cs @@ -1,8 +1,5 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using GeneXus.Data.Cosmos; using GeneXus.Utils; @@ -11,11 +8,8 @@ namespace GeneXus.Data.NTier public class CosmosDBDatastoreHelper : DynServiceDataStoreHelperBase { - //TODO Esto no aplica a CosmosDB public CosmosDBQuery NewQuery() => new CosmosDBQuery(this); - public CosmosDBQuery NewScan() => new CosmosDBQuery(this); - public CosmosDBMap Map(string name) { return new CosmosDBMap(name); diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs index 42b271aa3..8a7deff9d 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs @@ -4,7 +4,6 @@ using System.IO; using System.Linq; using System.Text; -using System.Threading.Tasks; using GeneXus.Data.NTier; namespace GeneXus.Data.Cosmos @@ -96,7 +95,6 @@ internal static bool AddItemValue(string parmName, Dictionary dy } return false; } - internal static object ToItemValue(DbType dbType, Object value) { object attValue; diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBRequestWrapper.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBRequestWrapper.cs index 46b258b68..ff9b3675c 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBRequestWrapper.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBRequestWrapper.cs @@ -1,63 +1,63 @@ using System; -using System.IO; using System.Threading.Tasks; +using log4net; using Microsoft.Azure.Cosmos; -using Newtonsoft.Json; namespace GeneXus.Data.NTier.CosmosDB { public class RequestWrapper { - private readonly Container mcontainer; - private readonly CosmosClient mcosmosClient; - private readonly QueryDefinition mqueryDefinition; - //wrapper string idValue, string partitionKeyValue + private readonly Container m_container; + private readonly CosmosClient m_cosmosClient; + private readonly QueryDefinition m_queryDefinition; + static readonly ILog logger = log4net.LogManager.GetLogger(typeof(RequestWrapper)); + public string idValue { get; set; } + public string partitionKeyValue { get; set; } + public bool queryByPK { get; set; } public RequestWrapper(CosmosClient cosmosClient, Container container, QueryDefinition queryDefinition) { - mcontainer = container; - mcosmosClient = cosmosClient; - mqueryDefinition = queryDefinition; + m_container = container; + m_cosmosClient = cosmosClient; + m_queryDefinition = queryDefinition; } - //TODO: Por PK usar este metodo que es mas performante - /*private async Task ReadItemAsyncByPK(string idValue, string partitionKeyValue) + private async Task ReadItemAsyncByPK(string idValue, string partitionKeyValue) { - using (ResponseMessage responseMessage = await mcontainer.ReadItemStreamAsync( + using (ResponseMessage responseMessage = await m_container.ReadItemStreamAsync( partitionKey: new PartitionKey(partitionKeyValue), id: idValue).ConfigureAwait(false)) { - - if (responseMessage.IsSuccessStatusCode) - { - return null; - // Log the diagnostics - //Console.WriteLine($"\n1.2.2 - Item Read Diagnostics: {responseMessage.Diagnostics.ToString()}"); - } - else + + if (!responseMessage.IsSuccessStatusCode) { - return new ResponseWrapper(responseMessage); - //Console.WriteLine($"Read item from stream failed. Status code: {responseMessage.StatusCode} Message: {responseMessage.ErrorMessage}"); + if (!responseMessage.ErrorMessage.Contains("404")) + { + if (responseMessage.Diagnostics != null) + GXLogging.Debug(logger, $"Read ReadItemAsyncByPK Diagnostics: {responseMessage.Diagnostics.ToString()}"); + throw new Exception(GeneXus.Data.Cosmos.CosmosDBHelper.FormatExceptionMessage(responseMessage.StatusCode.ToString(), responseMessage.ErrorMessage)); + } } + return new ResponseWrapper(responseMessage); } - }*/ - public ResponseWrapper Read() + } + public ResponseWrapper Read() { - /* if (queryByPK) + if (queryByPK) { Task task = Task.Run(async () => await ReadItemAsyncByPK(idValue, partitionKeyValue).ConfigureAwait(false)); return task.Result; - }*/ - //GetItemQueryStreamIterator + } + QueryRequestOptions requestOptions = new QueryRequestOptions() { MaxBufferedItemCount = 100 }; //options.MaxConcurrency = 1; //TODO Cancelation Token + request options - using (FeedIterator feedIterator = mcontainer.GetItemQueryStreamIterator(mqueryDefinition, null, requestOptions)) + using (FeedIterator feedIterator = m_container.GetItemQueryStreamIterator(m_queryDefinition, null, requestOptions)) return new ResponseWrapper(feedIterator); } internal void Close() { - mcosmosClient.Dispose(); + m_cosmosClient.Dispose(); } } } diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBResponseWrapper.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBResponseWrapper.cs index 67114e09c..62ae6849a 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBResponseWrapper.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBResponseWrapper.cs @@ -8,23 +8,21 @@ public class ResponseWrapper { public FeedIterator feedIterator; - Stream stream; + public Stream stream; public ResponseWrapper(ResponseMessage responseMessage, FeedIterator feedIter) { feedIterator = feedIter; stream = responseMessage.Content; - //Items = queryResponse.Items; - //ItemCount = queryResponse.Items.Count; } public ResponseWrapper(FeedIterator feedIter) { feedIterator = feedIter; - //stream = responseMessage.Content; - //Items = queryResponse.Items; - //ItemCount = queryResponse.Items.Count; } - + public ResponseWrapper(ResponseMessage responseMessage) + { + stream = responseMessage.Content; + } public List> Items { get; set; } public int ItemCount { get; set; } From 3adce9058537f9a1c5b7e21a6d23273ce1ec4f02 Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Thu, 29 Dec 2022 14:52:44 -0300 Subject: [PATCH 14/25] Fixes for case when querying by PK. --- .../DynService.Cosmos/CosmosDBConnection.cs | 466 ++++++++++-------- .../DynService.Cosmos/CosmosDBDataReader.cs | 46 +- .../DynService.Cosmos/CosmosDBHelper.cs | 5 +- .../CosmosDBRequestWrapper.cs | 41 +- .../CosmosDBResponseWrapper.cs | 10 +- 5 files changed, 345 insertions(+), 223 deletions(-) diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs index bb495cd02..a6a39e413 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs @@ -43,11 +43,11 @@ public class CosmosDBConnection : ServiceConnection //https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/query/select private const string TABLE_ALIAS = "t"; - private const string FILTER_PATTERN = @"\((.*) = :(.*)\)"; - + + //Options not supported by the spec yet //private const string DISTINCT = "DISTINCT"; - //static readonly ILog logger = log4net.LogManager.GetLogger(typeof(CosmosDBConnection)); + static readonly ILog logger = log4net.LogManager.GetLogger(typeof(CosmosDBConnection)); //TODO: Usar un Hashset para guardar los containers @@ -67,6 +67,7 @@ public override string ConnectionString } private static void InitializeDBConnection() { + //System.Diagnostics.Debugger.Launch(); DbConnectionStringBuilder builder = new DbConnectionStringBuilder(false); builder.ConnectionString = mConnectionString; @@ -138,6 +139,16 @@ private string SetupQuery(string projectionList, string filterExpression, string return $"{sqlSelect} {sqlFrom} {sqlWhere} {sqlOrder}"; } + + /// + /// Execute insert, update and delete queries. + /// + /// + /// + /// + /// + /// + /// public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCollection parms, CommandBehavior behavior) { Initialize(); @@ -153,77 +164,100 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo string partitionKey = query.PartitionKey; string partitionKeyValue = string.Empty; + + //Setup the json payload to execute the insert or update query. foreach (KeyValuePair asg in query.AssignAtts) { string name = asg.Key; - string parmName = asg.Value.Substring(1); + string parmName = asg.Value.Substring(1).Remove(asg.Value.Length - 2); CosmosDBHelper.AddItemValue(name, parmName, values, parms, query.Vars, ref jsonData); if (name == partitionKey) partitionKeyValue = values[name].ToString(); } - + Dictionary keyCondition = new Dictionary(); + //Get the values for id and partitionKey + + string regex1 = @"\(([^\)\(]+)\)"; + string regex2 = @"(.*)[^<>!=]\s*(=|!=|<|>|<=|>=|<>)\s*(:.*:)"; + + string keyFilterS; + string condition = string.Empty; + IEnumerable keyFilterQ = Array.Empty(); - foreach (string keyFilter in query.KeyFilters.Concat(query.Filters)) + IEnumerable allFilters = query.KeyFilters.Concat(query.Filters); + + foreach (string keyFilter in allFilters) { - Match match = Regex.Match(keyFilter, FILTER_PATTERN); - if (match.Groups.Count > 1) + keyFilterS = keyFilter; + condition = keyFilter; + + MatchCollection matchCollection = Regex.Matches(keyFilterS, regex1); + + foreach (Match match in matchCollection) { - string varName = match.Groups[2].Value; - string name = match.Groups[1].Value; - VarValue varValue = query.Vars.FirstOrDefault(v => v.Name == $":{varName}"); - - string jsonDataKey = String.Empty; - string jsonDataPartitionKey = string.Empty; - if (varValue != null) - { - keyCondition[name] = varValue.Value; - //keyCondition[name] = GeneXus.Data.Cosmos.CosmosDBHelper.ToItemValue(varValue.Type, varValue.Value); - - if (isUpdate && name == "id") - jsonDataKey = GeneXus.Data.Cosmos.CosmosDBHelper.AddToJsonStream(varValue.Type, name, varValue.Value); - if (isUpdate && name == partitionKey) - jsonDataPartitionKey = GeneXus.Data.Cosmos.CosmosDBHelper.AddToJsonStream(varValue.Type, name, varValue.Value); - - if (name == partitionKey) - //TODO Partition Key can be numeric - partitionKeyValue = varValue.Value.ToString(); - } - else - { - if (parms[varName] is ServiceParameter serviceParm) - { - keyCondition[name] = serviceParm.Value; - //keyCondition[name] = GeneXus.Data.Cosmos.CosmosDBHelper.ToItemValue(serviceParm.DbType, serviceParm.Value); - - if (isUpdate && name == "id") - jsonDataKey = GeneXus.Data.Cosmos.CosmosDBHelper.AddToJsonStream(serviceParm.DbType, name, serviceParm.Value); - if (isUpdate && name == partitionKey) - jsonDataPartitionKey = GeneXus.Data.Cosmos.CosmosDBHelper.AddToJsonStream(serviceParm.DbType, name, serviceParm.Value); - - if (name == partitionKey) - //TODO Partition Key can be numeric - partitionKeyValue = serviceParm.Value.ToString(); - } - } - if (!string.IsNullOrEmpty(jsonDataKey)) + if (match.Groups.Count > 0) { - if (!string.IsNullOrEmpty(jsonData)) + string cond = match.Groups[1].Value; + Match match2 = Regex.Match(cond, regex2); + if (match2.Success) { - jsonData = $"{jsonData},{jsonDataKey}"; - } - else - jsonData = jsonDataKey; - } + string varName = match2.Groups[3].Value; + varName = varName.Remove(varName.Length - 1).Substring(1); + string name = match2.Groups[1].Value; + VarValue varValue = query.Vars.FirstOrDefault(v => v.Name == $":{varName}"); + + string jsonDataKey = String.Empty; + string jsonDataPartitionKey = string.Empty; + if (varValue != null) + { + keyCondition[name] = varValue.Value; + //keyCondition[name] = GeneXus.Data.Cosmos.CosmosDBHelper.ToItemValue(varValue.Type, varValue.Value); - if (!string.IsNullOrEmpty(jsonDataPartitionKey)) - { - if (!string.IsNullOrEmpty(jsonData)) - jsonData = $"{jsonData},{jsonDataPartitionKey}"; - else - jsonData = jsonDataPartitionKey; - } + if (isUpdate && name == "id") + jsonDataKey = GeneXus.Data.Cosmos.CosmosDBHelper.AddToJsonStream(varValue.Type, name, varValue.Value); + if (isUpdate && name == partitionKey) + jsonDataPartitionKey = GeneXus.Data.Cosmos.CosmosDBHelper.AddToJsonStream(varValue.Type, name, varValue.Value); + + if (name == partitionKey) + //TODO Partition Key can be double, bool + partitionKeyValue = varValue.Value.ToString(); + } + else + { + if (parms[varName] is ServiceParameter serviceParm) + { + keyCondition[name] = serviceParm.Value; + //keyCondition[name] = GeneXus.Data.Cosmos.CosmosDBHelper.ToItemValue(serviceParm.DbType, serviceParm.Value); + + if (isUpdate && name == "id") + jsonDataKey = GeneXus.Data.Cosmos.CosmosDBHelper.AddToJsonStream(serviceParm.DbType, name, serviceParm.Value); + if (isUpdate && name == partitionKey) + jsonDataPartitionKey = GeneXus.Data.Cosmos.CosmosDBHelper.AddToJsonStream(serviceParm.DbType, name, serviceParm.Value); + + if (name == partitionKey) + //TODO Partition Key can be numeric + partitionKeyValue = serviceParm.Value.ToString(); + } + } + if (!string.IsNullOrEmpty(jsonDataKey)) + { + if (!string.IsNullOrEmpty(jsonData)) + jsonData = $"{jsonData},{jsonDataKey}"; + else + jsonData = jsonDataKey; + } + + if (!string.IsNullOrEmpty(jsonDataPartitionKey)) + { + if (!string.IsNullOrEmpty(jsonData)) + jsonData = $"{jsonData},{jsonDataPartitionKey}"; + else + jsonData = jsonDataPartitionKey; + } + } + } } } @@ -233,7 +267,6 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo Container container = GetContainer(query.TableName); switch (query.CursorType) { - case ServiceCursorDef.CursorType.Select: throw new NotImplementedException(); @@ -243,11 +276,17 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo { try { - if (keyCondition != null && keyCondition["id"] != null) + if (!keyCondition.Any() || !keyCondition.ContainsKey("id") || !keyCondition.ContainsKey(partitionKey)) + { + logger.Error($"Delete item failed: error parsing the query."); + throw new Exception($"Delete item failed: error parsing the query."); + } + else { - if (string.IsNullOrEmpty(partitionKeyValue)) - partitionKeyValue = keyCondition["id"].ToString(); // partitionKeyValue = id - Task task = Task.Run(async () => await container.DeleteItemStreamAsync(keyCondition["id"].ToString(), new PartitionKey(partitionKeyValue)).ConfigureAwait(false)); + object idField = keyCondition["id"]; + + logger.Debug($"Delete : id= {idField.ToString()}, partitionKey= {partitionKeyValue}"); + Task task = Task.Run(async () => await container.DeleteItemStreamAsync(idField.ToString(), new PartitionKey(partitionKeyValue)).ConfigureAwait(false)); if (task.Result.IsSuccessStatusCode) { //ResponseMessage wrapps the delete record @@ -256,90 +295,104 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo else { if (task.Result.ErrorMessage.Contains("404")) + { + logger.Debug(ServiceError.RecordNotFound); throw new ServiceException(ServiceError.RecordNotFound, null); + } else + { + logger.Error($"Delete item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); throw new Exception($"Delete item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); + } } } - else - throw new Exception($"Delete item failed: error parsing the query."); - } catch (Exception ex) - { throw ex; } - } - + { throw ex; } + } else { - throw new Exception("CosmosDB Execution failed. Container not found."); + logger.Error("CosmosDB Delete Execution failed. Container not found."); + throw new Exception("CosmosDB Delete Execution failed. Container not found."); } - case ServiceCursorDef.CursorType.Insert: - if (container != null) - { - try + case ServiceCursorDef.CursorType.Insert: + if (container != null) { - using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(jsonData))) + try { - - Task task = Task.Run(async () => await container.CreateItemStreamAsync(stream, new PartitionKey(partitionKeyValue)).ConfigureAwait(false)); - if (task.Result.IsSuccessStatusCode) - { - return 1; - } - else + logger.Debug($"Insert : {jsonData}"); + using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(jsonData))) { - if (task.Result.ErrorMessage.Contains("Conflict (409)")) - throw new ServiceException(ServiceError.RecordAlreadyExists, null); - else - throw new Exception($"Create item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); + Task task = Task.Run(async () => await container.CreateItemStreamAsync(stream, new PartitionKey(partitionKeyValue)).ConfigureAwait(false)); + if (task.Result.IsSuccessStatusCode) + return 1; + else + { + if (task.Result.ErrorMessage.Contains("Conflict (409)")) + { + logger.Debug(ServiceError.RecordAlreadyExists); + throw new ServiceException(ServiceError.RecordAlreadyExists, null); + } + else + { + logger.Error($"Create item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); + throw new Exception($"Create item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); + } + } } } + catch (Exception ex) + { + throw ex; + } } - catch (Exception ex) + else { - throw ex; + logger.Error("CosmosDB Insert Execution failed. Container not found."); + throw new Exception("CosmosDB Insert Execution failed. Container not found."); } - } - else - { - throw new Exception("CosmosDB Execution failed. Container not found."); - } - case ServiceCursorDef.CursorType.Update: + case ServiceCursorDef.CursorType.Update: if (container != null) { - try + if (!keyCondition.Any() || !keyCondition.ContainsKey("id") || !keyCondition.ContainsKey(partitionKey)) + { + logger.Error($"Update item failed: error parsing the query."); + throw new Exception($"Update item failed: error parsing the query."); + } + else { - if (string.IsNullOrEmpty(partitionKeyValue)) - partitionKeyValue = keyCondition["id"].ToString(); // partitionKeyValue = id + try + { + logger.Debug($"Update : {jsonData}"); + using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(jsonData))) { Task task = Task.Run(async () => await container.UpsertItemStreamAsync(stream, new PartitionKey(partitionKeyValue)).ConfigureAwait(false)); if (task.Result.IsSuccessStatusCode) - { return 1; - } else { - + logger.Error($"Update item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); throw new Exception($"Update item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); - } } - } - catch (Exception ex) - { - throw ex; + } + catch (Exception ex) + { + throw ex; + } } } else { - throw new Exception("CosmosDB Execution failed. Container not found."); + logger.Error("CosmosDB Update Execution failed. Container not found."); + throw new Exception("CosmosDB Update Execution failed. Container not found."); } } - return 0; + } public override IDataReader ExecuteReader(ServiceCursorDef cursorDef, IDataParameterCollection parms, CommandBehavior behavior) { @@ -362,26 +415,32 @@ public override IDataReader ExecuteReader(ServiceCursorDef cursorDef, IDataParam catch (Exception e) { throw e; } } - private VarValue GetDataParameterfromQueryVars(string filter, IEnumerable values) + private VarValue GetDataEqualParameterfromQueryVars(string filter, IEnumerable values, out string name) { + string Equal_Filter_pattern = @"\((.*) = :(.*)\)"; VarValue varValue = null; - Match match = Regex.Match(filter, FILTER_PATTERN); + name = string.Empty; + Match match = Regex.Match(filter, Equal_Filter_pattern); if (match.Groups.Count > 1) { string varName = match.Groups[2].Value; - string name = match.Groups[1].Value; + varName = varName.Remove(varName.Length - 1); + name = match.Groups[1].Value; varValue = values.FirstOrDefault(v => v.Name == $":{varName}"); } return varValue; } - private string GetDataParameterDataParameterCollection(string filter, IDataParameterCollection parms) + private string GetDataEqualParameterfromCollection(string filter, IDataParameterCollection parms, out string name) { - Match match = Regex.Match(filter, FILTER_PATTERN); + string Equal_Filter_pattern = @"\((.*) = :(.*)\)"; + name = string.Empty; + Match match = Regex.Match(filter, Equal_Filter_pattern); if (match.Groups.Count > 1) { string varName = match.Groups[2].Value; - + name = match.Groups[1].Value; + varName = varName.Remove(varName.Length - 1); if (parms[varName] is ServiceParameter serviceParm) { return serviceParm.Value.ToString(); @@ -389,35 +448,72 @@ private string GetDataParameterDataParameterCollection(string filter, IDataParam } return string.Empty; } - private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, IDataParameterCollection parms, Container container, out CosmosDBDataReader cosmosDBDataReader,out RequestWrapper requestWrapper) + + private string GetDataParameterfromCollectionFormatted(string attName, IDataParameterCollection parms) { + string varValuestr = string.Empty; + if (parms[attName] is ServiceParameter serviceParm) + if (GeneXus.Data.Cosmos.CosmosDBHelper.FormattedAsStringDbType(serviceParm.DbType)) + { + varValuestr = '"' + $"{serviceParm.Value.ToString()}" + '"'; + } + else + varValuestr = serviceParm.Value.ToString(); + return varValuestr; + } - if (query.KeyFilters.Any() && (!query.Filters.Any())) - { - //Key is just the id or - requestWrapper = new RequestWrapper(cosmosClient, container, null); - requestWrapper.queryByPK = true; - requestWrapper.idValue = GetDataParameterfromQueryVars(query.KeyFilters.First(), query.Vars)?.Value.ToString(); - requestWrapper.idValue = requestWrapper.idValue ?? GetDataParameterDataParameterCollection(query.KeyFilters.First(),parms); + private CosmosDBDataReader GetDataReaderQueryByPK(ServiceCursorDef cursorDef, Container container, string idValue, string partitionKeyValue,out RequestWrapper requestWrapper) + { + requestWrapper = new RequestWrapper(cosmosClient, container, null); + requestWrapper.idValue = idValue; + requestWrapper.partitionKeyValue = partitionKeyValue; + + logger.Debug($"Execute PK query id = {requestWrapper.idValue}, partitionKey = {requestWrapper.partitionKeyValue}"); + requestWrapper.queryByPK = true; + return new CosmosDBDataReader(cursorDef, requestWrapper); + + } - if (requestWrapper.idValue != null) + /// + /// Create object for querying the database. + /// + /// + /// + /// + /// + /// + /// + private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, IDataParameterCollection parms, Container container, out CosmosDBDataReader cosmosDBDataReader,out RequestWrapper requestWrapper) + { + //Check if the filters are the Primary Key + if (query.Filters.Any()) + { + if (query.Filters.Count() == 2) { - if (query.KeyFilters.Count() > 1) + string fieldValue1 = string.Empty; + string fieldValue2 = string.Empty; + + fieldValue1 = GetDataEqualParameterfromQueryVars(query.Filters.First(), query.Vars, out string fieldName1)?.Value.ToString(); + fieldValue1 = fieldValue1 ?? GetDataEqualParameterfromCollection(query.Filters.First(), parms, out fieldName1); + + fieldValue2 = GetDataEqualParameterfromQueryVars(query.Filters.Skip(1).First(), query.Vars, out string fieldName2)?.Value.ToString(); + fieldValue2 = fieldValue2 ?? GetDataEqualParameterfromCollection(query.Filters.Skip(1).First(), parms, out fieldName2); + + if (fieldName1 == "id" && fieldName2 == query.PartitionKey) { - requestWrapper.partitionKeyValue = GetDataParameterfromQueryVars(query.KeyFilters.Skip(1).Take(1).First(), query.Vars)?.Value.ToString(); - requestWrapper.partitionKeyValue = requestWrapper.partitionKeyValue ?? GetDataParameterDataParameterCollection(query.KeyFilters.Skip(1).Take(1).First(), parms); + cosmosDBDataReader = GetDataReaderQueryByPK(cursorDef, container, fieldValue1, fieldValue2, out requestWrapper); + return; } else - requestWrapper.partitionKeyValue = requestWrapper.idValue; - - if (requestWrapper.partitionKeyValue != null) - { - cosmosDBDataReader = new CosmosDBDataReader(cursorDef, requestWrapper); - return; + { + if (fieldName1 == query.PartitionKey && fieldName2 == "id") + { + cosmosDBDataReader = GetDataReaderQueryByPK(cursorDef, container, fieldValue2, fieldValue1, out requestWrapper); + return; + } } } } - //Create the query string tableName = query.TableName; IEnumerable projection = query.Projection; @@ -434,88 +530,55 @@ private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, I IEnumerable allFilters = query.KeyFilters.Concat(query.Filters); IEnumerable allFiltersQuery = Array.Empty(); - - string regex1 = @"\(([^\)\(]+)\)"; - string regex2 = @"(.*)[^<>!=]\s*(=|!=|<|>|<=|>=|<>)\s*(:.*)"; - - string keyFilterS; - string condition = string.Empty; + IEnumerable keyFilterQ = Array.Empty(); foreach (string keyFilter in allFilters) - { - keyFilterS = keyFilter; - condition = keyFilter; - - MatchCollection matchCollection = Regex.Matches(keyFilterS, regex1); - - foreach (Match match in matchCollection) + { + string filterProcess = keyFilter.ToString(); + foreach (VarValue item in query.Vars) { - if (match.Groups.Count > 0) + string varValuestr = string.Empty; + if (filterProcess.Contains(string.Format($"{item.Name}:"))) { - string cond = match.Groups[1].Value; - Match match2 = Regex.Match(cond, regex2); - if (match2.Success) + if (GeneXus.Data.Cosmos.CosmosDBHelper.FormattedAsStringGXType(item.Type)) + varValuestr = '"' + $"{item.Value.ToString()}" + '"'; + else { - //Get the value for this item - string varValuestr = string.Empty; - if (match2.Groups.Count > 0) - { - string column = match2.Groups[1].Value.Trim(); - string attName = match2.Groups[3].Value.Trim(); - if (attName.StartsWith(":")) - attName = attName.Substring(1); - - string op = match2.Groups[2].Value.Trim(); - - //look at IDataParameterCollection parms - if (parms[attName] is ServiceParameter serviceParm) - if (GeneXus.Data.Cosmos.CosmosDBHelper.FormattedAsStringDbType(serviceParm.DbType)) - { - varValuestr = '"' + $"{serviceParm.Value.ToString()}" + '"'; - } - else - varValuestr = serviceParm.Value.ToString(); - - - //look at query.vars - foreach (VarValue item in query.Vars) - { - if (item.Name == match2.Groups[3].Value) - { - - if (GeneXus.Data.Cosmos.CosmosDBHelper.FormattedAsStringGXType(item.Type)) - { - varValuestr = '"' + $"{item.Value.ToString()}" + '"'; - } - else - { - varValuestr = item.Value.ToString(); - varValuestr = varValuestr.Equals("True") ? "true" : varValuestr; - varValuestr = varValuestr.Equals("False") ? "false" : varValuestr; - } - break; - } - } - - //Controlar antes de mandar la sentencia a ejecutar - - condition = condition.Replace(cond, $"{column}{op}{varValuestr}"); - } + varValuestr = item.Value.ToString(); + varValuestr = varValuestr.Equals("True") ? "true" : varValuestr; + varValuestr = varValuestr.Equals("False") ? "false" : varValuestr; + } + filterProcess = filterProcess.Replace(string.Format($"{item.Name}:"), varValuestr); + } + } + foreach (object p in parms) + { + if (p is ServiceParameter) + { + ServiceParameter p1 = (ServiceParameter)p; + string varValuestr = string.Empty; + if (filterProcess.Contains(string.Format($":{p1.ParameterName}:"))) + { + if (GeneXus.Data.Cosmos.CosmosDBHelper.FormattedAsStringDbType(p1.DbType)) + varValuestr = '"' + $"{p1.Value.ToString()}" + '"'; else - { - //Check that cond is a valid attribute - boolean - } + varValuestr = p1.Value.ToString(); + + filterProcess = filterProcess.Replace(string.Format($":{p1.ParameterName}:"), varValuestr); } } } + filterProcess = filterProcess.Replace("Func.", ""); + filterProcess = filterProcess.Replace("[", "("); + filterProcess = filterProcess.Replace("]", ")"); foreach (string d in projection) { string wholeWordPattern = String.Format(@"\b{0}\b", d); - condition = Regex.Replace(condition, wholeWordPattern, $"{TABLE_ALIAS}.{d}"); + filterProcess = Regex.Replace(filterProcess, wholeWordPattern, $"{TABLE_ALIAS}.{d}"); } - keyFilterQ = new string[] { condition }; + keyFilterQ = new string[] { filterProcess }; allFiltersQuery = allFiltersQuery.Concat(keyFilterQ); } @@ -532,6 +595,9 @@ private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, I string orderExpression = String.Join(",", orderExpressionList); string sqlQuery = SetupQuery(projectionList, filterExpression, tableName, orderExpression); + + logger.Debug(sqlQuery); + QueryDefinition queryDefinition = new QueryDefinition(sqlQuery); requestWrapper = new RequestWrapper(cosmosClient, container, queryDefinition); requestWrapper.queryByPK = false; diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs index 43d948eba..12543c8ec 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs @@ -32,8 +32,10 @@ private void CheckCurrentPosition() if (m_currentEntry == null) throw new ServiceException(ServiceError.RecordNotFound); } - private void ProcessStream(Stream stream) + private void ProcessPKStream(Stream stream) { + //Query by PK -> only one record + if (stream != null) { using (StreamReader sr = new StreamReader(stream)) @@ -42,18 +44,22 @@ private void ProcessStream(Stream stream) Newtonsoft.Json.JsonSerializer jsonSerializer = new Newtonsoft.Json.JsonSerializer(); object array = jsonSerializer.Deserialize(jtr); - string json = ((Newtonsoft.Json.Linq.JToken)array).Root.ToString(); - var jsonDocument = JsonDocument.Parse(json); - var jsonDoc = jsonDocument.RootElement; - foreach (var jsonProperty in jsonDoc.EnumerateObject()) - { - if (jsonProperty.Name == "Documents") - { - Items = JsonConvert.DeserializeObject>>(jsonProperty.Value.ToString()); - break; - } - } - ItemCount = Items.Count; + Dictionary result = JsonConvert.DeserializeObject>(array.ToString()); + + //remove metadata + result.Remove("_rid"); + result.Remove("_self"); + result.Remove("_etag"); + result.Remove("_attachments"); + result.Remove("_ts"); + + Items = new List>(); + Items.Add(result); + + if (Items != null) + ItemCount = Items.Count; + else + ItemCount = 0; } } } @@ -66,7 +72,14 @@ public CosmosDBDataReader(ServiceCursorDef cursorDef, RequestWrapper request) m_response = m_request.Read(); m_feedIterator = m_response.feedIterator; if (m_feedIterator == null) - ProcessStream(m_response.stream); + { + if (m_response != null) + { + Items = m_response.Items; + ItemCount = Items.Count; + m_currentPosition = -1; + } + } } public object this[string name] { @@ -295,7 +308,10 @@ private async Task GetPage() break; } } - ItemCount = Items.Count; + if (Items != null) + ItemCount = Items.Count; + else + ItemCount = 0; } } } diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs index 8a7deff9d..076a6ecf9 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs @@ -11,8 +11,7 @@ namespace GeneXus.Data.Cosmos internal class CosmosDBHelper { internal static bool AddItemValue(string parmName, string fromName, Dictionary values, IDataParameterCollection parms, IEnumerable queryVars, ref string jsonData) - { - + { if (!AddItemValue(parmName, values, parms[fromName] as ServiceParameter, out string data)) { VarValue varValue = queryVars.FirstOrDefault(v => v.Name == $":{fromName}"); @@ -33,7 +32,6 @@ internal static bool AddItemValue(string parmName, string fromName, Dictionary> ProcessPKStream(Stream stream) + { + //Query by PK -> only one record + + List > Items = new List>(); + if (stream != null) + { + using (StreamReader sr = new StreamReader(stream)) + using (JsonTextReader jtr = new JsonTextReader(sr)) + { + Newtonsoft.Json.JsonSerializer jsonSerializer = new Newtonsoft.Json.JsonSerializer(); + object array = jsonSerializer.Deserialize(jtr); + + Dictionary result = JsonConvert.DeserializeObject>(array.ToString()); + + //remove metadata + result.Remove("_rid"); + result.Remove("_self"); + result.Remove("_etag"); + result.Remove("_attachments"); + result.Remove("_ts"); + Items.Add(result); + } + } + return Items; + } private async Task ReadItemAsyncByPK(string idValue, string partitionKeyValue) { + List> Items = new List>(); using (ResponseMessage responseMessage = await m_container.ReadItemStreamAsync( partitionKey: new PartitionKey(partitionKeyValue), id: idValue).ConfigureAwait(false)) + //ResponseMessage responseMessage = await m_container.ReadItemStreamAsync( + //partitionKey: new PartitionKey(partitionKeyValue), + //id: idValue).ConfigureAwait(false); { if (!responseMessage.IsSuccessStatusCode) @@ -37,10 +70,14 @@ private async Task ReadItemAsyncByPK(string idValue, string par throw new Exception(GeneXus.Data.Cosmos.CosmosDBHelper.FormatExceptionMessage(responseMessage.StatusCode.ToString(), responseMessage.ErrorMessage)); } } - return new ResponseWrapper(responseMessage); + else + { + Items = ProcessPKStream(responseMessage.Content); + } } + return new ResponseWrapper(Items); } - public ResponseWrapper Read() + public ResponseWrapper Read() { if (queryByPK) { diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBResponseWrapper.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBResponseWrapper.cs index 62ae6849a..c3da9e631 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBResponseWrapper.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBResponseWrapper.cs @@ -19,9 +19,15 @@ public ResponseWrapper(FeedIterator feedIter) { feedIterator = feedIter; } - public ResponseWrapper(ResponseMessage responseMessage) + public ResponseWrapper(Stream streamResponse) { - stream = responseMessage.Content; + stream = streamResponse; + } + + public ResponseWrapper(List> responseItems) + { + Items = responseItems; + ItemCount = responseItems.Count; } public List> Items { get; set; } public int ItemCount { get; set; } From e1afa4a4273ffbdddafce7d49f99f12af3f92cef Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Thu, 29 Dec 2022 17:14:24 -0300 Subject: [PATCH 15/25] remove comment, to force commit --- dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs index a6a39e413..1b60776b5 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs @@ -67,7 +67,6 @@ public override string ConnectionString } private static void InitializeDBConnection() { - //System.Diagnostics.Debugger.Launch(); DbConnectionStringBuilder builder = new DbConnectionStringBuilder(false); builder.ConnectionString = mConnectionString; From 22685fb4002c275e65524c291a21bbeac17700f4 Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Fri, 30 Dec 2022 17:28:59 -0300 Subject: [PATCH 16/25] Use invariantculture for decimals --- .../dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs | 3 ++- .../src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs index 12543c8ec..593e98b74 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Data; +using System.Globalization; using System.IO; using System.Linq; using System.Text.Json; @@ -197,7 +198,7 @@ public DateTime GetDateTime(int i) } public decimal GetDecimal(int i) { - return Convert.ToDecimal(GetAttValue(i)); + return Convert.ToDecimal(GetAttValue(i), CultureInfo.InvariantCulture); } public double GetDouble(int i) { diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs index 076a6ecf9..03262953b 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs +++ b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Data; +using System.Globalization; using System.IO; using System.Linq; using System.Text; @@ -121,13 +122,18 @@ internal static object ToItemValue(DbType dbType, Object value) case DbType.UInt32: case DbType.UInt64: case DbType.VarNumeric: - case DbType.Decimal: case DbType.Double: case DbType.Int16: case DbType.Int32: case DbType.Int64: attValue = value.ToString(); break; + case DbType.Decimal: + { + decimal itemvalue = (decimal)value; + attValue = itemvalue.ToString(CultureInfo.InvariantCulture); + break; + } default: string valueDefault = value.ToString().Replace("%", string.Empty); attValue = valueDefault; From b76d287a9aa7d079dc7274c1603b471ac8f96440 Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Mon, 2 Jan 2023 10:57:38 -0300 Subject: [PATCH 17/25] fix sln --- dotnet/DotNetStandardClasses.sln | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dotnet/DotNetStandardClasses.sln b/dotnet/DotNetStandardClasses.sln index 03e1fa6cb..885bbd4ea 100644 --- a/dotnet/DotNetStandardClasses.sln +++ b/dotnet/DotNetStandardClasses.sln @@ -529,7 +529,6 @@ Global {E85FDB0F-FA81-4CDD-8BF3-865269CE2DB3}.Debug|Any CPU.Build.0 = Debug|Any CPU {E85FDB0F-FA81-4CDD-8BF3-865269CE2DB3}.Release|Any CPU.ActiveCfg = Release|Any CPU {E85FDB0F-FA81-4CDD-8BF3-865269CE2DB3}.Release|Any CPU.Build.0 = Release|Any CPU -<<<<<<< HEAD {65048104-212A-4819-AECF-89CA9C08C83F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {65048104-212A-4819-AECF-89CA9C08C83F}.Debug|Any CPU.Build.0 = Debug|Any CPU {65048104-212A-4819-AECF-89CA9C08C83F}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -542,8 +541,6 @@ Global {19625C3D-CEE1-44B7-A11E-1AA27E6A5439}.Debug|Any CPU.Build.0 = Debug|Any CPU {19625C3D-CEE1-44B7-A11E-1AA27E6A5439}.Release|Any CPU.ActiveCfg = Release|Any CPU {19625C3D-CEE1-44B7-A11E-1AA27E6A5439}.Release|Any CPU.Build.0 = Release|Any CPU -======= ->>>>>>> master {3B1B5706-E896-4CEB-A551-E30226303BDB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3B1B5706-E896-4CEB-A551-E30226303BDB}.Debug|Any CPU.Build.0 = Debug|Any CPU {3B1B5706-E896-4CEB-A551-E30226303BDB}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -678,12 +675,9 @@ Global {DCEC0B38-93B6-4003-81E6-9FBC2BB4F163} = {7BA5A2CE-7992-4F87-9D84-91AE4D046F5A} {F8BA0D65-267D-491F-BFAB-33F5E5B61AD7} = {30159B0F-BE61-4DB7-AC02-02851426BE4B} {E85FDB0F-FA81-4CDD-8BF3-865269CE2DB3} = {7BA5A2CE-7992-4F87-9D84-91AE4D046F5A} -<<<<<<< HEAD {65048104-212A-4819-AECF-89CA9C08C83F} = {1D6F1776-FF4B-46C2-9B3D-BC46CCF049DC} {48430E50-043A-47A2-8278-B86A4420758A} = {1D6F1776-FF4B-46C2-9B3D-BC46CCF049DC} {19625C3D-CEE1-44B7-A11E-1AA27E6A5439} = {C264F34E-2CE0-4DCA-B22D-4155821BE611} -======= ->>>>>>> master {3B1B5706-E896-4CEB-A551-E30226303BDB} = {4C43F2DA-59E5-46F5-B691-195449498555} {F8ABEA82-F823-4E9C-96FA-26AF24C932E0} = {30159B0F-BE61-4DB7-AC02-02851426BE4B} {65048104-212A-4819-AECF-89CA9C08C83F} = {1D6F1776-FF4B-46C2-9B3D-BC46CCF049DC} From 9afa0f29973494fb59857ba0d5fbe909e9d1b8fb Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Tue, 3 Jan 2023 12:19:05 -0300 Subject: [PATCH 18/25] Move project, only support NET 6. --- dotnet/DotNetStandardClasses.sln | 25 +-- .../DynService.Cosmos/CosmosDBHelper.cs | 145 ------------------ .../DynService/Cosmos}/CosmosDBConnection.cs | 50 ++---- .../DynService/Cosmos}/CosmosDBDataReader.cs | 4 +- .../Cosmos}/CosmosDBDatastoreHelper.cs | 0 .../DynService/Cosmos/CosmosDBHelper.cs | 50 ++++++ .../DynService/Cosmos}/CosmosDBMaps.cs | 0 .../Cosmos}/CosmosDBRequestWrapper.cs | 0 .../Cosmos}/CosmosDBResponseWrapper.cs | 0 .../Cosmos}/DynService.CosmosDB.csproj | 14 +- 10 files changed, 79 insertions(+), 209 deletions(-) delete mode 100644 dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs rename dotnet/src/{dotnetcommon/DynService.Cosmos => dotnetcore/DynService/Cosmos}/CosmosDBConnection.cs (94%) rename dotnet/src/{dotnetcommon/DynService.Cosmos => dotnetcore/DynService/Cosmos}/CosmosDBDataReader.cs (98%) rename dotnet/src/{dotnetcommon/DynService.Cosmos => dotnetcore/DynService/Cosmos}/CosmosDBDatastoreHelper.cs (100%) create mode 100644 dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBHelper.cs rename dotnet/src/{dotnetcommon/DynService.Cosmos => dotnetcore/DynService/Cosmos}/CosmosDBMaps.cs (100%) rename dotnet/src/{dotnetcommon/DynService.Cosmos => dotnetcore/DynService/Cosmos}/CosmosDBRequestWrapper.cs (100%) rename dotnet/src/{dotnetcommon/DynService.Cosmos => dotnetcore/DynService/Cosmos}/CosmosDBResponseWrapper.cs (100%) rename dotnet/src/{dotnetcommon/DynService.Cosmos => dotnetcore/DynService/Cosmos}/DynService.CosmosDB.csproj (52%) diff --git a/dotnet/DotNetStandardClasses.sln b/dotnet/DotNetStandardClasses.sln index 885bbd4ea..34f6c71ce 100644 --- a/dotnet/DotNetStandardClasses.sln +++ b/dotnet/DotNetStandardClasses.sln @@ -220,7 +220,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ProjectHealthTest", "test\P EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotNetRedisTest", "test\DotNetRedisTest\DotNetRedisTest.csproj", "{48430E50-043A-47A2-8278-B86A4420758A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DynService.CosmosDB", "src\dotnetcommon\DynService.Cosmos\DynService.CosmosDB.csproj", "{19625C3D-CEE1-44B7-A11E-1AA27E6A5439}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotNetCoreWebUnitTest", "test\DotNetCoreWebUnitTest\DotNetCoreWebUnitTest.csproj", "{531863CA-93A0-42AA-AB5C-FA0E672C03B8}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "opentelemetry", "opentelemetry", "{BBE020D4-C0FF-41A9-9EB1-D1EE12CC4BB8}" @@ -229,7 +228,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GeneXus.OpenTelemetry.Light EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GeneXus.OpenTelemetry.AWS.AspNet", "src\dotnetcore\Providers\OpenTelemetry\OpenTelemetryAWSOtel\GeneXus.OpenTelemetry.AWS.AspNet.csproj", "{B5A9DEA7-67EC-49E4-924E-4729C34286EC}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GeneXus.OpenTelemetry", "src\dotnetcore\Providers\OpenTelemetry\OpenTelemetry\GeneXus.OpenTelemetry.csproj", "{00B1FA38-7D0B-47E4-860C-23490249A4D6}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GeneXus.OpenTelemetry", "src\dotnetcore\Providers\OpenTelemetry\OpenTelemetry\GeneXus.OpenTelemetry.csproj", "{00B1FA38-7D0B-47E4-860C-23490249A4D6}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DynService.CosmosDB", "src\dotnetcore\DynService\Cosmos\DynService.CosmosDB.csproj", "{52DC6C43-58ED-4310-996B-06E95105F848}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -529,18 +530,6 @@ Global {E85FDB0F-FA81-4CDD-8BF3-865269CE2DB3}.Debug|Any CPU.Build.0 = Debug|Any CPU {E85FDB0F-FA81-4CDD-8BF3-865269CE2DB3}.Release|Any CPU.ActiveCfg = Release|Any CPU {E85FDB0F-FA81-4CDD-8BF3-865269CE2DB3}.Release|Any CPU.Build.0 = Release|Any CPU - {65048104-212A-4819-AECF-89CA9C08C83F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {65048104-212A-4819-AECF-89CA9C08C83F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {65048104-212A-4819-AECF-89CA9C08C83F}.Release|Any CPU.ActiveCfg = Release|Any CPU - {65048104-212A-4819-AECF-89CA9C08C83F}.Release|Any CPU.Build.0 = Release|Any CPU - {48430E50-043A-47A2-8278-B86A4420758A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {48430E50-043A-47A2-8278-B86A4420758A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {48430E50-043A-47A2-8278-B86A4420758A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {48430E50-043A-47A2-8278-B86A4420758A}.Release|Any CPU.Build.0 = Release|Any CPU - {19625C3D-CEE1-44B7-A11E-1AA27E6A5439}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {19625C3D-CEE1-44B7-A11E-1AA27E6A5439}.Debug|Any CPU.Build.0 = Debug|Any CPU - {19625C3D-CEE1-44B7-A11E-1AA27E6A5439}.Release|Any CPU.ActiveCfg = Release|Any CPU - {19625C3D-CEE1-44B7-A11E-1AA27E6A5439}.Release|Any CPU.Build.0 = Release|Any CPU {3B1B5706-E896-4CEB-A551-E30226303BDB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3B1B5706-E896-4CEB-A551-E30226303BDB}.Debug|Any CPU.Build.0 = Debug|Any CPU {3B1B5706-E896-4CEB-A551-E30226303BDB}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -573,6 +562,10 @@ Global {00B1FA38-7D0B-47E4-860C-23490249A4D6}.Debug|Any CPU.Build.0 = Debug|Any CPU {00B1FA38-7D0B-47E4-860C-23490249A4D6}.Release|Any CPU.ActiveCfg = Release|Any CPU {00B1FA38-7D0B-47E4-860C-23490249A4D6}.Release|Any CPU.Build.0 = Release|Any CPU + {52DC6C43-58ED-4310-996B-06E95105F848}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {52DC6C43-58ED-4310-996B-06E95105F848}.Debug|Any CPU.Build.0 = Debug|Any CPU + {52DC6C43-58ED-4310-996B-06E95105F848}.Release|Any CPU.ActiveCfg = Release|Any CPU + {52DC6C43-58ED-4310-996B-06E95105F848}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -675,9 +668,6 @@ Global {DCEC0B38-93B6-4003-81E6-9FBC2BB4F163} = {7BA5A2CE-7992-4F87-9D84-91AE4D046F5A} {F8BA0D65-267D-491F-BFAB-33F5E5B61AD7} = {30159B0F-BE61-4DB7-AC02-02851426BE4B} {E85FDB0F-FA81-4CDD-8BF3-865269CE2DB3} = {7BA5A2CE-7992-4F87-9D84-91AE4D046F5A} - {65048104-212A-4819-AECF-89CA9C08C83F} = {1D6F1776-FF4B-46C2-9B3D-BC46CCF049DC} - {48430E50-043A-47A2-8278-B86A4420758A} = {1D6F1776-FF4B-46C2-9B3D-BC46CCF049DC} - {19625C3D-CEE1-44B7-A11E-1AA27E6A5439} = {C264F34E-2CE0-4DCA-B22D-4155821BE611} {3B1B5706-E896-4CEB-A551-E30226303BDB} = {4C43F2DA-59E5-46F5-B691-195449498555} {F8ABEA82-F823-4E9C-96FA-26AF24C932E0} = {30159B0F-BE61-4DB7-AC02-02851426BE4B} {65048104-212A-4819-AECF-89CA9C08C83F} = {1D6F1776-FF4B-46C2-9B3D-BC46CCF049DC} @@ -687,6 +677,7 @@ Global {27D54041-BDD5-428E-8CA6-C96D519F5451} = {BBE020D4-C0FF-41A9-9EB1-D1EE12CC4BB8} {B5A9DEA7-67EC-49E4-924E-4729C34286EC} = {BBE020D4-C0FF-41A9-9EB1-D1EE12CC4BB8} {00B1FA38-7D0B-47E4-860C-23490249A4D6} = {BBE020D4-C0FF-41A9-9EB1-D1EE12CC4BB8} + {52DC6C43-58ED-4310-996B-06E95105F848} = {79C9ECC6-2935-4C43-BF32-94698547F584} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {E18684C9-7D76-45CD-BF24-E3944B7F174C} diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs b/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs deleted file mode 100644 index 03262953b..000000000 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBHelper.cs +++ /dev/null @@ -1,145 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Data; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Text; -using GeneXus.Data.NTier; - -namespace GeneXus.Data.Cosmos -{ - internal class CosmosDBHelper - { - internal static bool AddItemValue(string parmName, string fromName, Dictionary values, IDataParameterCollection parms, IEnumerable queryVars, ref string jsonData) - { - if (!AddItemValue(parmName, values, parms[fromName] as ServiceParameter, out string data)) - { - VarValue varValue = queryVars.FirstOrDefault(v => v.Name == $":{fromName}"); - if (varValue != null) - { - values[parmName] = varValue; - jsonData = AddToJsonStream(varValue.Type, parmName, varValue); - } - return varValue != null; - } - StringBuilder stringBuilder = new StringBuilder(jsonData); - string concatData; - if (!string.IsNullOrEmpty(jsonData)) - { - concatData = $",{data}"; - stringBuilder.Append(concatData); - jsonData = stringBuilder.ToString(); - } - else - jsonData = data; - return true; - } - - public static bool FormattedAsStringGXType(GXType gXType) - { - return (gXType == GXType.Date || gXType == GXType.DateTime || gXType == GXType.DateTime2 || gXType == GXType.VarChar || gXType == GXType.DateAsChar || gXType == GXType.NVarChar || gXType == GXType.LongVarChar || gXType == GXType.NChar || gXType == GXType.Char || gXType == GXType.Text || gXType == GXType.NText); - } - internal static bool FormattedAsStringDbType(DbType dbType) - { - return (dbType == DbType.String || dbType == DbType.Date || dbType == DbType.DateTime || dbType == DbType.DateTime2 || dbType == DbType.DateTimeOffset || dbType == DbType.StringFixedLength || dbType == DbType.AnsiString || dbType == DbType.AnsiStringFixedLength || dbType == DbType.Guid || dbType == DbType.Time); - } - internal static string AddToJsonStream(GXType gXType, string parmName, object value) - { - string valueItem; - if (FormattedAsStringGXType(gXType)) - { - valueItem = string.Format("\"{0}\"", value); - return string.Format("\"{0}\": {1}", parmName, string.Join(",", valueItem)); - } - else - return string.Format("\"{0}\": {1}", parmName, string.Join(",", value)); - } - internal static string AddToJsonStream(DbType dbType, string parmName, object value) - { - string valueItem; - string data; - if (FormattedAsStringDbType(dbType)) - { - valueItem = string.Format("\"{0}\"", value); - return string.Format("\"{0}\": {1}", parmName, valueItem); - } - else - { - data = string.Format("\"{0}\": {1}", parmName, value); - if (dbType == DbType.Boolean || dbType == DbType.Byte) - { - data = data.Replace("True", "true"); - data = data.Replace("False", "false"); - } - return data; - } - } - internal static string FormatExceptionMessage(string statusCode, string message) - { - return ($"CosmosDB Execution failed. Status code: {statusCode}. Message: {message}"); - } - internal static bool AddItemValue(string parmName, Dictionary dynParm, ServiceParameter parm, out string jsonData) - { - jsonData = string.Empty; - if (parm == null) - return false; - object value = ToItemValue(parm.DbType, parm.Value); - if (value != null) - { - dynParm[parmName] = value; - jsonData = AddToJsonStream(parm.DbType, parmName, value); - return true; - } - return false; - } - internal static object ToItemValue(DbType dbType, Object value) - { - object attValue; - switch (dbType) - { - case DbType.Binary: - if (value is byte[] valueArr) - { - attValue = new MemoryStream(valueArr); - break; - } - else throw new ArgumentException("Required value not found"); - case DbType.Boolean: - case DbType.Byte: - attValue = Convert.ToByte(value) == 1 ? true : false; - break; - case DbType.Time: - case DbType.Date: - attValue = DateTime.SpecifyKind((DateTime)value, DateTimeKind.Utc).ToString("yyyy-MM-dd"); - break; - case DbType.DateTime2: - case DbType.DateTime: - attValue = DateTime.SpecifyKind((DateTime)value, DateTimeKind.Utc).ToString("yyyy-MM-ddTHH:mm:ssK"); - break; - - case DbType.UInt16: - case DbType.UInt32: - case DbType.UInt64: - case DbType.VarNumeric: - case DbType.Double: - case DbType.Int16: - case DbType.Int32: - case DbType.Int64: - attValue = value.ToString(); - break; - case DbType.Decimal: - { - decimal itemvalue = (decimal)value; - attValue = itemvalue.ToString(CultureInfo.InvariantCulture); - break; - } - default: - string valueDefault = value.ToString().Replace("%", string.Empty); - attValue = valueDefault; - break; - } - return attValue; - } - } -} diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs similarity index 94% rename from dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs rename to dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs index 1b60776b5..f9fc9bdaa 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBConnection.cs +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using System.Text; +using System.Text.Json.Nodes; using System.Text.RegularExpressions; using System.Threading.Tasks; using GeneXus.Data.Cosmos; @@ -67,6 +68,7 @@ public override string ConnectionString } private static void InitializeDBConnection() { + //System.Diagnostics.Debugger.Launch(); DbConnectionStringBuilder builder = new DbConnectionStringBuilder(false); builder.ConnectionString = mConnectionString; @@ -163,27 +165,27 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo string partitionKey = query.PartitionKey; string partitionKeyValue = string.Empty; + JsonObject jsonObject = new JsonObject(); //Setup the json payload to execute the insert or update query. foreach (KeyValuePair asg in query.AssignAtts) { string name = asg.Key; string parmName = asg.Value.Substring(1).Remove(asg.Value.Length - 2); - CosmosDBHelper.AddItemValue(name, parmName, values, parms, query.Vars, ref jsonData); + CosmosDBHelper.AddItemValue(name, parmName, values, parms, query.Vars, ref jsonObject); if (name == partitionKey) partitionKeyValue = values[name].ToString(); } Dictionary keyCondition = new Dictionary(); - //Get the values for id and partitionKey + //Get the values for id and partitionKey string regex1 = @"\(([^\)\(]+)\)"; string regex2 = @"(.*)[^<>!=]\s*(=|!=|<|>|<=|>=|<>)\s*(:.*:)"; string keyFilterS; string condition = string.Empty; IEnumerable keyFilterQ = Array.Empty(); - IEnumerable allFilters = query.KeyFilters.Concat(query.Filters); foreach (string keyFilter in allFilters) @@ -192,7 +194,6 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo condition = keyFilter; MatchCollection matchCollection = Regex.Matches(keyFilterS, regex1); - foreach (Match match in matchCollection) { if (match.Groups.Count > 0) @@ -206,18 +207,17 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo string name = match2.Groups[1].Value; VarValue varValue = query.Vars.FirstOrDefault(v => v.Name == $":{varName}"); - string jsonDataKey = String.Empty; - string jsonDataPartitionKey = string.Empty; if (varValue != null) { keyCondition[name] = varValue.Value; //keyCondition[name] = GeneXus.Data.Cosmos.CosmosDBHelper.ToItemValue(varValue.Type, varValue.Value); if (isUpdate && name == "id") - jsonDataKey = GeneXus.Data.Cosmos.CosmosDBHelper.AddToJsonStream(varValue.Type, name, varValue.Value); - if (isUpdate && name == partitionKey) - jsonDataPartitionKey = GeneXus.Data.Cosmos.CosmosDBHelper.AddToJsonStream(varValue.Type, name, varValue.Value); - + jsonObject.Add(name, JsonValue.Create(varValue.Value)); + + if (isUpdate && name == partitionKey && partitionKey != "id") + jsonObject.Add(name, JsonValue.Create(varValue.Value)); + if (name == partitionKey) //TODO Partition Key can be double, bool partitionKeyValue = varValue.Value.ToString(); @@ -230,37 +230,22 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo //keyCondition[name] = GeneXus.Data.Cosmos.CosmosDBHelper.ToItemValue(serviceParm.DbType, serviceParm.Value); if (isUpdate && name == "id") - jsonDataKey = GeneXus.Data.Cosmos.CosmosDBHelper.AddToJsonStream(serviceParm.DbType, name, serviceParm.Value); - if (isUpdate && name == partitionKey) - jsonDataPartitionKey = GeneXus.Data.Cosmos.CosmosDBHelper.AddToJsonStream(serviceParm.DbType, name, serviceParm.Value); - + jsonObject.Add(name, JsonValue.Create(serviceParm.Value)); + + if (isUpdate && name == partitionKey && partitionKey != "id") + jsonObject.Add(name, JsonValue.Create(serviceParm.Value)); + if (name == partitionKey) //TODO Partition Key can be numeric partitionKeyValue = serviceParm.Value.ToString(); } } - if (!string.IsNullOrEmpty(jsonDataKey)) - { - if (!string.IsNullOrEmpty(jsonData)) - jsonData = $"{jsonData},{jsonDataKey}"; - - else - jsonData = jsonDataKey; - } - - if (!string.IsNullOrEmpty(jsonDataPartitionKey)) - { - if (!string.IsNullOrEmpty(jsonData)) - jsonData = $"{jsonData},{jsonDataPartitionKey}"; - else - jsonData = jsonDataPartitionKey; - } } } } } - jsonData = "{" + jsonData + "}"; + jsonData = jsonObject.ToJsonString(); //TODO: Get container from HashSet for performance Container container = GetContainer(query.TableName); @@ -528,8 +513,7 @@ private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, I } IEnumerable allFilters = query.KeyFilters.Concat(query.Filters); - IEnumerable allFiltersQuery = Array.Empty(); - + IEnumerable allFiltersQuery = Array.Empty(); IEnumerable keyFilterQ = Array.Empty(); foreach (string keyFilter in allFilters) diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs similarity index 98% rename from dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs rename to dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs index 593e98b74..43868d8ef 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDataReader.cs +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs @@ -202,7 +202,7 @@ public decimal GetDecimal(int i) } public double GetDouble(int i) { - return Convert.ToDouble(GetAttValue(i)); + return Convert.ToDouble(GetAttValue(i),CultureInfo.InvariantCulture); } public Type GetFieldType(int i) { @@ -211,7 +211,7 @@ public Type GetFieldType(int i) public float GetFloat(int i) { - return Convert.ToSingle(GetAttValue(i)); + return Convert.ToSingle(GetAttValue(i),CultureInfo.InvariantCulture); } public Guid GetGuid(int i) { diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDatastoreHelper.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDatastoreHelper.cs similarity index 100% rename from dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBDatastoreHelper.cs rename to dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDatastoreHelper.cs diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBHelper.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBHelper.cs new file mode 100644 index 000000000..ac141aacd --- /dev/null +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBHelper.cs @@ -0,0 +1,50 @@ +using System.Collections.Generic; +using System.Data; +using System.Linq; +using System.Text.Json.Nodes; +using GeneXus.Data.NTier; + +namespace GeneXus.Data.Cosmos +{ + internal class CosmosDBHelper + { + internal static bool AddItemValue(string parmName, string fromName, Dictionary values, IDataParameterCollection parms, IEnumerable queryVars, ref JsonObject jsonObject) + { + if (!AddItemValue(parmName, values, parms[fromName] as ServiceParameter, ref jsonObject)) + { + VarValue varValue = queryVars.FirstOrDefault(v => v.Name == $":{fromName}"); + if (varValue != null) + { + values[parmName] = varValue; + jsonObject.Add(parmName, JsonValue.Create(varValue)); + } + return varValue != null; + } + return true; + } + public static bool FormattedAsStringGXType(GXType gXType) + { + return (gXType == GXType.Date || gXType == GXType.DateTime || gXType == GXType.DateTime2 || gXType == GXType.VarChar || gXType == GXType.DateAsChar || gXType == GXType.NVarChar || gXType == GXType.LongVarChar || gXType == GXType.NChar || gXType == GXType.Char || gXType == GXType.Text || gXType == GXType.NText); + } + internal static bool FormattedAsStringDbType(DbType dbType) + { + return (dbType == DbType.String || dbType == DbType.Date || dbType == DbType.DateTime || dbType == DbType.DateTime2 || dbType == DbType.DateTimeOffset || dbType == DbType.StringFixedLength || dbType == DbType.AnsiString || dbType == DbType.AnsiStringFixedLength || dbType == DbType.Guid || dbType == DbType.Time); + } + internal static string FormatExceptionMessage(string statusCode, string message) + { + return ($"CosmosDB Execution failed. Status code: {statusCode}. Message: {message}"); + } + internal static bool AddItemValue(string parmName, Dictionary dynParm, ServiceParameter parm, ref JsonObject jsonObject) + { + if (parm == null) + return false; + if (parm.Value != null) + { + dynParm[parmName] = parm.Value; + jsonObject.Add(parmName, JsonValue.Create(parm.Value)); + return true; + } + return false; + } + } +} diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBMaps.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBMaps.cs similarity index 100% rename from dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBMaps.cs rename to dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBMaps.cs diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBRequestWrapper.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBRequestWrapper.cs similarity index 100% rename from dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBRequestWrapper.cs rename to dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBRequestWrapper.cs diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBResponseWrapper.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBResponseWrapper.cs similarity index 100% rename from dotnet/src/dotnetcommon/DynService.Cosmos/CosmosDBResponseWrapper.cs rename to dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBResponseWrapper.cs diff --git a/dotnet/src/dotnetcommon/DynService.Cosmos/DynService.CosmosDB.csproj b/dotnet/src/dotnetcore/DynService/Cosmos/DynService.CosmosDB.csproj similarity index 52% rename from dotnet/src/dotnetcommon/DynService.Cosmos/DynService.CosmosDB.csproj rename to dotnet/src/dotnetcore/DynService/Cosmos/DynService.CosmosDB.csproj index b461f15f4..120fad8f9 100644 --- a/dotnet/src/dotnetcommon/DynService.Cosmos/DynService.CosmosDB.csproj +++ b/dotnet/src/dotnetcore/DynService/Cosmos/DynService.CosmosDB.csproj @@ -1,6 +1,6 @@ - net462;net6.0 + net6.0 GeneXus.Data.NTier GeneXus.Data.DynService.CosmosDB false @@ -8,16 +8,6 @@ GeneXus.DynService.CosmosDB - - NETCORE - - - - - - - - @@ -27,6 +17,6 @@ - + From 86a57c17a9ac647a8612b50609455850816a50be Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Wed, 4 Jan 2023 18:47:46 -0300 Subject: [PATCH 19/25] support bool partitionKey --- .../DynService/Cosmos/CosmosDBConnection.cs | 121 +++++++++--------- .../DynService/Cosmos/CosmosDBDataReader.cs | 2 - .../Cosmos/CosmosDBDatastoreHelper.cs | 8 +- 3 files changed, 64 insertions(+), 67 deletions(-) diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs index f9fc9bdaa..c9546ca45 100644 --- a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Data; using System.Data.Common; +using System.Globalization; using System.IO; using System.Linq; using System.Text; @@ -102,6 +103,19 @@ private static void Initialize() cosmosDatabase = cosmosClient.GetDatabase(mdatabase); } + private PartitionKey ToPartitionKey(object value) + { + + if (value is double) + return new PartitionKey((double)value); + if (value is bool) + return new PartitionKey((bool)value); + if (value is string) + return new PartitionKey((string)value); + else + throw new Exception("Partitionkey can be double, bool or string."); + + } private Container GetContainer(string containerName) { if (cosmosDatabase != null && !string.IsNullOrEmpty(containerName)) @@ -123,21 +137,20 @@ private string SetupQuery(string projectionList, string filterExpression, string if (!string.IsNullOrEmpty(projectionList)) sqlSelect = string.Format(SELECT_TEMPLATE, projectionList); else - { //ERROR - + { + throw new Exception("Error setting up the query. Projection list is empty."); } if (!string.IsNullOrEmpty(tableName)) sqlFrom = string.Format(FROM_TEMPLATE, tableName); else { - //ERROR + throw new Exception("Error setting up the query. Table name is empty."); } if (!string.IsNullOrEmpty(filterExpression)) sqlWhere = string.Format(WHERE_TEMPLATE, filterExpression); if (!string.IsNullOrEmpty(orderbys)) sqlOrder = string.Format(ORDER_TEMPLATE, orderbys); - return $"{sqlSelect} {sqlFrom} {sqlWhere} {sqlOrder}"; } @@ -164,9 +177,11 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo string jsonData = string.Empty; string partitionKey = query.PartitionKey; - string partitionKeyValue = string.Empty; + //object partitionKeyValue; JsonObject jsonObject = new JsonObject(); + Dictionary keyCondition = new Dictionary(); + //Setup the json payload to execute the insert or update query. foreach (KeyValuePair asg in query.AssignAtts) { @@ -174,11 +189,9 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo string parmName = asg.Value.Substring(1).Remove(asg.Value.Length - 2); CosmosDBHelper.AddItemValue(name, parmName, values, parms, query.Vars, ref jsonObject); if (name == partitionKey) - partitionKeyValue = values[name].ToString(); + keyCondition[partitionKey] = values[name]; } - Dictionary keyCondition = new Dictionary(); - //Get the values for id and partitionKey string regex1 = @"\(([^\)\(]+)\)"; string regex2 = @"(.*)[^<>!=]\s*(=|!=|<|>|<=|>=|<>)\s*(:.*:)"; @@ -210,34 +223,24 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo if (varValue != null) { keyCondition[name] = varValue.Value; - //keyCondition[name] = GeneXus.Data.Cosmos.CosmosDBHelper.ToItemValue(varValue.Type, varValue.Value); - if (isUpdate && name == "id") jsonObject.Add(name, JsonValue.Create(varValue.Value)); if (isUpdate && name == partitionKey && partitionKey != "id") jsonObject.Add(name, JsonValue.Create(varValue.Value)); - - if (name == partitionKey) - //TODO Partition Key can be double, bool - partitionKeyValue = varValue.Value.ToString(); } else { if (parms[varName] is ServiceParameter serviceParm) { keyCondition[name] = serviceParm.Value; - //keyCondition[name] = GeneXus.Data.Cosmos.CosmosDBHelper.ToItemValue(serviceParm.DbType, serviceParm.Value); - + if (isUpdate && name == "id") jsonObject.Add(name, JsonValue.Create(serviceParm.Value)); if (isUpdate && name == partitionKey && partitionKey != "id") jsonObject.Add(name, JsonValue.Create(serviceParm.Value)); - if (name == partitionKey) - //TODO Partition Key can be numeric - partitionKeyValue = serviceParm.Value.ToString(); } } } @@ -262,15 +265,15 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo { if (!keyCondition.Any() || !keyCondition.ContainsKey("id") || !keyCondition.ContainsKey(partitionKey)) { - logger.Error($"Delete item failed: error parsing the query."); - throw new Exception($"Delete item failed: error parsing the query."); + GXLogging.Debug(logger,"Delete item failed: error parsing the query."); + throw new Exception($"Delete item failed: error parsing the query."); } else { object idField = keyCondition["id"]; - logger.Debug($"Delete : id= {idField.ToString()}, partitionKey= {partitionKeyValue}"); - Task task = Task.Run(async () => await container.DeleteItemStreamAsync(idField.ToString(), new PartitionKey(partitionKeyValue)).ConfigureAwait(false)); + GXLogging.Debug(logger,$"Delete : id= {idField.ToString()}, partitionKey= {keyCondition[partitionKey].ToString()}"); + Task task = Task.Run(async () => await container.DeleteItemStreamAsync(idField.ToString(), ToPartitionKey(keyCondition[partitionKey])).ConfigureAwait(false)); if (task.Result.IsSuccessStatusCode) { //ResponseMessage wrapps the delete record @@ -280,12 +283,12 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo { if (task.Result.ErrorMessage.Contains("404")) { - logger.Debug(ServiceError.RecordNotFound); + GXLogging.Debug(logger, ServiceError.RecordNotFound); throw new ServiceException(ServiceError.RecordNotFound, null); } else { - logger.Error($"Delete item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); + GXLogging.Error(logger,$"Delete item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); throw new Exception($"Delete item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); } } @@ -296,7 +299,7 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo } else { - logger.Error("CosmosDB Delete Execution failed. Container not found."); + GXLogging.Error(logger,"CosmosDB Delete Execution failed. Container not found."); throw new Exception("CosmosDB Delete Execution failed. Container not found."); } @@ -305,27 +308,36 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo { try { - logger.Debug($"Insert : {jsonData}"); - using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(jsonData))) + if (!keyCondition.Any() || !keyCondition.ContainsKey(partitionKey)) { - - Task task = Task.Run(async () => await container.CreateItemStreamAsync(stream, new PartitionKey(partitionKeyValue)).ConfigureAwait(false)); - if (task.Result.IsSuccessStatusCode) - return 1; - else + GXLogging.Error(logger,$"Insert item failed: error parsing the query."); + throw new Exception($"Insert item failed: error parsing the query."); + } + else + { + GXLogging.Debug(logger,$"Insert : {jsonData}"); + using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(jsonData))) { - if (task.Result.ErrorMessage.Contains("Conflict (409)")) - { - logger.Debug(ServiceError.RecordAlreadyExists); - throw new ServiceException(ServiceError.RecordAlreadyExists, null); - } + PartitionKey p = ToPartitionKey(keyCondition[partitionKey]); + Task task = Task.Run(async () => await container.CreateItemStreamAsync(stream, ToPartitionKey(keyCondition[partitionKey])).ConfigureAwait(false)); + if (task.Result.IsSuccessStatusCode) + return 1; else { - logger.Error($"Create item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); - throw new Exception($"Create item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); + if (task.Result.ErrorMessage.Contains("Conflict (409)")) + { + GXLogging.Debug(logger,ServiceError.RecordAlreadyExists); + throw new ServiceException(ServiceError.RecordAlreadyExists, null); + } + else + { + GXLogging.Error(logger,$"Create item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); + throw new Exception($"Create item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); + } } } } + } catch (Exception ex) { @@ -334,7 +346,7 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo } else { - logger.Error("CosmosDB Insert Execution failed. Container not found."); + GXLogging.Error(logger,"CosmosDB Insert Execution failed. Container not found."); throw new Exception("CosmosDB Insert Execution failed. Container not found."); } case ServiceCursorDef.CursorType.Update: @@ -342,23 +354,23 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo { if (!keyCondition.Any() || !keyCondition.ContainsKey("id") || !keyCondition.ContainsKey(partitionKey)) { - logger.Error($"Update item failed: error parsing the query."); + GXLogging.Error(logger, $"Update item failed: error parsing the query."); throw new Exception($"Update item failed: error parsing the query."); } else { try { - logger.Debug($"Update : {jsonData}"); + GXLogging.Debug(logger,$"Update : {jsonData}"); using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(jsonData))) { - Task task = Task.Run(async () => await container.UpsertItemStreamAsync(stream, new PartitionKey(partitionKeyValue)).ConfigureAwait(false)); + Task task = Task.Run(async () => await container.UpsertItemStreamAsync(stream, ToPartitionKey(keyCondition[partitionKey])).ConfigureAwait(false)); if (task.Result.IsSuccessStatusCode) return 1; else { - logger.Error($"Update item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); + GXLogging.Error(logger, $"Update item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); throw new Exception($"Update item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); } } @@ -371,7 +383,7 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo } else { - logger.Error("CosmosDB Update Execution failed. Container not found."); + GXLogging.Error(logger,"CosmosDB Update Execution failed. Container not found."); throw new Exception("CosmosDB Update Execution failed. Container not found."); } } @@ -433,26 +445,13 @@ private string GetDataEqualParameterfromCollection(string filter, IDataParameter return string.Empty; } - private string GetDataParameterfromCollectionFormatted(string attName, IDataParameterCollection parms) - { - string varValuestr = string.Empty; - if (parms[attName] is ServiceParameter serviceParm) - if (GeneXus.Data.Cosmos.CosmosDBHelper.FormattedAsStringDbType(serviceParm.DbType)) - { - varValuestr = '"' + $"{serviceParm.Value.ToString()}" + '"'; - } - else - varValuestr = serviceParm.Value.ToString(); - return varValuestr; - } - private CosmosDBDataReader GetDataReaderQueryByPK(ServiceCursorDef cursorDef, Container container, string idValue, string partitionKeyValue,out RequestWrapper requestWrapper) { requestWrapper = new RequestWrapper(cosmosClient, container, null); requestWrapper.idValue = idValue; requestWrapper.partitionKeyValue = partitionKeyValue; - logger.Debug($"Execute PK query id = {requestWrapper.idValue}, partitionKey = {requestWrapper.partitionKeyValue}"); + GXLogging.Debug(logger,$"Execute PK query id = {requestWrapper.idValue}, partitionKey = {requestWrapper.partitionKeyValue}"); requestWrapper.queryByPK = true; return new CosmosDBDataReader(cursorDef, requestWrapper); @@ -579,7 +578,7 @@ private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, I string orderExpression = String.Join(",", orderExpressionList); string sqlQuery = SetupQuery(projectionList, filterExpression, tableName, orderExpression); - logger.Debug(sqlQuery); + GXLogging.Debug(logger,sqlQuery); QueryDefinition queryDefinition = new QueryDefinition(sqlQuery); requestWrapper = new RequestWrapper(cosmosClient, container, queryDefinition); diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs index 43868d8ef..56dbc99e5 100644 --- a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs @@ -193,8 +193,6 @@ public DateTime GetDateTime(int i) if (GetAttValue(i) is DateTime value) return value; return default(DateTime); - // DateTime.TryParse(GetAttValue(i).ToString(), null, DateTimeStyles.AdjustToUniversal, out DateTime dt); - // return dt; } public decimal GetDecimal(int i) { diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDatastoreHelper.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDatastoreHelper.cs index 6430baf2b..0b8e921ce 100644 --- a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDatastoreHelper.cs +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDatastoreHelper.cs @@ -57,11 +57,11 @@ public class CosmosDBQuery : Query { public string Index { get; set; } - public override Query OrderBy(string index) - { + //public override Query OrderBy(string index) + //{ ////TO DO/// - return this; - } + //return this; + //} public string PartitionKey { get; private set; } public override Query SetKey(string partitionKey) From 7c7bdd2cacf2937206fa19e599cf985323fae1ca Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Wed, 4 Jan 2023 20:35:59 -0300 Subject: [PATCH 20/25] fix error with params --- dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBHelper.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBHelper.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBHelper.cs index ac141aacd..03bc924a8 100644 --- a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBHelper.cs +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBHelper.cs @@ -15,8 +15,8 @@ internal static bool AddItemValue(string parmName, string fromName, Dictionary v.Name == $":{fromName}"); if (varValue != null) { - values[parmName] = varValue; - jsonObject.Add(parmName, JsonValue.Create(varValue)); + values[parmName] = varValue.Value; + jsonObject.Add(parmName, JsonValue.Create(varValue.Value)); } return varValue != null; } From 77a0c592c3da2e68329402c08396b575cb0f520f Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Fri, 6 Jan 2023 12:50:20 -0300 Subject: [PATCH 21/25] Support inserting null value --- .../DynService/Cosmos/CosmosDBConnection.cs | 4 ---- .../DynService/Cosmos/CosmosDBHelper.cs | 18 ++++++++++++++++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs index c9546ca45..ed1ba06b6 100644 --- a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Data; using System.Data.Common; -using System.Globalization; using System.IO; using System.Linq; using System.Text; @@ -69,7 +68,6 @@ public override string ConnectionString } private static void InitializeDBConnection() { - //System.Diagnostics.Debugger.Launch(); DbConnectionStringBuilder builder = new DbConnectionStringBuilder(false); builder.ConnectionString = mConnectionString; @@ -426,7 +424,6 @@ private VarValue GetDataEqualParameterfromQueryVars(string filter, IEnumerable v.Name == $":{fromName}"); if (varValue != null) { + if (varValue.Value == DBNull.Value) + { + KeyValuePair keyvalue = new KeyValuePair(parmName, null); + jsonObject.Add(keyvalue); + } + else + jsonObject.Add(parmName, JsonValue.Create(varValue.Value)); values[parmName] = varValue.Value; - jsonObject.Add(parmName, JsonValue.Create(varValue.Value)); } return varValue != null; } @@ -40,8 +48,14 @@ internal static bool AddItemValue(string parmName, Dictionary dy return false; if (parm.Value != null) { + if (parm.Value == DBNull.Value) + { + KeyValuePair keyvalue = new KeyValuePair(parmName, null); + jsonObject.Add(keyvalue); + } + else + jsonObject.Add(parmName, JsonValue.Create(parm.Value)); dynParm[parmName] = parm.Value; - jsonObject.Add(parmName, JsonValue.Create(parm.Value)); return true; } return false; From b6c2de993e49823039b6a01e3e2cbf3905dbb920 Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Fri, 6 Jan 2023 15:15:04 -0300 Subject: [PATCH 22/25] remove comments from code --- .../src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs | 2 -- .../src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs | 1 - .../dotnetcore/DynService/Cosmos/CosmosDBDatastoreHelper.cs | 6 ------ dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBHelper.cs | 1 - dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBMaps.cs | 1 - .../dotnetcore/DynService/Cosmos/CosmosDBRequestWrapper.cs | 3 --- 6 files changed, 14 deletions(-) diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs index ed1ba06b6..bd936012c 100644 --- a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs @@ -175,7 +175,6 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo string jsonData = string.Empty; string partitionKey = query.PartitionKey; - //object partitionKeyValue; JsonObject jsonObject = new JsonObject(); Dictionary keyCondition = new Dictionary(); @@ -274,7 +273,6 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo Task task = Task.Run(async () => await container.DeleteItemStreamAsync(idField.ToString(), ToPartitionKey(keyCondition[partitionKey])).ConfigureAwait(false)); if (task.Result.IsSuccessStatusCode) { - //ResponseMessage wrapps the delete record return 1; } else diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs index 56dbc99e5..355997dc2 100644 --- a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs @@ -6,7 +6,6 @@ using System.Linq; using System.Text.Json; using System.Threading.Tasks; -using GeneXus.Configuration; using GeneXus.Data.NTier; using GeneXus.Data.NTier.CosmosDB; using log4net; diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDatastoreHelper.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDatastoreHelper.cs index 0b8e921ce..6f9964777 100644 --- a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDatastoreHelper.cs +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDatastoreHelper.cs @@ -57,12 +57,6 @@ public class CosmosDBQuery : Query { public string Index { get; set; } - //public override Query OrderBy(string index) - //{ - ////TO DO/// - //return this; - //} - public string PartitionKey { get; private set; } public override Query SetKey(string partitionKey) { diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBHelper.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBHelper.cs index e9d72f6dd..62495d18e 100644 --- a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBHelper.cs +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBHelper.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Data; using System.Linq; -using System.Text.Json; using System.Text.Json.Nodes; using GeneXus.Data.NTier; diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBMaps.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBMaps.cs index de8b5311d..b7ea2bac8 100644 --- a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBMaps.cs +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBMaps.cs @@ -8,7 +8,6 @@ public class CosmosDBMap : Map { internal bool NeedsAttributeMap { get; } - //TODO public CosmosDBMap(string name): base(name) { diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBRequestWrapper.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBRequestWrapper.cs index 5e7d52fe8..0dd7d4bf7 100644 --- a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBRequestWrapper.cs +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBRequestWrapper.cs @@ -56,9 +56,6 @@ private async Task ReadItemAsyncByPK(string idValue, string par using (ResponseMessage responseMessage = await m_container.ReadItemStreamAsync( partitionKey: new PartitionKey(partitionKeyValue), id: idValue).ConfigureAwait(false)) - //ResponseMessage responseMessage = await m_container.ReadItemStreamAsync( - //partitionKey: new PartitionKey(partitionKeyValue), - //id: idValue).ConfigureAwait(false); { if (!responseMessage.IsSuccessStatusCode) From 76cb3fa474aff6e8b5ab064a11c057f17763efed Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Sun, 8 Jan 2023 14:44:40 -0300 Subject: [PATCH 23/25] remove comments --- .../DynService/Cosmos/CosmosDBConnection.cs | 25 +++---------------- .../DynService/Cosmos/CosmosDBDataReader.cs | 3 +-- .../DynService/Cosmos/CosmosDBMaps.cs | 5 +--- .../Cosmos/CosmosDBResponseWrapper.cs | 4 +-- 4 files changed, 6 insertions(+), 31 deletions(-) diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs index bd936012c..52e338221 100644 --- a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs @@ -18,14 +18,7 @@ namespace GeneXus.Data.NTier public class CosmosDBService : GxService { - public CosmosDBService(string id, string providerId) : base(id, providerId, typeof(CosmosDBConnection)) - { - - } - /*public override IDataReader GetCacheDataReader(CacheItem item, bool computeSize, string keyCache) - { - return new GxDynamoDBCacheDataReader(item, computeSize, keyCache); - }*/ + public CosmosDBService(string id, string providerId) : base(id, providerId, typeof(CosmosDBConnection)){} } public class CosmosDBConnection : ServiceConnection @@ -49,9 +42,6 @@ public class CosmosDBConnection : ServiceConnection //private const string DISTINCT = "DISTINCT"; static readonly ILog logger = log4net.LogManager.GetLogger(typeof(CosmosDBConnection)); - - //TODO: Usar un Hashset para guardar los containers - public override string ConnectionString { get @@ -103,7 +93,6 @@ private static void Initialize() private PartitionKey ToPartitionKey(object value) { - if (value is double) return new PartitionKey((double)value); if (value is bool) @@ -111,8 +100,7 @@ private PartitionKey ToPartitionKey(object value) if (value is string) return new PartitionKey((string)value); else - throw new Exception("Partitionkey can be double, bool or string."); - + throw new Exception("Partitionkey can be double, bool or string."); } private Container GetContainer(string containerName) { @@ -202,7 +190,6 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo { keyFilterS = keyFilter; condition = keyFilter; - MatchCollection matchCollection = Regex.Matches(keyFilterS, regex1); foreach (Match match in matchCollection) { @@ -236,18 +223,14 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo jsonObject.Add(name, JsonValue.Create(serviceParm.Value)); if (isUpdate && name == partitionKey && partitionKey != "id") - jsonObject.Add(name, JsonValue.Create(serviceParm.Value)); - + jsonObject.Add(name, JsonValue.Create(serviceParm.Value)); } } } } } } - jsonData = jsonObject.ToJsonString(); - - //TODO: Get container from HashSet for performance Container container = GetContainer(query.TableName); switch (query.CursorType) { @@ -391,7 +374,6 @@ public override IDataReader ExecuteReader(ServiceCursorDef cursorDef, IDataParam Initialize(); CosmosDBQuery query = cursorDef.Query as CosmosDBQuery; - //Get container from hashset for performance Container container = GetContainer(query?.TableName); try { @@ -400,7 +382,6 @@ public override IDataReader ExecuteReader(ServiceCursorDef cursorDef, IDataParam } catch (CosmosException cosmosException) { - //TODO: Handle cases throw cosmosException; } diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs index 355997dc2..55ba51527 100644 --- a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs @@ -279,8 +279,7 @@ private async Task GetPage() try { using (ResponseMessage response = await m_feedIterator.ReadNextAsync().ConfigureAwait(false)) - { - + { if (!response.IsSuccessStatusCode) { if (response.Diagnostics != null) diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBMaps.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBMaps.cs index b7ea2bac8..235f861fe 100644 --- a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBMaps.cs +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBMaps.cs @@ -8,10 +8,7 @@ public class CosmosDBMap : Map { internal bool NeedsAttributeMap { get; } - public CosmosDBMap(string name): base(name) - { - - } + public CosmosDBMap(string name): base(name){} public override object GetValue(IOServiceContext context, RecordEntryRow currentEntry) { Dictionary values = ((CosmosDBRecordEntry)currentEntry).CurrentRow; diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBResponseWrapper.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBResponseWrapper.cs index c3da9e631..717935ddb 100644 --- a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBResponseWrapper.cs +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBResponseWrapper.cs @@ -6,7 +6,6 @@ namespace GeneXus.Data.NTier.CosmosDB { public class ResponseWrapper { - public FeedIterator feedIterator; public Stream stream; @@ -30,7 +29,6 @@ public ResponseWrapper(List> responseItems) ItemCount = responseItems.Count; } public List> Items { get; set; } - public int ItemCount { get; set; } - + public int ItemCount { get; set; } } } From bdd21bdc18049b114858b7eab1e53243e71d3033 Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Tue, 10 Jan 2023 11:49:37 -0300 Subject: [PATCH 24/25] fix connectionstring options --- .../DynService/Cosmos/CosmosDBConnection.cs | 58 +++++++++++-------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs index 52e338221..9e35a3b4c 100644 --- a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs @@ -27,15 +27,15 @@ public class CosmosDBConnection : ServiceConnection private const string DATABASE = "database"; private const string SERVICE_URI = "serviceURI"; private const string ACCOUNT_KEY = "AccountKey"; + private const string ACCOUNT_DATASOURCE = "data source"; private static CosmosClient cosmosClient; private static Database cosmosDatabase; private static string mapplicationRegion; private static string mdatabase; private static string mAccountKey; + private static string mAccountEndpoint; private static string mserviceURI; private static string mConnectionString; - - //https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/query/select private const string TABLE_ALIAS = "t"; //Options not supported by the spec yet @@ -69,6 +69,11 @@ private static void InitializeDBConnection() { mapplicationRegion = region.ToString(); } + if (string.IsNullOrEmpty(mserviceURI) && (builder.TryGetValue(ACCOUNT_DATASOURCE, out object accountEndpoint))) + { + mAccountEndpoint = accountEndpoint.ToString(); + mserviceURI = mAccountEndpoint; + } if (builder.TryGetValue(ACCOUNT_KEY, out object accountKey)) { mAccountKey = accountKey.ToString(); @@ -78,17 +83,23 @@ private static void InitializeDBConnection() { mdatabase = database.ToString(); } - //TODO: check Mandatory parameters - //TODO: Connect using connection string + connection key } private static void Initialize() { - if (!string.IsNullOrEmpty(mserviceURI) && !string.IsNullOrEmpty(mapplicationRegion)) + if (!string.IsNullOrEmpty(mserviceURI) && !string.IsNullOrEmpty(mapplicationRegion) && (!string.IsNullOrEmpty(mdatabase))) cosmosClient = new CosmosClient(mserviceURI, new CosmosClientOptions() { ApplicationRegion = mapplicationRegion }); - - if (!string.IsNullOrEmpty(mdatabase)) - cosmosDatabase = cosmosClient.GetDatabase(mdatabase); + else + { + if (string.IsNullOrEmpty(mapplicationRegion)) + throw new Exception("Application Region is a mandatory additional connection string attribute."); + else + if (string.IsNullOrEmpty(mdatabase)) + throw new Exception("Database is a mandatory additional connection string attribute."); + else + throw new Exception("Connection string is not set or is not valid. Unable to connect."); + } + cosmosDatabase = cosmosClient.GetDatabase(mdatabase); } private PartitionKey ToPartitionKey(object value) @@ -375,17 +386,24 @@ public override IDataReader ExecuteReader(ServiceCursorDef cursorDef, IDataParam Initialize(); CosmosDBQuery query = cursorDef.Query as CosmosDBQuery; Container container = GetContainer(query?.TableName); - try + if (container == null) { - CreateCosmosQuery(query,cursorDef, parms, container, out CosmosDBDataReader dataReader, out RequestWrapper requestWrapper); - return dataReader; + GXLogging.Error(logger, "Container not found."); + throw new Exception("Container not found."); } - catch (CosmosException cosmosException) - { - throw cosmosException; + else + { + try + { + CreateCosmosQuery(query,cursorDef, parms, container, out CosmosDBDataReader dataReader, out RequestWrapper requestWrapper); + return dataReader; + } + catch (CosmosException cosmosException) + { + throw cosmosException; + } + catch (Exception e) { throw e; } } - - catch (Exception e) { throw e; } } private VarValue GetDataEqualParameterfromQueryVars(string filter, IEnumerable values, out string name) @@ -428,8 +446,7 @@ private CosmosDBDataReader GetDataReaderQueryByPK(ServiceCursorDef cursorDef, Co GXLogging.Debug(logger,$"Execute PK query id = {requestWrapper.idValue}, partitionKey = {requestWrapper.partitionKeyValue}"); requestWrapper.queryByPK = true; - return new CosmosDBDataReader(cursorDef, requestWrapper); - + return new CosmosDBDataReader(cursorDef, requestWrapper); } /// @@ -563,9 +580,4 @@ private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, I } internal static IOServiceContext NewServiceContext() => null; } - public class CosmosDBErrors - { - public const string ValidationException = "ValidationException"; - public const string ValidationExceptionMessageKey = "The AttributeValue for a key attribute cannot contain an empty string value."; - } } From 5f4000026b9d581dda18eb508128b7f36f35e3f0 Mon Sep 17 00:00:00 2001 From: sjuarezgx Date: Tue, 10 Jan 2023 12:51:36 -0300 Subject: [PATCH 25/25] support like using wildcard characters --- .../DynService/Cosmos/CosmosDBConnection.cs | 17 +++-------------- .../DynService/Cosmos/CosmosDBDataReader.cs | 1 - 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs index 9e35a3b4c..35b8de3e2 100644 --- a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs @@ -256,7 +256,6 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo { if (!keyCondition.Any() || !keyCondition.ContainsKey("id") || !keyCondition.ContainsKey(partitionKey)) { - GXLogging.Debug(logger,"Delete item failed: error parsing the query."); throw new Exception($"Delete item failed: error parsing the query."); } else @@ -273,12 +272,10 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo { if (task.Result.ErrorMessage.Contains("404")) { - GXLogging.Debug(logger, ServiceError.RecordNotFound); throw new ServiceException(ServiceError.RecordNotFound, null); } else { - GXLogging.Error(logger,$"Delete item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); throw new Exception($"Delete item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); } } @@ -289,7 +286,6 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo } else { - GXLogging.Error(logger,"CosmosDB Delete Execution failed. Container not found."); throw new Exception("CosmosDB Delete Execution failed. Container not found."); } @@ -300,7 +296,6 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo { if (!keyCondition.Any() || !keyCondition.ContainsKey(partitionKey)) { - GXLogging.Error(logger,$"Insert item failed: error parsing the query."); throw new Exception($"Insert item failed: error parsing the query."); } else @@ -316,12 +311,10 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo { if (task.Result.ErrorMessage.Contains("Conflict (409)")) { - GXLogging.Debug(logger,ServiceError.RecordAlreadyExists); throw new ServiceException(ServiceError.RecordAlreadyExists, null); } else { - GXLogging.Error(logger,$"Create item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); throw new Exception($"Create item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); } } @@ -336,7 +329,6 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo } else { - GXLogging.Error(logger,"CosmosDB Insert Execution failed. Container not found."); throw new Exception("CosmosDB Insert Execution failed. Container not found."); } case ServiceCursorDef.CursorType.Update: @@ -344,7 +336,6 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo { if (!keyCondition.Any() || !keyCondition.ContainsKey("id") || !keyCondition.ContainsKey(partitionKey)) { - GXLogging.Error(logger, $"Update item failed: error parsing the query."); throw new Exception($"Update item failed: error parsing the query."); } else @@ -360,7 +351,6 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo return 1; else { - GXLogging.Error(logger, $"Update item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); throw new Exception($"Update item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); } } @@ -373,7 +363,6 @@ public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCo } else { - GXLogging.Error(logger,"CosmosDB Update Execution failed. Container not found."); throw new Exception("CosmosDB Update Execution failed. Container not found."); } } @@ -388,7 +377,6 @@ public override IDataReader ExecuteReader(ServiceCursorDef cursorDef, IDataParam Container container = GetContainer(query?.TableName); if (container == null) { - GXLogging.Error(logger, "Container not found."); throw new Exception("Container not found."); } else @@ -510,6 +498,9 @@ private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, I foreach (string keyFilter in allFilters) { string filterProcess = keyFilter.ToString(); + filterProcess = filterProcess.Replace("[", "("); + filterProcess = filterProcess.Replace("]", ")"); + foreach (VarValue item in query.Vars) { string varValuestr = string.Empty; @@ -545,8 +536,6 @@ private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, I } filterProcess = filterProcess.Replace("Func.", ""); - filterProcess = filterProcess.Replace("[", "("); - filterProcess = filterProcess.Replace("]", ")"); foreach (string d in projection) { string wholeWordPattern = String.Format(@"\b{0}\b", d); diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs index 55ba51527..2f50338d4 100644 --- a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs @@ -316,7 +316,6 @@ private async Task GetPage() } catch (CosmosException ex) { - GXLogging.Error(logger, ex); throw ex; } }