diff --git a/dotnet/DotNetStandardClasses.sln b/dotnet/DotNetStandardClasses.sln index 228e16a5e..f212f1a4f 100644 --- a/dotnet/DotNetStandardClasses.sln +++ b/dotnet/DotNetStandardClasses.sln @@ -224,6 +224,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GeneXus.OpenTelemetry.AWS.A EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GeneXus.OpenTelemetry", "src\dotnetcore\Providers\OpenTelemetry\OpenTelemetry\GeneXus.OpenTelemetry.csproj", "{00B1FA38-7D0B-47E4-860C-23490249A4D6}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DynService.CosmosDB", "src\dotnetcore\DynService\Cosmos\DynService.CosmosDB.csproj", "{52DC6C43-58ED-4310-996B-06E95105F848}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -542,6 +544,10 @@ Global {00B1FA38-7D0B-47E4-860C-23490249A4D6}.Debug|Any CPU.Build.0 = Debug|Any CPU {00B1FA38-7D0B-47E4-860C-23490249A4D6}.Release|Any CPU.ActiveCfg = Release|Any CPU {00B1FA38-7D0B-47E4-860C-23490249A4D6}.Release|Any CPU.Build.0 = Release|Any CPU + {52DC6C43-58ED-4310-996B-06E95105F848}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {52DC6C43-58ED-4310-996B-06E95105F848}.Debug|Any CPU.Build.0 = Debug|Any CPU + {52DC6C43-58ED-4310-996B-06E95105F848}.Release|Any CPU.ActiveCfg = Release|Any CPU + {52DC6C43-58ED-4310-996B-06E95105F848}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -650,6 +656,7 @@ Global {27D54041-BDD5-428E-8CA6-C96D519F5451} = {BBE020D4-C0FF-41A9-9EB1-D1EE12CC4BB8} {B5A9DEA7-67EC-49E4-924E-4729C34286EC} = {BBE020D4-C0FF-41A9-9EB1-D1EE12CC4BB8} {00B1FA38-7D0B-47E4-860C-23490249A4D6} = {BBE020D4-C0FF-41A9-9EB1-D1EE12CC4BB8} + {52DC6C43-58ED-4310-996B-06E95105F848} = {79C9ECC6-2935-4C43-BF32-94698547F584} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {E18684C9-7D76-45CD-BF24-E3944B7F174C} diff --git a/dotnet/src/dotnetcommon/DynService.Core/DynServiceCommon.cs b/dotnet/src/dotnetcommon/DynService.Core/DynServiceCommon.cs index 695113439..17965e3ab 100644 --- a/dotnet/src/dotnetcommon/DynService.Core/DynServiceCommon.cs +++ b/dotnet/src/dotnetcommon/DynService.Core/DynServiceCommon.cs @@ -79,6 +79,19 @@ public Query AddConst(GXType gxType, object parm) mVarValues.Add(new VarValue($":const{ mVarValues.Count + 1 }", gxType, parm)); return this; } + + public virtual Query SetKey(string key) + { + return null; + } + public virtual Query KeyFilter(string[] filters) + { + return null; + } + public virtual Query OrderBy(string index) + { + return null; + } } public class VarValue diff --git a/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBConnection.cs b/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBConnection.cs index 240d2a052..18825e036 100644 --- a/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBConnection.cs +++ b/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBConnection.cs @@ -75,7 +75,6 @@ private void Initialize() { InitializeDBConnection(); State = ConnectionState.Executing; - mDynamoDB = new AmazonDynamoDBClient(mCredentials, mConfig); } diff --git a/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBDataStoreHelper.cs b/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBDataStoreHelper.cs index 0c3cb20b8..d28a2ddd1 100644 --- a/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBDataStoreHelper.cs +++ b/dotnet/src/dotnetcommon/DynService.Dynamo/DynamoDBDataStoreHelper.cs @@ -55,22 +55,6 @@ public object empty(GXType gxtype) } } } - - public static class DynamoFluentExtensions - { - public static Query OrderBy(this Query dynamoQuery, string index) - { - return (dynamoQuery as DynamoQuery)?.OrderBy(index); - } - public static Query SetKey(this Query dynamoQuery, string partitionKey) - { - return (dynamoQuery as DynamoQuery)?.SetKey(partitionKey); - } - public static Query KeyFilter(this Query dynamoQuery, string[] filters) - { - return (dynamoQuery as DynamoQuery)?.KeyFilter(filters); - } - } public class DynamoQuery : Query { @@ -79,7 +63,7 @@ public class DynamoQuery : Query private const string RANGE_KEY_INDEX = "RangeKey"; private static readonly char[] indexTrimChars = new char[] { '(', ')' }; - public DynamoQuery OrderBy(string index) + public override Query OrderBy(string index) { if(index.StartsWith("(", StringComparison.InvariantCulture)) { @@ -92,18 +76,17 @@ public DynamoQuery OrderBy(string index) } public string PartitionKey { get; private set; } - public DynamoQuery SetKey(string partitionKey) + public override Query SetKey(string partitionKey) { PartitionKey = partitionKey; return this; } internal IEnumerable KeyFilters { get; set; } = Array.Empty(); - public DynamoQuery KeyFilter(string[] filters) + public override Query KeyFilter(string[] filters) { KeyFilters = filters; return this; } - public DynamoQuery(DynamoDBDataStoreHelper dataStoreHelper) : base(dataStoreHelper) { } } } diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs new file mode 100644 index 000000000..35b8de3e2 --- /dev/null +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBConnection.cs @@ -0,0 +1,572 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.Common; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using GeneXus.Data.Cosmos; +using GeneXus.Data.NTier.CosmosDB; +using log4net; +using Microsoft.Azure.Cosmos; + +namespace GeneXus.Data.NTier +{ + + public class CosmosDBService : GxService + { + public CosmosDBService(string id, string providerId) : base(id, providerId, typeof(CosmosDBConnection)){} + } + + public class CosmosDBConnection : ServiceConnection + { + private const string REGION = "ApplicationRegion"; + private const string DATABASE = "database"; + private const string SERVICE_URI = "serviceURI"; + private const string ACCOUNT_KEY = "AccountKey"; + private const string ACCOUNT_DATASOURCE = "data source"; + private static CosmosClient cosmosClient; + private static Database cosmosDatabase; + private static string mapplicationRegion; + private static string mdatabase; + private static string mAccountKey; + private static string mAccountEndpoint; + private static string mserviceURI; + private static string mConnectionString; + private const string TABLE_ALIAS = "t"; + + //Options not supported by the spec yet + //private const string DISTINCT = "DISTINCT"; + + static readonly ILog logger = log4net.LogManager.GetLogger(typeof(CosmosDBConnection)); + public override string ConnectionString + { + get + { + return mConnectionString; + } + + set + { + mConnectionString = value; + State = ConnectionState.Executing; + InitializeDBConnection(); + } + } + private static void InitializeDBConnection() + { + DbConnectionStringBuilder builder = new DbConnectionStringBuilder(false); + builder.ConnectionString = mConnectionString; + + if (builder.TryGetValue(SERVICE_URI, out object serviceURI)) + { + mserviceURI = serviceURI.ToString(); + } + if (builder.TryGetValue(REGION, out object region)) + { + mapplicationRegion = region.ToString(); + } + if (string.IsNullOrEmpty(mserviceURI) && (builder.TryGetValue(ACCOUNT_DATASOURCE, out object accountEndpoint))) + { + mAccountEndpoint = accountEndpoint.ToString(); + mserviceURI = mAccountEndpoint; + } + if (builder.TryGetValue(ACCOUNT_KEY, out object accountKey)) + { + mAccountKey = accountKey.ToString(); + mserviceURI = $"{mserviceURI};AccountKey={mAccountKey}"; + } + if (builder.TryGetValue(DATABASE, out object database)) + { + mdatabase = database.ToString(); + } + } + + private static void Initialize() + { + if (!string.IsNullOrEmpty(mserviceURI) && !string.IsNullOrEmpty(mapplicationRegion) && (!string.IsNullOrEmpty(mdatabase))) + cosmosClient = new CosmosClient(mserviceURI, new CosmosClientOptions() { ApplicationRegion = mapplicationRegion }); + else + { + if (string.IsNullOrEmpty(mapplicationRegion)) + throw new Exception("Application Region is a mandatory additional connection string attribute."); + else + if (string.IsNullOrEmpty(mdatabase)) + throw new Exception("Database is a mandatory additional connection string attribute."); + else + throw new Exception("Connection string is not set or is not valid. Unable to connect."); + } + cosmosDatabase = cosmosClient.GetDatabase(mdatabase); + } + + private PartitionKey ToPartitionKey(object value) + { + if (value is double) + return new PartitionKey((double)value); + if (value is bool) + return new PartitionKey((bool)value); + if (value is string) + return new PartitionKey((string)value); + else + throw new Exception("Partitionkey can be double, bool or string."); + } + private Container GetContainer(string containerName) + { + if (cosmosDatabase != null && !string.IsNullOrEmpty(containerName)) + return cosmosClient.GetContainer(cosmosDatabase.Id, containerName); + return null; + } + private string SetupQuery(string projectionList, string filterExpression, string tableName, string orderbys) + { + string sqlSelect = string.Empty; + string sqlFrom = string.Empty; + string sqlWhere = string.Empty; + string sqlOrder = string.Empty; + + string SELECT_TEMPLATE = "select {0}"; + string FROM_TEMPLATE = "from {0} t"; + string WHERE_TEMPLATE = "where {0}"; + string ORDER_TEMPLATE = "order by {0}"; + + if (!string.IsNullOrEmpty(projectionList)) + sqlSelect = string.Format(SELECT_TEMPLATE, projectionList); + else + { + throw new Exception("Error setting up the query. Projection list is empty."); + } + if (!string.IsNullOrEmpty(tableName)) + sqlFrom = string.Format(FROM_TEMPLATE, tableName); + else + { + throw new Exception("Error setting up the query. Table name is empty."); + } + if (!string.IsNullOrEmpty(filterExpression)) + sqlWhere = string.Format(WHERE_TEMPLATE, filterExpression); + if (!string.IsNullOrEmpty(orderbys)) + sqlOrder = string.Format(ORDER_TEMPLATE, orderbys); + + return $"{sqlSelect} {sqlFrom} {sqlWhere} {sqlOrder}"; + } + + /// + /// Execute insert, update and delete queries. + /// + /// + /// + /// + /// + /// + /// + public override int ExecuteNonQuery(ServiceCursorDef cursorDef, IDataParameterCollection parms, CommandBehavior behavior) + { + Initialize(); + CosmosDBQuery query = cursorDef.Query as CosmosDBQuery; + if (query == null) + return 0; + + bool isInsert = query.CursorType == ServiceCursorDef.CursorType.Insert; + bool isUpdate = query.CursorType == ServiceCursorDef.CursorType.Update; + + Dictionary values = new Dictionary(); + string jsonData = string.Empty; + + string partitionKey = query.PartitionKey; + JsonObject jsonObject = new JsonObject(); + + Dictionary keyCondition = new Dictionary(); + + //Setup the json payload to execute the insert or update query. + foreach (KeyValuePair asg in query.AssignAtts) + { + string name = asg.Key; + string parmName = asg.Value.Substring(1).Remove(asg.Value.Length - 2); + CosmosDBHelper.AddItemValue(name, parmName, values, parms, query.Vars, ref jsonObject); + if (name == partitionKey) + keyCondition[partitionKey] = values[name]; + } + + //Get the values for id and partitionKey + string regex1 = @"\(([^\)\(]+)\)"; + string regex2 = @"(.*)[^<>!=]\s*(=|!=|<|>|<=|>=|<>)\s*(:.*:)"; + + string keyFilterS; + string condition = string.Empty; + IEnumerable keyFilterQ = Array.Empty(); + IEnumerable allFilters = query.KeyFilters.Concat(query.Filters); + + foreach (string keyFilter in allFilters) + { + keyFilterS = keyFilter; + condition = keyFilter; + MatchCollection matchCollection = Regex.Matches(keyFilterS, regex1); + foreach (Match match in matchCollection) + { + if (match.Groups.Count > 0) + { + string cond = match.Groups[1].Value; + Match match2 = Regex.Match(cond, regex2); + if (match2.Success) + { + string varName = match2.Groups[3].Value; + varName = varName.Remove(varName.Length - 1).Substring(1); + string name = match2.Groups[1].Value; + VarValue varValue = query.Vars.FirstOrDefault(v => v.Name == $":{varName}"); + + if (varValue != null) + { + keyCondition[name] = varValue.Value; + if (isUpdate && name == "id") + jsonObject.Add(name, JsonValue.Create(varValue.Value)); + + if (isUpdate && name == partitionKey && partitionKey != "id") + jsonObject.Add(name, JsonValue.Create(varValue.Value)); + } + else + { + if (parms[varName] is ServiceParameter serviceParm) + { + keyCondition[name] = serviceParm.Value; + + if (isUpdate && name == "id") + jsonObject.Add(name, JsonValue.Create(serviceParm.Value)); + + if (isUpdate && name == partitionKey && partitionKey != "id") + jsonObject.Add(name, JsonValue.Create(serviceParm.Value)); + } + } + } + } + } + } + jsonData = jsonObject.ToJsonString(); + Container container = GetContainer(query.TableName); + switch (query.CursorType) + { + case ServiceCursorDef.CursorType.Select: + throw new NotImplementedException(); + + case ServiceCursorDef.CursorType.Delete: + + if (container != null) + { + try + { + if (!keyCondition.Any() || !keyCondition.ContainsKey("id") || !keyCondition.ContainsKey(partitionKey)) + { + throw new Exception($"Delete item failed: error parsing the query."); + } + else + { + object idField = keyCondition["id"]; + + GXLogging.Debug(logger,$"Delete : id= {idField.ToString()}, partitionKey= {keyCondition[partitionKey].ToString()}"); + Task task = Task.Run(async () => await container.DeleteItemStreamAsync(idField.ToString(), ToPartitionKey(keyCondition[partitionKey])).ConfigureAwait(false)); + if (task.Result.IsSuccessStatusCode) + { + return 1; + } + else + { + if (task.Result.ErrorMessage.Contains("404")) + { + throw new ServiceException(ServiceError.RecordNotFound, null); + } + else + { + throw new Exception($"Delete item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); + } + } + } + } + catch (Exception ex) + { throw ex; } + } + else + { + throw new Exception("CosmosDB Delete Execution failed. Container not found."); + } + + case ServiceCursorDef.CursorType.Insert: + if (container != null) + { + try + { + if (!keyCondition.Any() || !keyCondition.ContainsKey(partitionKey)) + { + throw new Exception($"Insert item failed: error parsing the query."); + } + else + { + GXLogging.Debug(logger,$"Insert : {jsonData}"); + using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(jsonData))) + { + PartitionKey p = ToPartitionKey(keyCondition[partitionKey]); + Task task = Task.Run(async () => await container.CreateItemStreamAsync(stream, ToPartitionKey(keyCondition[partitionKey])).ConfigureAwait(false)); + if (task.Result.IsSuccessStatusCode) + return 1; + else + { + if (task.Result.ErrorMessage.Contains("Conflict (409)")) + { + throw new ServiceException(ServiceError.RecordAlreadyExists, null); + } + else + { + throw new Exception($"Create item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); + } + } + } + } + + } + catch (Exception ex) + { + throw ex; + } + } + else + { + throw new Exception("CosmosDB Insert Execution failed. Container not found."); + } + case ServiceCursorDef.CursorType.Update: + if (container != null) + { + if (!keyCondition.Any() || !keyCondition.ContainsKey("id") || !keyCondition.ContainsKey(partitionKey)) + { + throw new Exception($"Update item failed: error parsing the query."); + } + else + { + try + { + GXLogging.Debug(logger,$"Update : {jsonData}"); + + using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(jsonData))) + { + Task task = Task.Run(async () => await container.UpsertItemStreamAsync(stream, ToPartitionKey(keyCondition[partitionKey])).ConfigureAwait(false)); + if (task.Result.IsSuccessStatusCode) + return 1; + else + { + throw new Exception($"Update item from stream failed. Status code: {task.Result.StatusCode}. Message: {task.Result.ErrorMessage}"); + } + } + } + catch (Exception ex) + { + throw ex; + } + } + } + else + { + throw new Exception("CosmosDB Update Execution failed. Container not found."); + } + } + return 0; + + } + public override IDataReader ExecuteReader(ServiceCursorDef cursorDef, IDataParameterCollection parms, CommandBehavior behavior) + { + + Initialize(); + CosmosDBQuery query = cursorDef.Query as CosmosDBQuery; + Container container = GetContainer(query?.TableName); + if (container == null) + { + throw new Exception("Container not found."); + } + else + { + try + { + CreateCosmosQuery(query,cursorDef, parms, container, out CosmosDBDataReader dataReader, out RequestWrapper requestWrapper); + return dataReader; + } + catch (CosmosException cosmosException) + { + throw cosmosException; + } + catch (Exception e) { throw e; } + } + } + + private VarValue GetDataEqualParameterfromQueryVars(string filter, IEnumerable values, out string name) + { + string Equal_Filter_pattern = @"\((.*) = :(.*)\)"; + VarValue varValue = null; + name = string.Empty; + Match match = Regex.Match(filter, Equal_Filter_pattern); + if (match.Groups.Count > 1) + { + string varName = match.Groups[2].Value; + varName = varName.Remove(varName.Length - 1); + name = match.Groups[1].Value; + varValue = values.FirstOrDefault(v => v.Name == $":{varName}"); + } + return varValue; + } + private string GetDataEqualParameterfromCollection(string filter, IDataParameterCollection parms, out string name) + { + string Equal_Filter_pattern = @"\((.*) = :(.*)\)"; + name = string.Empty; + Match match = Regex.Match(filter, Equal_Filter_pattern); + if (match.Groups.Count > 1) + { + string varName = match.Groups[2].Value; + name = match.Groups[1].Value; + varName = varName.Remove(varName.Length - 1); + if (parms[varName] is ServiceParameter serviceParm) + { + return serviceParm.Value.ToString(); + } + } + return string.Empty; + } + private CosmosDBDataReader GetDataReaderQueryByPK(ServiceCursorDef cursorDef, Container container, string idValue, string partitionKeyValue,out RequestWrapper requestWrapper) + { + requestWrapper = new RequestWrapper(cosmosClient, container, null); + requestWrapper.idValue = idValue; + requestWrapper.partitionKeyValue = partitionKeyValue; + + GXLogging.Debug(logger,$"Execute PK query id = {requestWrapper.idValue}, partitionKey = {requestWrapper.partitionKeyValue}"); + requestWrapper.queryByPK = true; + return new CosmosDBDataReader(cursorDef, requestWrapper); + } + + /// + /// Create object for querying the database. + /// + /// + /// + /// + /// + /// + /// + private void CreateCosmosQuery(CosmosDBQuery query,ServiceCursorDef cursorDef, IDataParameterCollection parms, Container container, out CosmosDBDataReader cosmosDBDataReader,out RequestWrapper requestWrapper) + { + //Check if the filters are the Primary Key + if (query.Filters.Any()) + { + if (query.Filters.Count() == 2) + { + string fieldValue1 = string.Empty; + string fieldValue2 = string.Empty; + + fieldValue1 = GetDataEqualParameterfromQueryVars(query.Filters.First(), query.Vars, out string fieldName1)?.Value.ToString(); + fieldValue1 = fieldValue1 ?? GetDataEqualParameterfromCollection(query.Filters.First(), parms, out fieldName1); + + fieldValue2 = GetDataEqualParameterfromQueryVars(query.Filters.Skip(1).First(), query.Vars, out string fieldName2)?.Value.ToString(); + fieldValue2 = fieldValue2 ?? GetDataEqualParameterfromCollection(query.Filters.Skip(1).First(), parms, out fieldName2); + + if (fieldName1 == "id" && fieldName2 == query.PartitionKey) + { + cosmosDBDataReader = GetDataReaderQueryByPK(cursorDef, container, fieldValue1, fieldValue2, out requestWrapper); + return; + } + else + { + if (fieldName1 == query.PartitionKey && fieldName2 == "id") + { + cosmosDBDataReader = GetDataReaderQueryByPK(cursorDef, container, fieldValue2, fieldValue1, out requestWrapper); + return; + } + } + } + } + //Create the query + string tableName = query.TableName; + IEnumerable projection = query.Projection; + string element; + string projectionList = string.Empty; + foreach (string key in projection) + { + element = $"{TABLE_ALIAS}.{key}"; + if (!string.IsNullOrEmpty(projectionList)) + projectionList = $"{element},{projectionList}"; + else + projectionList = $"{element}"; + } + + IEnumerable allFilters = query.KeyFilters.Concat(query.Filters); + IEnumerable allFiltersQuery = Array.Empty(); + IEnumerable keyFilterQ = Array.Empty(); + + foreach (string keyFilter in allFilters) + { + string filterProcess = keyFilter.ToString(); + filterProcess = filterProcess.Replace("[", "("); + filterProcess = filterProcess.Replace("]", ")"); + + foreach (VarValue item in query.Vars) + { + string varValuestr = string.Empty; + if (filterProcess.Contains(string.Format($"{item.Name}:"))) + { + if (GeneXus.Data.Cosmos.CosmosDBHelper.FormattedAsStringGXType(item.Type)) + varValuestr = '"' + $"{item.Value.ToString()}" + '"'; + else + { + varValuestr = item.Value.ToString(); + varValuestr = varValuestr.Equals("True") ? "true" : varValuestr; + varValuestr = varValuestr.Equals("False") ? "false" : varValuestr; + } + filterProcess = filterProcess.Replace(string.Format($"{item.Name}:"), varValuestr); + } + } + foreach (object p in parms) + { + if (p is ServiceParameter) + { + ServiceParameter p1 = (ServiceParameter)p; + string varValuestr = string.Empty; + if (filterProcess.Contains(string.Format($":{p1.ParameterName}:"))) + { + if (GeneXus.Data.Cosmos.CosmosDBHelper.FormattedAsStringDbType(p1.DbType)) + varValuestr = '"' + $"{p1.Value.ToString()}" + '"'; + else + varValuestr = p1.Value.ToString(); + + filterProcess = filterProcess.Replace(string.Format($":{p1.ParameterName}:"), varValuestr); + } + } + } + + filterProcess = filterProcess.Replace("Func.", ""); + foreach (string d in projection) + { + string wholeWordPattern = String.Format(@"\b{0}\b", d); + filterProcess = Regex.Replace(filterProcess, wholeWordPattern, $"{TABLE_ALIAS}.{d}"); + } + keyFilterQ = new string[] { filterProcess }; + allFiltersQuery = allFiltersQuery.Concat(keyFilterQ); + + } + string filterExpression = allFiltersQuery.Any() ? String.Join(" AND ", allFiltersQuery) : null; + + IEnumerable orderExpressionList = Array.Empty(); + string expression = string.Empty; + + foreach (string orderAtt in query.OrderBys) + { + expression = orderAtt.StartsWith("(") ? $"{TABLE_ALIAS}.{orderAtt.Remove(orderAtt.Length-1,1).Remove(0,1)} DESC" : $"{TABLE_ALIAS}.{orderAtt} ASC"; + orderExpressionList = orderExpressionList.Concat(new string[] { expression }); + } + + string orderExpression = String.Join(",", orderExpressionList); + string sqlQuery = SetupQuery(projectionList, filterExpression, tableName, orderExpression); + + GXLogging.Debug(logger,sqlQuery); + + QueryDefinition queryDefinition = new QueryDefinition(sqlQuery); + requestWrapper = new RequestWrapper(cosmosClient, container, queryDefinition); + requestWrapper.queryByPK = false; + + cosmosDBDataReader = new CosmosDBDataReader(cursorDef, requestWrapper); + } + internal static IOServiceContext NewServiceContext() => null; + } +} diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs new file mode 100644 index 000000000..2f50338d4 --- /dev/null +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDataReader.cs @@ -0,0 +1,348 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using GeneXus.Data.NTier; +using GeneXus.Data.NTier.CosmosDB; +using log4net; +using Microsoft.Azure.Cosmos; +using Newtonsoft.Json; + +namespace GeneXus.Data.Cosmos +{ + public class CosmosDBDataReader : IDataReader + { + private readonly RequestWrapper m_request; + private ResponseWrapper m_response; + private readonly IODataMap2[] m_selectList; + private FeedIterator m_feedIterator; + private CosmosDBRecordEntry m_currentEntry; + private int m_currentPosition; + + private int ItemCount; + private List> Items = null; + + static readonly ILog logger = log4net.LogManager.GetLogger(typeof(CosmosDBDataReader)); + private void CheckCurrentPosition() + { + if (m_currentEntry == null) + throw new ServiceException(ServiceError.RecordNotFound); + } + private void ProcessPKStream(Stream stream) + { + //Query by PK -> only one record + + if (stream != null) + { + using (StreamReader sr = new StreamReader(stream)) + using (JsonTextReader jtr = new JsonTextReader(sr)) + { + Newtonsoft.Json.JsonSerializer jsonSerializer = new Newtonsoft.Json.JsonSerializer(); + object array = jsonSerializer.Deserialize(jtr); + + Dictionary result = JsonConvert.DeserializeObject>(array.ToString()); + + //remove metadata + result.Remove("_rid"); + result.Remove("_self"); + result.Remove("_etag"); + result.Remove("_attachments"); + result.Remove("_ts"); + + Items = new List>(); + Items.Add(result); + + if (Items != null) + ItemCount = Items.Count; + else + ItemCount = 0; + } + } + } + public CosmosDBDataReader(ServiceCursorDef cursorDef, RequestWrapper request) + { + Query query = cursorDef.Query as Query; + if (query != null) + m_selectList = query.SelectList.ToArray(); + m_request = request; + m_response = m_request.Read(); + m_feedIterator = m_response.feedIterator; + if (m_feedIterator == null) + { + if (m_response != null) + { + Items = m_response.Items; + ItemCount = Items.Count; + m_currentPosition = -1; + } + } + } + public object this[string name] + { + get + { + throw new NotImplementedException(); + } + } + + public object this[int i] + { + get + { + throw new NotImplementedException(); + } + } + public int Depth + { + get + { + return 0; + } + } + + public int FieldCount + { + get + { + return m_selectList.Length; + } + } + public bool IsClosed + { + get + { + return false; + } + } + + public int RecordsAffected + { + get + { + return -1; + } + } + + public void Close() + { + if (m_request != null) + { + m_request.Close(); + } + } + public void Dispose() + { + } + public long getLong(int i) + { + return Convert.ToInt64(GetAttValue(i)); + } + + private object GetAttValue(int i) + { + return (m_selectList[i].GetValue(CosmosDBConnection.NewServiceContext(), m_currentEntry)); + } + public bool GetBoolean(int i) + { + if (GetAttValue(i) is bool value) + { + return value; + } + return false; + } + + public byte GetByte(int i) + { + return Convert.ToByte(GetAttValue(i)); + } + + public long GetBytes(int i, long fieldOffset, byte[] buffer, int bufferoffset, int length) + { + MemoryStream ms = (MemoryStream)GetAttValue(i); + if (ms == null) + return 0; + ms.Seek(fieldOffset, SeekOrigin.Begin); + return ms.Read(buffer, bufferoffset, length); + } + public char GetChar(int i) + { + return Convert.ToChar(GetAttValue(i)); + } + public long GetChars(int i, long fieldoffset, char[] buffer, int bufferoffset, int length) + { + throw new NotImplementedException(); + } + + public IDataReader GetData(int i) + { + throw new NotImplementedException(); + } + + public string GetDataTypeName(int i) + { + throw new NotImplementedException(); + } + + public DateTime GetDateTime(int i) + { + if (GetAttValue(i) is DateTime value) + return value; + return default(DateTime); + } + public decimal GetDecimal(int i) + { + return Convert.ToDecimal(GetAttValue(i), CultureInfo.InvariantCulture); + } + public double GetDouble(int i) + { + return Convert.ToDouble(GetAttValue(i),CultureInfo.InvariantCulture); + } + public Type GetFieldType(int i) + { + return m_selectList[i].GetValue(CosmosDBConnection.NewServiceContext(), m_currentEntry).GetType(); + } + + public float GetFloat(int i) + { + return Convert.ToSingle(GetAttValue(i),CultureInfo.InvariantCulture); + } + public Guid GetGuid(int i) + { + return new Guid(GetAttValue(i).ToString()); + } + + public short GetInt16(int i) + { + return Convert.ToInt16(GetAttValue(i)); + } + + public int GetInt32(int i) + { + return Convert.ToInt32(GetAttValue(i)); + } + + public long GetInt64(int i) + { + return Convert.ToInt64(GetAttValue(i)); ; + } + + public string GetName(int i) + { + return m_selectList[i].GetName(null); + } + + public int GetOrdinal(string name) + { + CheckCurrentPosition(); + int ordinal = m_currentEntry.CurrentRow.ToList().FindIndex(col => col.Key.ToLower() == name.ToLower()); + if (ordinal == -1) + throw new ArgumentOutOfRangeException(nameof(name)); + else return ordinal; + } + public DataTable GetSchemaTable() + { + throw new NotImplementedException(); + } + public string GetString(int i) + { + return GetAttValue(i).ToString(); + } + public int GetValues(object[] values) + { + System.Diagnostics.Debug.Assert(m_selectList.Length == values.Length, "Values mismatch"); + for (int i = 0; i < m_selectList.Length && i < values.Length; i++) + { + values[i] = GetAttValue(i); + } + return m_selectList.Length; + } + public bool IsDBNull(int i) + { + return GetAttValue(i) == null; + } + + public bool NextResult() + { + m_currentPosition++; + m_currentEntry = (m_currentPosition < ItemCount) ? new CosmosDBRecordEntry(Items[m_currentPosition]) : null; + return m_currentEntry != null; + } + + private async Task GetPage() + { + while (m_feedIterator.HasMoreResults) + { + try + { + using (ResponseMessage response = await m_feedIterator.ReadNextAsync().ConfigureAwait(false)) + { + if (!response.IsSuccessStatusCode) + { + if (response.Diagnostics != null) + GXLogging.Debug(logger, $"Read ItemStreamFeed Diagnostics: {response.Diagnostics.ToString()}"); + throw new Exception(GeneXus.Data.Cosmos.CosmosDBHelper.FormatExceptionMessage(response.StatusCode.ToString(),response.ErrorMessage)); + } + else + { + using (StreamReader sr = new StreamReader(response.Content)) + using (JsonTextReader jtr = new JsonTextReader(sr)) + { + Newtonsoft.Json.JsonSerializer jsonSerializer = new Newtonsoft.Json.JsonSerializer(); + object array = jsonSerializer.Deserialize(jtr); + + string json = ((Newtonsoft.Json.Linq.JToken)array).Root.ToString(); + var jsonDocument = JsonDocument.Parse(json); + var jsonDoc = jsonDocument.RootElement; + foreach (var jsonProperty in jsonDoc.EnumerateObject()) + { + if (jsonProperty.Name == "Documents") + { + Items = JsonConvert.DeserializeObject>>(jsonProperty.Value.ToString()); + break; + } + } + if (Items != null) + ItemCount = Items.Count; + else + ItemCount = 0; + } + } + } + return true; + } + catch (CosmosException ex) + { + throw ex; + } + } + return false; + } + public bool Read() + { + Task task; + if (NextResult()) + return true; + else + if (m_feedIterator != null) + { + task = Task.Run(async () => await GetPage().ConfigureAwait(false)); + if (task.Result) + { + m_currentPosition = -1; + return NextResult(); + } + } + return false; + } + + public object GetValue(int i) + { + return GetAttValue(i); + } + } + +} diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDatastoreHelper.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDatastoreHelper.cs new file mode 100644 index 000000000..6f9964777 --- /dev/null +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBDatastoreHelper.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Generic; +using GeneXus.Data.Cosmos; +using GeneXus.Utils; + +namespace GeneXus.Data.NTier +{ + public class CosmosDBDatastoreHelper : DynServiceDataStoreHelperBase + { + + public CosmosDBQuery NewQuery() => new CosmosDBQuery(this); + + public CosmosDBMap Map(string name) + { + return new CosmosDBMap(name); + } + public object empty(GXType gxtype) + { + switch (gxtype) + { + case GXType.Number: + case GXType.Int16: + case GXType.Int32: + case GXType.Int64: return 0; + case GXType.Date: + case GXType.DateTime: + case GXType.DateTime2: return DateTimeUtil.NullDate(); + case GXType.Byte: + case GXType.NChar: + case GXType.NClob: + case GXType.NVarChar: + case GXType.Char: + case GXType.LongVarChar: + case GXType.Clob: + case GXType.VarChar: + case GXType.Raw: + case GXType.Blob: return string.Empty; + case GXType.Boolean: return false; + case GXType.Decimal: return 0f; + case GXType.NText: + case GXType.Text: + case GXType.Image: + case GXType.UniqueIdentifier: + case GXType.Xml: return string.Empty; + case GXType.Geography: + case GXType.Geopoint: + case GXType.Geoline: + case GXType.Geopolygon: return new Geospatial(); + case GXType.DateAsChar: return string.Empty; + case GXType.Undefined: + default: return null; + + } + } + } + public class CosmosDBQuery : Query + { + public string Index { get; set; } + + public string PartitionKey { get; private set; } + public override Query SetKey(string partitionKey) + { + PartitionKey = partitionKey; + return this; + } + internal IEnumerable KeyFilters { get; set; } = Array.Empty(); + public override Query KeyFilter(string[] filters) + { + KeyFilters = filters; + return this; + } + + public CosmosDBQuery(CosmosDBDatastoreHelper dataStoreHelper) : base(dataStoreHelper) { } + } +} diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBHelper.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBHelper.cs new file mode 100644 index 000000000..62495d18e --- /dev/null +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBHelper.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; +using System.Text.Json.Nodes; +using GeneXus.Data.NTier; + +namespace GeneXus.Data.Cosmos +{ + internal class CosmosDBHelper + { + internal static bool AddItemValue(string parmName, string fromName, Dictionary values, IDataParameterCollection parms, IEnumerable queryVars, ref JsonObject jsonObject) + { + if (!AddItemValue(parmName, values, parms[fromName] as ServiceParameter, ref jsonObject)) + { + VarValue varValue = queryVars.FirstOrDefault(v => v.Name == $":{fromName}"); + if (varValue != null) + { + if (varValue.Value == DBNull.Value) + { + KeyValuePair keyvalue = new KeyValuePair(parmName, null); + jsonObject.Add(keyvalue); + } + else + jsonObject.Add(parmName, JsonValue.Create(varValue.Value)); + values[parmName] = varValue.Value; + } + return varValue != null; + } + return true; + } + public static bool FormattedAsStringGXType(GXType gXType) + { + return (gXType == GXType.Date || gXType == GXType.DateTime || gXType == GXType.DateTime2 || gXType == GXType.VarChar || gXType == GXType.DateAsChar || gXType == GXType.NVarChar || gXType == GXType.LongVarChar || gXType == GXType.NChar || gXType == GXType.Char || gXType == GXType.Text || gXType == GXType.NText); + } + internal static bool FormattedAsStringDbType(DbType dbType) + { + return (dbType == DbType.String || dbType == DbType.Date || dbType == DbType.DateTime || dbType == DbType.DateTime2 || dbType == DbType.DateTimeOffset || dbType == DbType.StringFixedLength || dbType == DbType.AnsiString || dbType == DbType.AnsiStringFixedLength || dbType == DbType.Guid || dbType == DbType.Time); + } + internal static string FormatExceptionMessage(string statusCode, string message) + { + return ($"CosmosDB Execution failed. Status code: {statusCode}. Message: {message}"); + } + internal static bool AddItemValue(string parmName, Dictionary dynParm, ServiceParameter parm, ref JsonObject jsonObject) + { + if (parm == null) + return false; + if (parm.Value != null) + { + if (parm.Value == DBNull.Value) + { + KeyValuePair keyvalue = new KeyValuePair(parmName, null); + jsonObject.Add(keyvalue); + } + else + jsonObject.Add(parmName, JsonValue.Create(parm.Value)); + dynParm[parmName] = parm.Value; + return true; + } + return false; + } + } +} diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBMaps.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBMaps.cs new file mode 100644 index 000000000..235f861fe --- /dev/null +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBMaps.cs @@ -0,0 +1,35 @@ +using GeneXus.Data.NTier; +using System; +using System.Collections.Generic; + +namespace GeneXus.Data.Cosmos +{ + public class CosmosDBMap : Map + { + internal bool NeedsAttributeMap { get; } + + public CosmosDBMap(string name): base(name){} + public override object GetValue(IOServiceContext context, RecordEntryRow currentEntry) + { + Dictionary values = ((CosmosDBRecordEntry)currentEntry).CurrentRow; + + values.TryGetValue(GetName(context), out object val); + return val; + } + + public override void SetValue(RecordEntryRow currentEntry, object value) + { + throw new NotImplementedException(); + } + + } + public class CosmosDBRecordEntry : RecordEntryRow + { + public Dictionary CurrentRow { get; } + + public CosmosDBRecordEntry(Dictionary cRow) + { + CurrentRow = cRow; + } + } +} diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBRequestWrapper.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBRequestWrapper.cs new file mode 100644 index 000000000..0dd7d4bf7 --- /dev/null +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBRequestWrapper.cs @@ -0,0 +1,97 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using log4net; +using Microsoft.Azure.Cosmos; +using Newtonsoft.Json; + +namespace GeneXus.Data.NTier.CosmosDB +{ + public class RequestWrapper + { + private readonly Container m_container; + private readonly CosmosClient m_cosmosClient; + private readonly QueryDefinition m_queryDefinition; + static readonly ILog logger = log4net.LogManager.GetLogger(typeof(RequestWrapper)); + public string idValue { get; set; } + public string partitionKeyValue { get; set; } + public bool queryByPK { get; set; } + public RequestWrapper(CosmosClient cosmosClient, Container container, QueryDefinition queryDefinition) + { + m_container = container; + m_cosmosClient = cosmosClient; + m_queryDefinition = queryDefinition; + } + + private List> ProcessPKStream(Stream stream) + { + //Query by PK -> only one record + + List > Items = new List>(); + if (stream != null) + { + using (StreamReader sr = new StreamReader(stream)) + using (JsonTextReader jtr = new JsonTextReader(sr)) + { + Newtonsoft.Json.JsonSerializer jsonSerializer = new Newtonsoft.Json.JsonSerializer(); + object array = jsonSerializer.Deserialize(jtr); + + Dictionary result = JsonConvert.DeserializeObject>(array.ToString()); + + //remove metadata + result.Remove("_rid"); + result.Remove("_self"); + result.Remove("_etag"); + result.Remove("_attachments"); + result.Remove("_ts"); + Items.Add(result); + } + } + return Items; + } + private async Task ReadItemAsyncByPK(string idValue, string partitionKeyValue) + { + List> Items = new List>(); + using (ResponseMessage responseMessage = await m_container.ReadItemStreamAsync( + partitionKey: new PartitionKey(partitionKeyValue), + id: idValue).ConfigureAwait(false)) + { + + if (!responseMessage.IsSuccessStatusCode) + { + if (!responseMessage.ErrorMessage.Contains("404")) + { + if (responseMessage.Diagnostics != null) + GXLogging.Debug(logger, $"Read ReadItemAsyncByPK Diagnostics: {responseMessage.Diagnostics.ToString()}"); + throw new Exception(GeneXus.Data.Cosmos.CosmosDBHelper.FormatExceptionMessage(responseMessage.StatusCode.ToString(), responseMessage.ErrorMessage)); + } + } + else + { + Items = ProcessPKStream(responseMessage.Content); + } + } + return new ResponseWrapper(Items); + } + public ResponseWrapper Read() + { + if (queryByPK) + { + Task task = Task.Run(async () => await ReadItemAsyncByPK(idValue, partitionKeyValue).ConfigureAwait(false)); + return task.Result; + } + + QueryRequestOptions requestOptions = new QueryRequestOptions() { MaxBufferedItemCount = 100 }; + //options.MaxConcurrency = 1; + //TODO Cancelation Token + request options + using (FeedIterator feedIterator = m_container.GetItemQueryStreamIterator(m_queryDefinition, null, requestOptions)) + + return new ResponseWrapper(feedIterator); + } + internal void Close() + { + m_cosmosClient.Dispose(); + } + } +} diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBResponseWrapper.cs b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBResponseWrapper.cs new file mode 100644 index 000000000..717935ddb --- /dev/null +++ b/dotnet/src/dotnetcore/DynService/Cosmos/CosmosDBResponseWrapper.cs @@ -0,0 +1,34 @@ +using Microsoft.Azure.Cosmos; +using System.Collections.Generic; +using System.IO; + +namespace GeneXus.Data.NTier.CosmosDB +{ + public class ResponseWrapper + { + public FeedIterator feedIterator; + public Stream stream; + + public ResponseWrapper(ResponseMessage responseMessage, FeedIterator feedIter) + { + feedIterator = feedIter; + stream = responseMessage.Content; + } + public ResponseWrapper(FeedIterator feedIter) + { + feedIterator = feedIter; + } + public ResponseWrapper(Stream streamResponse) + { + stream = streamResponse; + } + + public ResponseWrapper(List> responseItems) + { + Items = responseItems; + ItemCount = responseItems.Count; + } + public List> Items { get; set; } + public int ItemCount { get; set; } + } +} diff --git a/dotnet/src/dotnetcore/DynService/Cosmos/DynService.CosmosDB.csproj b/dotnet/src/dotnetcore/DynService/Cosmos/DynService.CosmosDB.csproj new file mode 100644 index 000000000..120fad8f9 --- /dev/null +++ b/dotnet/src/dotnetcore/DynService/Cosmos/DynService.CosmosDB.csproj @@ -0,0 +1,22 @@ + + + net6.0 + GeneXus.Data.NTier + GeneXus.Data.DynService.CosmosDB + false + CosmosDB + GeneXus.DynService.CosmosDB + + + + + + + + + + + + + +