From 32a825b925e1bfa40f486931927dbcb9eb667d49 Mon Sep 17 00:00:00 2001 From: Gustavo Brown Date: Thu, 13 Jan 2022 12:46:42 -0300 Subject: [PATCH 01/24] DynamoDB support initial commit (WIP) --- gxdynamodb/pom.xml | 32 +++ .../db/dynamodb/DataStoreHelperDynamoDB.java | 123 +++++++++++ .../db/dynamodb/DynamoDBConnection.java | 199 ++++++++++++++++++ .../genexus/db/dynamodb/DynamoDBDriver.java | 73 +++++++ .../com/genexus/db/dynamodb/DynamoDBMap.java | 29 +++ .../com/genexus/db/dynamodb/DynamoQuery.java | 44 ++++ .../com/genexus/db/dynamodb/DynamoScan.java | 7 + .../java/com/genexus/db/service/IOMap.java | 27 +++ .../java/com/genexus/db/service/Query.java | 84 ++++++++ pom.xml | 1 + 10 files changed, 619 insertions(+) create mode 100644 gxdynamodb/pom.xml create mode 100644 gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java create mode 100644 gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java create mode 100644 gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBDriver.java create mode 100644 gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBMap.java create mode 100644 gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java create mode 100644 gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoScan.java create mode 100644 java/src/main/java/com/genexus/db/service/IOMap.java create mode 100644 java/src/main/java/com/genexus/db/service/Query.java diff --git a/gxdynamodb/pom.xml b/gxdynamodb/pom.xml new file mode 100644 index 000000000..ebccb13c7 --- /dev/null +++ b/gxdynamodb/pom.xml @@ -0,0 +1,32 @@ + + + 4.0.0 + + + com.genexus + parent + 2.4-SNAPSHOT + + + gxdynamodb + GeneXus DynamoDB + + + + software.amazon.awssdk + dynamodb + 2.17.103 + + + ${project.groupId} + gxclassR + ${project.version} + + + + + gxdynamodb + + \ No newline at end of file diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java new file mode 100644 index 000000000..712f0865e --- /dev/null +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java @@ -0,0 +1,123 @@ +package com.genexus.db.dynamodb; + +import com.genexus.db.ServiceCursorBase; +import com.genexus.db.driver.GXConnection; +import com.genexus.db.driver.GXPreparedStatement; +import com.genexus.db.service.*; +import java.util.function.*; + +public class DataStoreHelperDynamoDB extends ServiceDataStoreHelper +{ + public DynamoQuery newQuery() + { + return new DynamoQuery(this); + } + public DynamoQuery newScan() + { + return new DynamoScan(this); + } + + public DynamoDBMap Map(String name) + { + return new DynamoDBMap(name); + } + +/* public object empty(GXType gxtype) + { + switch(gxtype) + { + case GXType.Number: + case GXType.Int16: + case GXType.Int32: + case GXType.Int64: return 0; + case GXType.Date: + case GXType.DateTime: + case GXType.DateTime2: return DateTimeUtil.NullDate(); + case GXType.Byte: + case GXType.NChar: + case GXType.NClob: + case GXType.NVarChar: + case GXType.Char: + case GXType.LongVarChar: + case GXType.Clob: + case GXType.VarChar: + case GXType.Raw: + case GXType.Blob: return string.Empty; + case GXType.Boolean: return false; + case GXType.Decimal: return 0f; + case GXType.NText: + case GXType.Text: + case GXType.Image: + case GXType.UniqueIdentifier: + case GXType.Xml: return string.Empty; + case GXType.Geography: + case GXType.Geopoint: + case GXType.Geoline: + case GXType.Geopolygon: return new Geospatial(); + case GXType.DateAsChar: return string.Empty; + case GXType.Undefined: + default: return null; + } + } +*/ + + @Override + public GXPreparedStatement getPreparedStatement(GXConnection con, IQuery query, ServiceCursorBase cursor, int cursorNum, boolean currentOf, Object[] parms) { + return null; + } + /** + private final CurrentOfManager currentOfManager = new CurrentOfManager(); + public CurrentOfManager getCurrentOfManager() + { + return currentOfManager; + } + + public ODataQuery getQuery(BiFunction query, IODataMap[] selectList) + { + return new ODataQuery(query, selectList); + } + + public ODataQuery getQuery(BiFunction query, String queryType) + { + return new ODataQuery(query, queryType); + } + + public ODataQuery getQuery(BiFunction query, ODataQuery continuation) + { + return new ODataQuery(query, continuation); + } + + public ODataQuery getQuery(BiFunction query, String queryType, ODataQuery continuation) + { + return new ODataQuery(query, queryType, continuation); + } + + public ComplexHashMap complex(String entity) + { + return new ComplexHashMap(entity); + } + + public CurrentOf currentOf(String cursorName, String entity) + { + return new CurrentOf(currentOfManager, cursorName, entity); + } + + protected ODataMapDomain MapDomain(String name) + { + return new ODataMapDomain(name); + } + + @Override + public GXPreparedStatement getPreparedStatement(GXConnection con, IQuery query, ServiceCursorBase cursor, int cursorNum, boolean currentOf, Object[] parms) + { + return new GXPreparedStatement(new ODataPreparedStatement(con.getJDBCConnection(), (ODataQuery)query, cursor, parms, con), con, con.getHandle(), "", cursor.getCursorId(), currentOf); + } + + @Override + public Object GetParmDateTime(Object parm) + { + java.util.Date date = GetParmDate(parm); + return date != null ? new java.sql.Timestamp(date.getTime()) : null; + } + */ +} diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java new file mode 100644 index 000000000..9a53f265a --- /dev/null +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java @@ -0,0 +1,199 @@ +package com.genexus.db.dynamodb; + +import com.genexus.db.service.ServiceConnection; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.QueryRequest; +import software.amazon.awssdk.services.dynamodb.model.QueryResponse; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import java.util.HashMap; +import software.amazon.awssdk.services.dynamodb.model.DynamoDbException; +import software.amazon.awssdk.regions.Region;import java.nio.charset.StandardCharsets; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.*; +import java.util.concurrent.Executor; + +public class DynamoDBConnection extends ServiceConnection +{ + private static final String GXDYNAMODB_VERSION = "1.0"; + + DynamoDbClient mDynamoDB; + + public DynamoDBConnection(String connUrl, Properties initialConnProps) throws SQLException + { + super(connUrl, initialConnProps); /// Luego de la inicialización usar props de la clase base para obtener las propiedades + initializeModel(connUrl); + } + + private void initializeModel(String connUrl) + { + for(Enumeration keys = props.keys(); keys.hasMoreElements(); ) + { + String key = ((String)keys.nextElement()); + String value = props.getProperty(key, key); + switch(key.toLowerCase()) + { + default: break; + } + } +/* try + { + }catch(URISyntaxException | IOException ex) + { + LogManager.getLogger(ODataConnection.class).warn(String.format("Could not load metadata file: %s%s", metadataLocation, metadataDocName), ex); + } + */ + + } + +/* ServiceError getServiceError(String errorCode) + { + if(errorCode != null) + { + if (modelInfo.recordNotFoundServiceCodes != null && modelInfo.recordNotFoundServiceCodes.contains(errorCode)) + return ServiceError.OBJECT_NOT_FOUND; + if (modelInfo.recordAlreadyExistsServiceCodes != null && modelInfo.recordAlreadyExistsServiceCodes.contains(errorCode)) + return ServiceError.DUPLICATE_KEY; + } + return ServiceError.INVALID_QUERY; + }*/ + +/* private boolean getBoolean(String value) + { + return value.equalsIgnoreCase("y") || value.equalsIgnoreCase("true") || value.equalsIgnoreCase("yes"); + } + + boolean needsCheckOptimisticConcurrency(URI updURI) + { + return modelInfo.needsCheckOptimisticConcurrency(updURI); + } + + public int getEnumValue(ClientEnumValue enumValue) + { + String typeName = enumValue.getTypeName(); + return Integer.parseInt(modelInfo.getModel().getEnumType(new FullQualifiedName(typeName)).getMember(enumValue.getValue()).getValue()); + } + + public String toEnumValue(EdmEnumType type, int value) + { + String sValue = Integer.toString(value); + for(String memberName:type.getMemberNames()) + { + EdmMember member = type.getMember(memberName); + if(member.getValue().equals(sValue)) + return member.getName(); + } + throw new RuntimeException(String.format("Cannot parse enum value %s - %d", type.toString(), value)); + } + + private String entity(String name) + { + return modelInfo.entity(name); + } + + public String entity(EdmEntityType [] fromEntity, String name) + { + if(fromEntity == null || fromEntity[0] == null) + return entity((EdmEntityType)null, name); + String entityName = entity(fromEntity[0], name); + if(entityName == null) + return entity(name); + EdmNavigationProperty navProp = fromEntity[0].getNavigationProperty(entityName); + if(navProp != null) + fromEntity[0] = navProp.getType(); + return entityName; + } + + public String entity(EdmEntityType fromEntity, String name) + { + return modelInfo.entity(fromEntity, name); + } + + Edm getModel() + { + return modelInfo.getModel(); + } +*/ +//---------------------------------------------------------------------------------------------------- + + @Override + public void close() + { + mDynamoDB.close(); + mDynamoDB = null; + } + + @Override + public boolean isClosed() + { + return mDynamoDB != null; + } + + //---------------------------------------------------------------------------------------------------- + @Override + public String getDatabaseProductName() + { + return "DynamoDB"; + } + + @Override + public String getDatabaseProductVersion() + { + return ""; + } + + @Override + public String getDriverName() + { + return mDynamoDB.getClass().getName(); + } + + @Override + public String getDriverVersion() + { + return String.format("%s/%s", mDynamoDB.serviceName(), GXDYNAMODB_VERSION); + } + + // JDK8: + @Override + public void setSchema(String schema) + { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public String getSchema() + { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public void abort(Executor executor) + { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public void setNetworkTimeout(Executor executor, int milliseconds) + { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public int getNetworkTimeout() + { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public ResultSet getPseudoColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) + { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public boolean generatedKeyAlwaysReturned() + { + throw new UnsupportedOperationException("Not supported yet."); + } +} diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBDriver.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBDriver.java new file mode 100644 index 000000000..a25c5cc2d --- /dev/null +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBDriver.java @@ -0,0 +1,73 @@ +package com.genexus.db.dynamodb; + +import java.sql.*; +import java.util.Properties; +import java.util.logging.Logger; + +public class DynamoDBDriver implements Driver +{ + private static final int MAJOR_VERSION = 1; + private static final int MINOR_VERSION = 0; + private static final String DRIVER_ID = "dynamodb:"; + + private static final DynamoDBDriver DYNAMODB_DRIVER; + static + { + DYNAMODB_DRIVER = new ODataDriver(); + try + { + DriverManager.registerDriver(DYNAMODB_DRIVER); + }catch(SQLException e) + { + e.printStackTrace(); + } + } + + public DynamoDBDriver() + { + } + + @Override + public Connection connect(String url, Properties info) throws SQLException + { + if(!acceptsURL(url)) + return null; + return new DynamoDBConnection(url.substring(DRIVER_ID.length()), info); + } + + @Override + public boolean acceptsURL(String url) + { + return url.startsWith(DRIVER_ID); + } + + @Override + public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) + { + return new DriverPropertyInfo[0]; + } + + @Override + public int getMajorVersion() + { + return MAJOR_VERSION; + } + + @Override + public int getMinorVersion() + { + return MINOR_VERSION; + } + + @Override + public boolean jdbcCompliant() + { + return false; + } + + @Override + public Logger getParentLogger() + { + throw new UnsupportedOperationException("Not supported yet."); + } +} diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBMap.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBMap.java new file mode 100644 index 000000000..2695e5fb3 --- /dev/null +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBMap.java @@ -0,0 +1,29 @@ +package com.genexus.db.dynamodb; + +import com.genexus.db.service.*; + +public class DynamoDBMap extends IODataMapName{ +//ver DynamoDBMaps.cs + public DynamoDBMap(String name) { + super(mapAttributeMap(name)); + needsAttributeMap = _needsAttributeMap(name); + } + + private static String mapAttributeMap(String name) + { + if(_needsAttributeMap(name)) + return name.substring(1); + else return name; + } + + private static boolean _needsAttributeMap(String name) + { + return name.startsWith("#"); + } + + private final boolean needsAttributeMap; + + public boolean needsAttributeMap() { + return needsAttributeMap; + } +} diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java new file mode 100644 index 000000000..85c7319b4 --- /dev/null +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java @@ -0,0 +1,44 @@ +package com.genexus.db.dynamodb; +import com.genexus.db.service.*; + +public class DynamoQuery extends Query{ + private String index; + + private boolean scanIndexForward = true; + private static final String RANGE_KEY_INDEX = "RangeKey"; + private static final char[] indexTrimChars = new char[] { '(', ')' }; + + public DynamoQuery orderBy(String index) + { + index = index.trim(); + if(index.startsWith("(") && index.endsWith(")")) + { + scanIndexForward = false; + index = index.substring(1, index.length()-2); + } + if (!RANGE_KEY_INDEX.equals(index)) + setIndex(index); + return this; + } + + public DynamoQuery(DataStoreHelperDynamoDB dataStoreHelper) + { + super(dataStoreHelper); + } + + public String getIndex() { + return index; + } + + public void setIndex(String index) { + this.index = index; + } + + public boolean isScanIndexForward() { + return scanIndexForward; + } + + public void setScanIndexForward(boolean scanIndexForward) { + this.scanIndexForward = scanIndexForward; + } +} diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoScan.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoScan.java new file mode 100644 index 000000000..5619edba7 --- /dev/null +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoScan.java @@ -0,0 +1,7 @@ +package com.genexus.db.dynamodb; + +public class DynamoScan extends DynamoQuery { + public DynamoScan(DataStoreHelperDynamoDB dataStoreHelperDynamoDB) { + super(dataStoreHelperDynamoDB); + } +} diff --git a/java/src/main/java/com/genexus/db/service/IOMap.java b/java/src/main/java/com/genexus/db/service/IOMap.java new file mode 100644 index 000000000..ff9622723 --- /dev/null +++ b/java/src/main/java/com/genexus/db/service/IOMap.java @@ -0,0 +1,27 @@ +package com.genexus.db.service; + +import java.util.HashMap; + +public class IOMap implements IODataMap{ + private final String name; + + public IOMap(String name) + { + this.name = name; + } + + @Override + public Object getValue(IOServiceContext context, HashMap currentEntry) { + return null; + } + + @Override + public String getName() { + return null; + } + + @Override + public void setValue(HashMap currentEntry, Object value) { + + } +} diff --git a/java/src/main/java/com/genexus/db/service/Query.java b/java/src/main/java/com/genexus/db/service/Query.java new file mode 100644 index 000000000..48e8f7d0b --- /dev/null +++ b/java/src/main/java/com/genexus/db/service/Query.java @@ -0,0 +1,84 @@ +package com.genexus.db.service; + +import java.util.Collections; + +public class Query implements IQuery { + private static final String [] EMPTY_ARR_STRING = new String [0]; + ServiceDataStoreHelper dataStoreHelper; + public Query(ServiceDataStoreHelper dataStoreHelper) + { + this.dataStoreHelper = dataStoreHelper; + } + + private String tableName; + public String[] projection = EMPTY_ARR_STRING; +/* + public string[] OrderBys { get; set; } = Array.Empty(); + public string[] Filters { get; set; } = Array.Empty(); + private List> mAssignAtts; + public IEnumerable> AssignAtts { get { return mAssignAtts ?? Array.Empty>() as IEnumerable>; } } +*/ + public IODataMap[] selectList = new IODataMap[0]; +/* + private List mVarValues; + public IEnumerable Vars { get { return (mVarValues ?? Array.Empty() as IEnumerable); } } + public CursorType CursorType { get; set; } = CursorType.Select; + */ + + public Query For(String tableName) + { + this.tableName = tableName; + return this; + } + + public Query select(String[] columns) + { + this.projection = columns; + return this; + } +/* + public Query OrderBy(string[] orders) + { + OrderBys = orders; + return this; + } + + public Query Filter(string[] filters) + { + Filters = filters; + return this; + } + + public Query Set(string name, string value) + { + mAssignAtts = mAssignAtts ?? new List>(); + mAssignAtts.Add(new KeyValuePair(name, value)); + return this; + } +*/ + public Query setMaps(IODataMap[] selectList) + { + this.selectList = selectList; + return this; + } + + @Override + public QueryType getQueryType() { + return QueryType.QUERY; + } +/* + + public Query SetType(CursorType cType) + { + CursorType = cType; + return this; + } + + public Query AddParm(GXType gxType, object parm) + { + mVarValues = mVarValues ?? new List(); + mVarValues.Add(new VarValue($":parm{ mVarValues.Count + 1 }", gxType, parm)); + return this; + } +*/ +} diff --git a/pom.xml b/pom.xml index 4d2e3161b..cc3d94280 100644 --- a/pom.xml +++ b/pom.xml @@ -38,6 +38,7 @@ android gxgeospatial gxodata + gxdynamodb gxexternalproviders gxwebsocket gxwebsocketjakarta From 7caf40fadf43475f0ed1e4cdf87a28e835902675 Mon Sep 17 00:00:00 2001 From: Gustavo Brown Date: Fri, 11 Mar 2022 13:47:47 -0300 Subject: [PATCH 02/24] Add support for querying tables --- gxdynamodb/pom.xml | 24 +- .../db/dynamodb/DataStoreHelperDynamoDB.java | 122 ++----- .../db/dynamodb/DynamoDBConnection.java | 39 ++- .../genexus/db/dynamodb/DynamoDBDriver.java | 2 +- .../genexus/db/dynamodb/DynamoDBHelper.java | 95 ++++++ .../dynamodb/DynamoDBPreparedStatement.java | 48 +++ .../db/dynamodb/DynamoDBResultSet.java | 312 ++++++++++++++++++ .../com/genexus/db/dynamodb/DynamoQuery.java | 1 + .../java/com/genexus/db/service/GXType.java | 41 +++ .../java/com/genexus/db/service/Query.java | 84 +++-- .../genexus/db/service/ServiceResultSet.java | 52 +-- .../java/com/genexus/db/service/VarValue.java | 15 + 12 files changed, 680 insertions(+), 155 deletions(-) create mode 100644 gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java create mode 100644 gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java create mode 100644 gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java create mode 100644 java/src/main/java/com/genexus/db/service/GXType.java create mode 100644 java/src/main/java/com/genexus/db/service/VarValue.java diff --git a/gxdynamodb/pom.xml b/gxdynamodb/pom.xml index ebccb13c7..d4942a2ee 100644 --- a/gxdynamodb/pom.xml +++ b/gxdynamodb/pom.xml @@ -13,11 +13,31 @@ gxdynamodb GeneXus DynamoDB - - + + + + software.amazon.awssdk + bom + 2.17.136 + pom + import + + + + + + + software.amazon.awssdk + dynamodb-enhanced + 2.17.136 + + + software.amazon.awssdk + dynamodb ${project.groupId} diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java index 712f0865e..bb56c4fb2 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java @@ -1,5 +1,6 @@ package com.genexus.db.dynamodb; +import com.genexus.CommonUtil; import com.genexus.db.ServiceCursorBase; import com.genexus.db.driver.GXConnection; import com.genexus.db.driver.GXPreparedStatement; @@ -22,102 +23,49 @@ public DynamoDBMap Map(String name) return new DynamoDBMap(name); } -/* public object empty(GXType gxtype) + public Object empty(GXType gxtype) { switch(gxtype) { - case GXType.Number: - case GXType.Int16: - case GXType.Int32: - case GXType.Int64: return 0; - case GXType.Date: - case GXType.DateTime: - case GXType.DateTime2: return DateTimeUtil.NullDate(); - case GXType.Byte: - case GXType.NChar: - case GXType.NClob: - case GXType.NVarChar: - case GXType.Char: - case GXType.LongVarChar: - case GXType.Clob: - case GXType.VarChar: - case GXType.Raw: - case GXType.Blob: return string.Empty; - case GXType.Boolean: return false; - case GXType.Decimal: return 0f; - case GXType.NText: - case GXType.Text: - case GXType.Image: - case GXType.UniqueIdentifier: - case GXType.Xml: return string.Empty; - case GXType.Geography: - case GXType.Geopoint: - case GXType.Geoline: - case GXType.Geopolygon: return new Geospatial(); - case GXType.DateAsChar: return string.Empty; - case GXType.Undefined: + case Number: + case Int16: + case Int32: + case Int64: return 0; + case Date: + case DateTime: + case DateTime2: return CommonUtil.nullDate(); + case Byte: + case NChar: + case NClob: + case NVarChar: + case Char: + case LongVarChar: + case Clob: + case VarChar: + case Raw: + case Blob: return ""; + case Boolean: return false; + case Decimal: return 0f; + case NText: + case Text: + case Image: + case UniqueIdentifier: + case Xml: return ""; + case DateAsChar: return ""; + + case Geography: + case Geopoint: + case Geoline: + case Geopolygon: + + case Undefined: default: return null; } } -*/ - - @Override - public GXPreparedStatement getPreparedStatement(GXConnection con, IQuery query, ServiceCursorBase cursor, int cursorNum, boolean currentOf, Object[] parms) { - return null; - } - /** - private final CurrentOfManager currentOfManager = new CurrentOfManager(); - public CurrentOfManager getCurrentOfManager() - { - return currentOfManager; - } - - public ODataQuery getQuery(BiFunction query, IODataMap[] selectList) - { - return new ODataQuery(query, selectList); - } - - public ODataQuery getQuery(BiFunction query, String queryType) - { - return new ODataQuery(query, queryType); - } - - public ODataQuery getQuery(BiFunction query, ODataQuery continuation) - { - return new ODataQuery(query, continuation); - } - public ODataQuery getQuery(BiFunction query, String queryType, ODataQuery continuation) - { - return new ODataQuery(query, queryType, continuation); - } - - public ComplexHashMap complex(String entity) - { - return new ComplexHashMap(entity); - } - - public CurrentOf currentOf(String cursorName, String entity) - { - return new CurrentOf(currentOfManager, cursorName, entity); - } - - protected ODataMapDomain MapDomain(String name) - { - return new ODataMapDomain(name); - } - @Override public GXPreparedStatement getPreparedStatement(GXConnection con, IQuery query, ServiceCursorBase cursor, int cursorNum, boolean currentOf, Object[] parms) { - return new GXPreparedStatement(new ODataPreparedStatement(con.getJDBCConnection(), (ODataQuery)query, cursor, parms, con), con, con.getHandle(), "", cursor.getCursorId(), currentOf); - } - - @Override - public Object GetParmDateTime(Object parm) - { - java.util.Date date = GetParmDate(parm); - return date != null ? new java.sql.Timestamp(date.getTime()) : null; + return new GXPreparedStatement(new DynamoDBPreparedStatement(con.getJDBCConnection(), (DynamoQuery)query, cursor, parms, con), con, con.getHandle(), "", cursor.getCursorId(), currentOf); } - */ } diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java index 9a53f265a..ec2091788 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java @@ -1,10 +1,17 @@ package com.genexus.db.dynamodb; import com.genexus.db.service.ServiceConnection; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.enhanced.dynamodb.DynamoDbEnhancedClient; +import software.amazon.awssdk.services.dynamodb.DynamoDbClientBuilder; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.QueryRequest; import software.amazon.awssdk.services.dynamodb.model.QueryResponse; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; + +import java.net.URI; import java.util.HashMap; import software.amazon.awssdk.services.dynamodb.model.DynamoDbException; import software.amazon.awssdk.regions.Region;import java.nio.charset.StandardCharsets; @@ -17,33 +24,47 @@ public class DynamoDBConnection extends ServiceConnection { private static final String GXDYNAMODB_VERSION = "1.0"; - DynamoDbClient mDynamoDB; + private static final String CLIENT_ID = "user"; + private static final String CLIENT_SECRET = "password"; + private static final String REGION = "region"; + private static final String LOCAL_URL = "localurl"; + + + DynamoDbClient mDynamoDB; //mDynamoDB + Region mRegion = Region.US_EAST_1; public DynamoDBConnection(String connUrl, Properties initialConnProps) throws SQLException { - super(connUrl, initialConnProps); /// Luego de la inicialización usar props de la clase base para obtener las propiedades - initializeModel(connUrl); + super(connUrl, initialConnProps); // After initialization use props variable from super class to manage properties + initializeDBConnection(connUrl); } - private void initializeModel(String connUrl) + private void initializeDBConnection(String connUrl) { + String mLocalUrl = null, mClientId = null, mClientSecret = null; for(Enumeration keys = props.keys(); keys.hasMoreElements(); ) { String key = ((String)keys.nextElement()); String value = props.getProperty(key, key); switch(key.toLowerCase()) { + case LOCAL_URL: mLocalUrl = value; break; + case CLIENT_ID: mClientId = value; break; + case CLIENT_SECRET: mClientSecret = value; break; + case REGION: mRegion = Region.of(value); break; default: break; } } -/* try - { - }catch(URISyntaxException | IOException ex) + DynamoDbClientBuilder builder = DynamoDbClient.builder().region(mRegion); + if(mLocalUrl != null) + builder = builder.endpointOverride(URI.create(mLocalUrl)); + if(mClientId != null && mClientSecret != null) { - LogManager.getLogger(ODataConnection.class).warn(String.format("Could not load metadata file: %s%s", metadataLocation, metadataDocName), ex); + AwsBasicCredentials mCredentials = AwsBasicCredentials.create(mClientId, mClientSecret); + builder = builder.credentialsProvider(StaticCredentialsProvider.create(mCredentials)); } - */ + mDynamoDB = builder.build(); } /* ServiceError getServiceError(String errorCode) diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBDriver.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBDriver.java index a25c5cc2d..5fbcf34db 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBDriver.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBDriver.java @@ -13,7 +13,7 @@ public class DynamoDBDriver implements Driver private static final DynamoDBDriver DYNAMODB_DRIVER; static { - DYNAMODB_DRIVER = new ODataDriver(); + DYNAMODB_DRIVER = new DynamoDBDriver(); try { DriverManager.registerDriver(DYNAMODB_DRIVER); diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java new file mode 100644 index 000000000..dda644032 --- /dev/null +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java @@ -0,0 +1,95 @@ +package com.genexus.db.dynamodb; + +import com.genexus.db.service.VarValue; +import json.org.json.JSONArray; +import json.org.json.JSONObject; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; + +import java.sql.SQLException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class DynamoDBHelper +{ + public static AttributeValue ToAttributeValue(VarValue var) throws SQLException + { + Object value = var.value; + AttributeValue.Builder builder = AttributeValue.builder(); + switch (var.type) + { + case Number: + case Int16: + case Int32: + case Int64: + case Decimal: + return builder.n(value.toString()).build(); + case Date: + case DateTime: + case DateTime2: + return builder.s(value.toString()).build(); + case Boolean: + case Byte: + return builder.bool((Boolean) value).build(); + case Char: + case VarChar: + case LongVarChar: + case Text: + case NChar: + case NVarChar: + case NText: + return builder.s(value.toString()).build(); + case UniqueIdentifier: + case Xml: + case Geography: + case Geopoint: + case Geoline: + case Geopolygon: + return builder.s(value.toString()).build(); + case NClob: + case Clob: + case Raw: + case Blob: + case Undefined: + case Image: + case DateAsChar: + default: + throw new SQLException(String.format("DynamoDB unsupported type (%s)", var.type.toString())); + } + } + + public static String getString(AttributeValue attValue) + { + if(attValue == null) + return null; + String value = attValue.s(); + if (value != null) + return value; + else if (!attValue.ns().isEmpty()) + return setToString(attValue.ns()); + else if (!attValue.ss().isEmpty()) + return setToString(attValue.ss()); + else if (attValue.hasM()) + return new JSONObject(convertToDictionary(attValue.m())).toString(); + else if (attValue.hasL()) + return new JSONArray(attValue.l().stream().map(item -> getString(item)).collect(Collectors.toList())).toString(); + return null; + } + + private static HashMap convertToDictionary(Map m) + { + HashMap dict = new HashMap<>(); + for (Map.Entry keyValues : m.entrySet()) + { + dict.put(keyValues.getKey(), getString(keyValues.getValue())); + } + return dict; + } + + private static String setToString(List nS) + { + return String.format("[ %s ]", String.join(", ", nS)); + } + +} diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java new file mode 100644 index 000000000..ae5fed4a3 --- /dev/null +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java @@ -0,0 +1,48 @@ +package com.genexus.db.dynamodb; + +import com.genexus.db.ServiceCursorBase; +import com.genexus.db.driver.GXConnection; +import com.genexus.db.service.ServicePreparedStatement; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; + +public class DynamoDBPreparedStatement extends ServicePreparedStatement +{ + final DynamoQuery query; + final ServiceCursorBase cursor; + + DynamoDBPreparedStatement(Connection con, DynamoQuery query, ServiceCursorBase cursor, Object[] parms, GXConnection gxCon) + { + super(con, parms, gxCon); + this.query = query; + this.cursor = cursor; + query.initializeParms(parms); + } + + @Override + public ResultSet executeQuery() throws SQLException + { + return new DynamoDBResultSet(this); + } + + DynamoDbClient getClient() throws SQLException + { + return ((DynamoDBConnection)getConnection()).mDynamoDB; + } + + /// JDK8 + @Override + public void closeOnCompletion() + { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public boolean isCloseOnCompletion() + { + throw new UnsupportedOperationException("Not supported yet."); + } +} diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java new file mode 100644 index 000000000..45ef65eab --- /dev/null +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java @@ -0,0 +1,312 @@ +package com.genexus.db.dynamodb; + +import com.genexus.db.service.IOServiceContext; +import com.genexus.db.service.ServiceResultSet; +import com.genexus.db.service.VarValue; +import com.genexus.util.NameValuePair; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.QueryRequest; +import software.amazon.awssdk.services.dynamodb.model.ScanRequest; + +import java.math.BigDecimal; +import java.sql.SQLException; +import java.sql.Time; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class DynamoDBResultSet extends ServiceResultSet +{ + private DynamoDBPreparedStatement stmt; + public DynamoDBResultSet(DynamoDBPreparedStatement stmt) throws SQLException + { + this.stmt = stmt; + execute(stmt.query, stmt.parms); + } + + private static final Pattern FILTER_PATTERN = Pattern.compile("\\((.*) = :(.*)\\)"); +// private DynamoDbRequest req = null; + private void execute(DynamoQuery query, Object[] parms) throws SQLException + { + DynamoDbClient client = stmt.getClient(); + + HashMap values = new HashMap(); + + for(Iterator it = query.getVars(); it.hasNext();) + { + VarValue var = it.next(); + values.put(var.name, DynamoDBHelper.ToAttributeValue(var)); + } + + for (Iterator it = query.getAssignAtts(); it.hasNext(); ) + { + NameValuePair asg = it.next(); + String name = trimSharp(asg.name); + String parmName = asg.value.substring(1); +//@todo +// if(!DynamoDBHelper.addAttributeValue(name, values, parms, parmName)) +// throw new SQLException(String.format("Cannot assign attribute value (name: %s)", parmName); + } + String pattern = "\\((.*) = :(.*)\\)"; + HashMap keyCondition = new HashMap<>(); + ArrayList filters = new ArrayList(); + + HashMap expressionAttributeNames = null; + + for (Iterator it = Arrays.stream(query.selectList) + .filter(selItem -> ((DynamoDBMap) selItem).needsAttributeMap()) + .map(selItem -> selItem.getName()).iterator(); it.hasNext(); ) + { + if(expressionAttributeNames == null) + expressionAttributeNames = new HashMap<>(); + String mappedName = it.next(); + String key = "#" + mappedName; + String value = mappedName; + expressionAttributeNames.put(key, value); + } + + for(String keyFilter : query.filters) + { + Matcher match = FILTER_PATTERN.matcher(keyFilter); + if(match.matches() && match.groupCount() > 1) + { + String varName = match.group(2); + String name = trimSharp(match.group(1)); +//@todo +// if(!DynamoDBHelper.addAttributeValue(name, values, parms[varName]as ServiceParameter); +// throw new SQLException(String.format("Cannot assign attribute value (name: %s)", parmName); + keyCondition.put(name, values.get(name)); + } + } + + switch (query.getQueryType()) + { + case QUERY: + { + if(query instanceof DynamoScan) + { + ScanRequest.Builder builder = ScanRequest.builder() + .tableName(query.tableName) + .projectionExpression(String.join(",", query.projection)); + if(query.filters.length > 0) + { + builder.filterExpression(String.join(" AND ", query.filters)).expressionAttributeValues(values); + } + if(expressionAttributeNames != null) + builder.expressionAttributeNames(expressionAttributeNames); + + iterator = client.scanPaginator(builder.build()) + .stream() + .flatMap(response -> response.items() + .stream() + .map(map -> new HashMap(map))) + .iterator(); + }else + { + QueryRequest.Builder builder = QueryRequest.builder() + .tableName(query.tableName) + .keyConditionExpression(String.join(" AND ", query.filters)) + .expressionAttributeValues(values) + .projectionExpression(String.join(", ", query.projection)) + .indexName(query.getIndex()) + .scanIndexForward(query.isScanIndexForward()); + if(expressionAttributeNames != null) + builder.expressionAttributeNames(expressionAttributeNames); + + iterator = client.queryPaginator(builder.build()) + .stream() + .flatMap(response -> response.items() + .stream() + .map(map -> new HashMap(map))) + .iterator(); + } + break; + } + default: throw new UnsupportedOperationException(String.format("Work in progress: %s", query.getQueryType())); + } + } + + private static String trimSharp(String name) + { + return name.startsWith("#") ? name.substring(1) : name; + } + + @Override + public boolean next() throws SQLException + { + if(iterator.hasNext()) + { + currentEntry = (HashMap) iterator.next(); + return true; + } + return false; + } + + private static final IOServiceContext SERVICE_CONTEXT = null; + + private boolean lastWasNull; + @Override + public boolean wasNull() throws SQLException + { + return value == null || lastWasNull; + } + + private AttributeValue getAttValue(int columnIndex) + { + value = (AttributeValue)stmt.query.selectList[columnIndex-1].getValue(SERVICE_CONTEXT, currentEntry); + return value; + } + + private long getNumeric(int columnIndex) + { + AttributeValue value = getAttValue(columnIndex); + String sNumber = value.n(); + lastWasNull = false; + if(sNumber != null) + return Long.parseLong(sNumber); + else if(value.bool() != null) + return value.bool().booleanValue() ? 1 : 0; + else + { + lastWasNull = true; + return 0; + } + } + + private double getDecimal(int columnIndex) + { + AttributeValue value = getAttValue(columnIndex); + String sNumber = value.n(); + if(sNumber == null) + { + lastWasNull = true; + return 0; + } + lastWasNull = false; + return Double.parseDouble(sNumber); + } + + private Instant getInstant(int columnIndex) throws SQLException + { + String value = getString(columnIndex); + if(value == null) + { + lastWasNull = true; + return Instant.EPOCH; + } + return Instant.parse(value); + } + + @Override + public T getAs(Class reference, int columnIndex, T defaultValue) throws SQLException + { + throw new UnsupportedOperationException(String.format("Data Type: %s", reference.getName())); + } + + @Override + public String getString(int columnIndex) throws SQLException + { + String value = DynamoDBHelper.getString(getAttValue(columnIndex)); + lastWasNull = value == null; + return value; + } + + @Override + public boolean getBoolean(int columnIndex) throws SQLException + { + Boolean value = getAttValue(columnIndex).bool(); + lastWasNull = value == null; + return !lastWasNull && value.booleanValue(); + } + + @Override + public byte getByte(int columnIndex) throws SQLException + { + return (byte)getNumeric(columnIndex); + } + + @Override + public short getShort(int columnIndex) throws SQLException + { + return (short)getNumeric(columnIndex); + } + + @Override + public int getInt(int columnIndex) throws SQLException + { + return (int)getNumeric(columnIndex); + } + + @Override + public long getLong(int columnIndex) throws SQLException + { + return (long)getNumeric(columnIndex); + } + + @Override + public float getFloat(int columnIndex) throws SQLException + { + return (float)getDecimal(columnIndex); + } + + @Override + public double getDouble(int columnIndex) throws SQLException + { + return (double)getDecimal(columnIndex); + } + + @Override + @Deprecated + public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException + { + return getBigDecimal(columnIndex).setScale(scale); + } + + @Override + public BigDecimal getBigDecimal(int columnIndex) throws SQLException + { + AttributeValue value = getAttValue(columnIndex); + String sNumber = value.n(); + if(sNumber == null) + { + lastWasNull = true; + return BigDecimal.ZERO; + } + lastWasNull = false; + return new BigDecimal(sNumber); + } + + @Override + public java.sql.Date getDate(int columnIndex) throws SQLException + { + return getAs(java.sql.Date.class, columnIndex, new java.sql.Date(0)); + } + + @Override + public Time getTime(int columnIndex) throws SQLException + { + return getAs(Time.class, columnIndex, new Time(0)); + } + + @Override + public Timestamp getTimestamp(int columnIndex) throws SQLException + { + return java.sql.Timestamp.from(getInstant(columnIndex)); + } + + // JDK8 + @Override + public T getObject(int columnIndex, Class type) + { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public T getObject(String columnLabel, Class type) + { + throw new UnsupportedOperationException("Not supported yet."); + } +} diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java index 85c7319b4..0b5a1e43a 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java @@ -8,6 +8,7 @@ public class DynamoQuery extends Query{ private static final String RANGE_KEY_INDEX = "RangeKey"; private static final char[] indexTrimChars = new char[] { '(', ')' }; + @Override public DynamoQuery orderBy(String index) { index = index.trim(); diff --git a/java/src/main/java/com/genexus/db/service/GXType.java b/java/src/main/java/com/genexus/db/service/GXType.java new file mode 100644 index 000000000..d9cacf4e3 --- /dev/null +++ b/java/src/main/java/com/genexus/db/service/GXType.java @@ -0,0 +1,41 @@ +package com.genexus.db.service; + +public enum GXType +{ + Number(0), + Int16(1), + Int32(2), + Int64(3), + Date(4), + DateTime(5), + DateTime2(17), + Byte(6), + NChar(7), + NClob(8), + NVarChar(9), + Char(10), + LongVarChar(11), + Clob(12), + VarChar(13), + Raw(14), + Blob(15), + Undefined(16), + Boolean(18), + Decimal(19), + NText(20), + Text(21), + Image(22), + UniqueIdentifier(23), + Xml(24), + Geography(25), + Geopoint(26), + Geoline(27), + Geopolygon(28), + DateAsChar(29); + + private final int value; + GXType(final int value) + { + this.value = value; + } +} \ No newline at end of file diff --git a/java/src/main/java/com/genexus/db/service/Query.java b/java/src/main/java/com/genexus/db/service/Query.java index 48e8f7d0b..bea7e4375 100644 --- a/java/src/main/java/com/genexus/db/service/Query.java +++ b/java/src/main/java/com/genexus/db/service/Query.java @@ -1,6 +1,10 @@ package com.genexus.db.service; -import java.util.Collections; +import com.genexus.util.NameValuePair; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; public class Query implements IQuery { private static final String [] EMPTY_ARR_STRING = new String [0]; @@ -10,20 +14,19 @@ public Query(ServiceDataStoreHelper dataStoreHelper) this.dataStoreHelper = dataStoreHelper; } - private String tableName; + public String tableName; public String[] projection = EMPTY_ARR_STRING; -/* - public string[] OrderBys { get; set; } = Array.Empty(); - public string[] Filters { get; set; } = Array.Empty(); - private List> mAssignAtts; - public IEnumerable> AssignAtts { get { return mAssignAtts ?? Array.Empty>() as IEnumerable>; } } -*/ + public String[] orderBys = EMPTY_ARR_STRING; + public String[] filters = EMPTY_ARR_STRING; + + private final ArrayList mAssignAtts = new ArrayList<>(); + public Iterator getAssignAtts() { return mAssignAtts.iterator(); } + public IODataMap[] selectList = new IODataMap[0]; -/* - private List mVarValues; - public IEnumerable Vars { get { return (mVarValues ?? Array.Empty() as IEnumerable); } } - public CursorType CursorType { get; set; } = CursorType.Select; - */ + private final HashMap mVarValues = new HashMap<>(); + public Iterator getVars() { return mVarValues.values().iterator(); } + + protected final HashMap parmTypes = new HashMap<>(); public Query For(String tableName) { @@ -36,49 +39,68 @@ public Query select(String[] columns) this.projection = columns; return this; } -/* - public Query OrderBy(string[] orders) + + public Query orderBy(String []orders) { - OrderBys = orders; + orderBys = orders; return this; } - public Query Filter(string[] filters) + public Query orderBy(String order) { - Filters = filters; + return orderBy(new String[]{ order }); + } + + public Query filter(String[] filters) + { + this.filters = filters; return this; } - public Query Set(string name, string value) + public Query set(String name, String value) { - mAssignAtts = mAssignAtts ?? new List>(); - mAssignAtts.Add(new KeyValuePair(name, value)); + mAssignAtts.add(new NameValuePair(name, value)); return this; } -*/ + public Query setMaps(IODataMap[] selectList) { this.selectList = selectList; return this; } + protected QueryType queryType = QueryType.QUERY; @Override - public QueryType getQueryType() { - return QueryType.QUERY; + public QueryType getQueryType() + { + return queryType; + } + + public Query setType(QueryType queryType) + { + this.queryType = queryType; + return this; } -/* - public Query SetType(CursorType cType) + public Query addConst(GXType gxType, Object parm) { - CursorType = cType; + String parmName = String.format(":const%d", mVarValues.size() + 1); + mVarValues.put(parmName, new VarValue(parmName, gxType, parm)); return this; } - public Query AddParm(GXType gxType, object parm) + public Query setParmType(int parmId, GXType gxType) { - mVarValues = mVarValues ?? new List(); - mVarValues.Add(new VarValue($":parm{ mVarValues.Count + 1 }", gxType, parm)); + parmTypes.put(parmId, gxType); return this; } -*/ + + public void initializeParms(Object[] parms) + { + for(int idx : parmTypes.keySet()) + { + String parmName = String.format(":parm%d", idx); + mVarValues.put(parmName, new VarValue(parmName, parmTypes.get(idx), parms[idx])); + } + } } diff --git a/java/src/main/java/com/genexus/db/service/ServiceResultSet.java b/java/src/main/java/com/genexus/db/service/ServiceResultSet.java index d31a46c37..f1f74084f 100644 --- a/java/src/main/java/com/genexus/db/service/ServiceResultSet.java +++ b/java/src/main/java/com/genexus/db/service/ServiceResultSet.java @@ -107,12 +107,6 @@ public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException return getAs(BigDecimal.class, columnIndex, BigDecimal.ZERO); } - @Override - public byte[] getBytes(int columnIndex) throws SQLException - { - throw new UnsupportedOperationException("Not supported yet."); - } - @Override public Date getDate(int columnIndex) throws SQLException { @@ -150,7 +144,33 @@ public InputStream getBinaryStream(int columnIndex) throws SQLException return getAs(InputStream.class, columnIndex, new ByteArrayInputStream(new byte[0])); } - @Override + @Override + public Object getObject(int columnIndex) throws SQLException + { + return getAs(Object.class, columnIndex, new Object()); + } + + @Override + public Reader getCharacterStream(int columnIndex) throws SQLException + { + return getAs(Reader.class, columnIndex, new StringReader("")); + } + + @Override + public BigDecimal getBigDecimal(int columnIndex) throws SQLException + { + return getAs(BigDecimal.class, columnIndex, BigDecimal.ZERO); + } + + // Unsupported methods + + @Override + public byte[] getBytes(int columnIndex) throws SQLException + { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override public String getString(String columnLabel) throws SQLException { throw new UnsupportedOperationException("Not supported yet."); @@ -272,12 +292,6 @@ public ResultSetMetaData getMetaData() throws SQLException throw new UnsupportedOperationException("Not supported yet."); } - @Override - public Object getObject(int columnIndex) throws SQLException - { - return getAs(Object.class, columnIndex, new Object()); - } - @Override public Object getObject(String columnLabel) throws SQLException { @@ -290,24 +304,12 @@ public int findColumn(String columnLabel) throws SQLException throw new UnsupportedOperationException("Not supported yet."); } - @Override - public Reader getCharacterStream(int columnIndex) throws SQLException - { - return getAs(Reader.class, columnIndex, new StringReader("")); - } - @Override public Reader getCharacterStream(String columnLabel) throws SQLException { throw new UnsupportedOperationException("Not supported yet."); } - @Override - public BigDecimal getBigDecimal(int columnIndex) throws SQLException - { - return getAs(BigDecimal.class, columnIndex, BigDecimal.ZERO); - } - @Override public BigDecimal getBigDecimal(String columnLabel) throws SQLException { diff --git a/java/src/main/java/com/genexus/db/service/VarValue.java b/java/src/main/java/com/genexus/db/service/VarValue.java new file mode 100644 index 000000000..29423dfa1 --- /dev/null +++ b/java/src/main/java/com/genexus/db/service/VarValue.java @@ -0,0 +1,15 @@ +package com.genexus.db.service; + +public class VarValue +{ + public String name; + public Object value; + public GXType type; + + public VarValue(String name, GXType type, Object value) + { + this.name = name; + this.type = type; + this.value = value; + } +} From a3247de55f31c1cf8d87aff48c02b3ef6fba2d9c Mon Sep 17 00:00:00 2001 From: Gustavo Brown Date: Mon, 14 Mar 2022 15:14:49 -0300 Subject: [PATCH 03/24] Add support for CRUD operations --- .../genexus/db/dynamodb/DynamoDBHelper.java | 19 +- .../dynamodb/DynamoDBPreparedStatement.java | 172 +++++++++++++++++- .../db/dynamodb/DynamoDBResultSet.java | 67 ++++--- .../com/genexus/db/dynamodb/DynamoQuery.java | 2 +- .../java/com/genexus/db/service/Query.java | 5 + 5 files changed, 233 insertions(+), 32 deletions(-) diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java index dda644032..2364bac7b 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java @@ -13,7 +13,7 @@ public class DynamoDBHelper { - public static AttributeValue ToAttributeValue(VarValue var) throws SQLException + public static AttributeValue toAttributeValue(VarValue var) throws SQLException { Object value = var.value; AttributeValue.Builder builder = AttributeValue.builder(); @@ -55,7 +55,7 @@ public static AttributeValue ToAttributeValue(VarValue var) throws SQLException case Image: case DateAsChar: default: - throw new SQLException(String.format("DynamoDB unsupported type (%s)", var.type.toString())); + throw new SQLException(String.format("DynamoDB unsupported type (%s)", var.type)); } } @@ -73,7 +73,7 @@ else if (!attValue.ss().isEmpty()) else if (attValue.hasM()) return new JSONObject(convertToDictionary(attValue.m())).toString(); else if (attValue.hasL()) - return new JSONArray(attValue.l().stream().map(item -> getString(item)).collect(Collectors.toList())).toString(); + return new JSONArray(attValue.l().stream().map(DynamoDBHelper::getString).collect(Collectors.toList())).toString(); return null; } @@ -92,4 +92,17 @@ private static String setToString(List nS) return String.format("[ %s ]", String.join(", ", nS)); } + public static boolean addAttributeValue(String parmName, HashMap values, VarValue parm) throws SQLException + { + if(parm == null) + return false; + AttributeValue value = toAttributeValue(parm); + if (value != null) + { + values.put(parmName, value); + return true; + } + return false; + + } } diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java index ae5fed4a3..58f4b4bd9 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java @@ -2,12 +2,20 @@ import com.genexus.db.ServiceCursorBase; import com.genexus.db.driver.GXConnection; +import com.genexus.db.service.IODataMap; +import com.genexus.db.service.QueryType; import com.genexus.db.service.ServicePreparedStatement; +import com.genexus.db.service.VarValue; +import com.genexus.util.NameValuePair; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.*; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class DynamoDBPreparedStatement extends ServicePreparedStatement { @@ -25,7 +33,169 @@ public class DynamoDBPreparedStatement extends ServicePreparedStatement @Override public ResultSet executeQuery() throws SQLException { - return new DynamoDBResultSet(this); + DynamoDBResultSet resultSet = new DynamoDBResultSet(this); + _executeQuery(resultSet); + return resultSet; + } + + @Override + public int executeUpdate() throws SQLException + { + return _executeQuery(null); + } + + private static final Pattern FILTER_PATTERN = Pattern.compile("\\((.*) = (:.*)\\)"); + private int _executeQuery(DynamoDBResultSet resultSet) throws SQLException + { + DynamoDbClient client = getClient(); + + HashMap values = new HashMap<>(); + + for(Iterator it = query.getVars(); it.hasNext();) + { + VarValue var = it.next(); + values.put(var.name, DynamoDBHelper.toAttributeValue(var)); + } + + for (Iterator it = query.getAssignAtts(); it.hasNext(); ) + { + NameValuePair asg = it.next(); + String name = trimSharp(asg.name); + String parmName = asg.value; + if(!DynamoDBHelper.addAttributeValue(name, values, query.getParm(parmName))) + throw new SQLException(String.format("Cannot assign attribute value (name: %s)", parmName)); + } + String pattern = "\\((.*) = :(.*)\\)"; + HashMap keyCondition = new HashMap<>(); + ArrayList filters = new ArrayList<>(); + + HashMap expressionAttributeNames = null; + + for (Iterator it = Arrays.stream(query.selectList) + .filter(selItem -> ((DynamoDBMap) selItem).needsAttributeMap()) + .map(IODataMap::getName).iterator(); it.hasNext(); ) + { + if(expressionAttributeNames == null) + expressionAttributeNames = new HashMap<>(); + String mappedName = it.next(); + String key = "#" + mappedName; + String value = mappedName; + expressionAttributeNames.put(key, value); + } + + if(query.getQueryType() != QueryType.QUERY) + { + for (String keyFilter : query.filters) + { + Matcher match = FILTER_PATTERN.matcher(keyFilter); + if (match.matches() && match.groupCount() > 1) + { + String varName = match.group(2); + String name = trimSharp(match.group(1)); + if (!DynamoDBHelper.addAttributeValue(name, values, query.getParm(varName))) + throw new SQLException(String.format("Cannot assign attribute value (name: %s)", varName)); + keyCondition.put(name, values.get(name)); + } + } + } + + switch (query.getQueryType()) + { + case QUERY: + { + Iterator> iterator; + if(query instanceof DynamoScan) + { + ScanRequest.Builder builder = ScanRequest.builder() + .tableName(query.tableName) + .projectionExpression(String.join(",", query.projection)); + if(query.filters.length > 0) + { + builder.filterExpression(String.join(" AND ", query.filters)).expressionAttributeValues(values); + } + if(expressionAttributeNames != null) + builder.expressionAttributeNames(expressionAttributeNames); + + iterator = client.scanPaginator(builder.build()) + .stream() + .flatMap(response -> response.items() + .stream() + .map(map -> new HashMap(map))) + .iterator(); + }else + { + QueryRequest.Builder builder = QueryRequest.builder() + .tableName(query.tableName) + .keyConditionExpression(String.join(" AND ", query.filters)) + .expressionAttributeValues(values) + .projectionExpression(String.join(", ", query.projection)) + .indexName(query.getIndex()) + .scanIndexForward(query.isScanIndexForward()); + if(expressionAttributeNames != null) + builder.expressionAttributeNames(expressionAttributeNames); + + iterator = client.queryPaginator(builder.build()) + .stream() + .flatMap(response -> response.items() + .stream() + .map(map -> new HashMap(map))) + .iterator(); + } + resultSet.iterator = iterator; + return 0; + } + case INS: + { + PutItemRequest request = PutItemRequest.builder() + .tableName(query.tableName) + .item(values) + .build(); + client.putItem(request); + break; + } + case UPD: + { + UpdateItemRequest request = UpdateItemRequest.builder() + .tableName(query.tableName) + .key(keyCondition) + .attributeUpdates(toAttributeUpdates(keyCondition, values)) + .build(); + client.updateItem(request); + break; + } + case DLT: + { + DeleteItemRequest request = DeleteItemRequest.builder() + .tableName(query.tableName) + .key(keyCondition) + .build(); + client.deleteItem(request); + break; + } + default: throw new UnsupportedOperationException(String.format("Work in progress: %s", query.getQueryType())); + } + return 0; + } + + private HashMap toAttributeUpdates(HashMap keyConditions, HashMap values) + { + HashMap updates = new HashMap<>(); + for(Map.Entry item : values.entrySet()) + { + if (!keyConditions.containsKey(item.getKey()) && !item.getKey().startsWith("AV")) + { + updates.put(item.getKey(), AttributeValueUpdate.builder() + .value(item.getValue()) + .action(AttributeAction.PUT) + .build()); + } + } + return updates; + } + + private static String trimSharp(String name) + { + return name.startsWith("#") ? name.substring(1) : name; } DynamoDbClient getClient() throws SQLException diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java index 45ef65eab..b65d57012 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java @@ -24,11 +24,10 @@ public class DynamoDBResultSet extends ServiceResultSet public DynamoDBResultSet(DynamoDBPreparedStatement stmt) throws SQLException { this.stmt = stmt; - execute(stmt.query, stmt.parms); +// execute(stmt.query, stmt.parms); } - +/* private static final Pattern FILTER_PATTERN = Pattern.compile("\\((.*) = :(.*)\\)"); -// private DynamoDbRequest req = null; private void execute(DynamoQuery query, Object[] parms) throws SQLException { DynamoDbClient client = stmt.getClient(); @@ -133,6 +132,7 @@ private static String trimSharp(String name) { return name.startsWith("#") ? name.substring(1) : name; } + */ @Override public boolean next() throws SQLException @@ -163,30 +163,31 @@ private AttributeValue getAttValue(int columnIndex) private long getNumeric(int columnIndex) { AttributeValue value = getAttValue(columnIndex); - String sNumber = value.n(); - lastWasNull = false; - if(sNumber != null) - return Long.parseLong(sNumber); - else if(value.bool() != null) - return value.bool().booleanValue() ? 1 : 0; - else + if(value != null) { - lastWasNull = true; - return 0; + lastWasNull = false; + String sNumber = value.n(); + if (sNumber != null) + return Long.parseLong(sNumber); + else if (value.bool() != null) + return value.bool().booleanValue() ? 1 : 0; } + lastWasNull = true; + return 0; } private double getDecimal(int columnIndex) { AttributeValue value = getAttValue(columnIndex); - String sNumber = value.n(); - if(sNumber == null) + if(value != null) { - lastWasNull = true; - return 0; + lastWasNull = false; + String sNumber = value.n(); + if (sNumber != null) + return Double.parseDouble(sNumber); } - lastWasNull = false; - return Double.parseDouble(sNumber); + lastWasNull = true; + return 0; } private Instant getInstant(int columnIndex) throws SQLException @@ -217,9 +218,18 @@ public String getString(int columnIndex) throws SQLException @Override public boolean getBoolean(int columnIndex) throws SQLException { - Boolean value = getAttValue(columnIndex).bool(); - lastWasNull = value == null; - return !lastWasNull && value.booleanValue(); + AttributeValue value = getAttValue(columnIndex); + if(value != null) + { + Boolean boolValue = value.bool(); + if(boolValue != null) + { + lastWasNull = false; + return boolValue.booleanValue(); + } + } + lastWasNull = true; + return false; } @Override @@ -269,14 +279,17 @@ public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException public BigDecimal getBigDecimal(int columnIndex) throws SQLException { AttributeValue value = getAttValue(columnIndex); - String sNumber = value.n(); - if(sNumber == null) + if(value != null) { - lastWasNull = true; - return BigDecimal.ZERO; + String sNumber = value.n(); + if (sNumber != null) + { + lastWasNull = false; + return new BigDecimal(sNumber); + } } - lastWasNull = false; - return new BigDecimal(sNumber); + lastWasNull = true; + return BigDecimal.ZERO; } @Override diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java index 0b5a1e43a..375139c11 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java @@ -15,7 +15,7 @@ public DynamoQuery orderBy(String index) if(index.startsWith("(") && index.endsWith(")")) { scanIndexForward = false; - index = index.substring(1, index.length()-2); + index = index.substring(1, index.length()-1); } if (!RANGE_KEY_INDEX.equals(index)) setIndex(index); diff --git a/java/src/main/java/com/genexus/db/service/Query.java b/java/src/main/java/com/genexus/db/service/Query.java index bea7e4375..b80696509 100644 --- a/java/src/main/java/com/genexus/db/service/Query.java +++ b/java/src/main/java/com/genexus/db/service/Query.java @@ -26,6 +26,11 @@ public Query(ServiceDataStoreHelper dataStoreHelper) private final HashMap mVarValues = new HashMap<>(); public Iterator getVars() { return mVarValues.values().iterator(); } + public VarValue getParm(String parmName) + { + return mVarValues.get(parmName); + } + protected final HashMap parmTypes = new HashMap<>(); public Query For(String tableName) From c411c572bf5325136af8b3f97ebd5730a97e3d1b Mon Sep 17 00:00:00 2001 From: Gustavo Brown Date: Tue, 15 Mar 2022 17:43:12 -0300 Subject: [PATCH 04/24] Round some edges --- gxdynamodb/pom.xml | 22 +++- .../db/dynamodb/DataStoreHelperDynamoDB.java | 5 +- .../db/dynamodb/DynamoDBConnection.java | 71 +---------- .../com/genexus/db/dynamodb/DynamoDBMap.java | 1 - .../dynamodb/DynamoDBPreparedStatement.java | 3 +- .../db/dynamodb/DynamoDBResultSet.java | 120 ++---------------- 6 files changed, 35 insertions(+), 187 deletions(-) diff --git a/gxdynamodb/pom.xml b/gxdynamodb/pom.xml index d4942a2ee..f0a647eb5 100644 --- a/gxdynamodb/pom.xml +++ b/gxdynamodb/pom.xml @@ -7,7 +7,7 @@ com.genexus parent - 2.4-SNAPSHOT + 2.5-SNAPSHOT gxdynamodb @@ -44,9 +44,25 @@ gxclassR ${project.version} - + + com.genexus + gxcommon + 2.5-SNAPSHOT + compile + + gxdynamodb - + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + \ No newline at end of file diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java index bb56c4fb2..1c34a4bc6 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java @@ -4,8 +4,9 @@ import com.genexus.db.ServiceCursorBase; import com.genexus.db.driver.GXConnection; import com.genexus.db.driver.GXPreparedStatement; -import com.genexus.db.service.*; -import java.util.function.*; +import com.genexus.db.service.GXType; +import com.genexus.db.service.IQuery; +import com.genexus.db.service.ServiceDataStoreHelper; public class DataStoreHelperDynamoDB extends ServiceDataStoreHelper { diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java index ec2091788..2a246c08b 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java @@ -30,7 +30,7 @@ public class DynamoDBConnection extends ServiceConnection private static final String LOCAL_URL = "localurl"; - DynamoDbClient mDynamoDB; //mDynamoDB + DynamoDbClient mDynamoDB; Region mRegion = Region.US_EAST_1; public DynamoDBConnection(String connUrl, Properties initialConnProps) throws SQLException @@ -63,78 +63,9 @@ private void initializeDBConnection(String connUrl) AwsBasicCredentials mCredentials = AwsBasicCredentials.create(mClientId, mClientSecret); builder = builder.credentialsProvider(StaticCredentialsProvider.create(mCredentials)); } - mDynamoDB = builder.build(); } -/* ServiceError getServiceError(String errorCode) - { - if(errorCode != null) - { - if (modelInfo.recordNotFoundServiceCodes != null && modelInfo.recordNotFoundServiceCodes.contains(errorCode)) - return ServiceError.OBJECT_NOT_FOUND; - if (modelInfo.recordAlreadyExistsServiceCodes != null && modelInfo.recordAlreadyExistsServiceCodes.contains(errorCode)) - return ServiceError.DUPLICATE_KEY; - } - return ServiceError.INVALID_QUERY; - }*/ - -/* private boolean getBoolean(String value) - { - return value.equalsIgnoreCase("y") || value.equalsIgnoreCase("true") || value.equalsIgnoreCase("yes"); - } - - boolean needsCheckOptimisticConcurrency(URI updURI) - { - return modelInfo.needsCheckOptimisticConcurrency(updURI); - } - - public int getEnumValue(ClientEnumValue enumValue) - { - String typeName = enumValue.getTypeName(); - return Integer.parseInt(modelInfo.getModel().getEnumType(new FullQualifiedName(typeName)).getMember(enumValue.getValue()).getValue()); - } - - public String toEnumValue(EdmEnumType type, int value) - { - String sValue = Integer.toString(value); - for(String memberName:type.getMemberNames()) - { - EdmMember member = type.getMember(memberName); - if(member.getValue().equals(sValue)) - return member.getName(); - } - throw new RuntimeException(String.format("Cannot parse enum value %s - %d", type.toString(), value)); - } - - private String entity(String name) - { - return modelInfo.entity(name); - } - - public String entity(EdmEntityType [] fromEntity, String name) - { - if(fromEntity == null || fromEntity[0] == null) - return entity((EdmEntityType)null, name); - String entityName = entity(fromEntity[0], name); - if(entityName == null) - return entity(name); - EdmNavigationProperty navProp = fromEntity[0].getNavigationProperty(entityName); - if(navProp != null) - fromEntity[0] = navProp.getType(); - return entityName; - } - - public String entity(EdmEntityType fromEntity, String name) - { - return modelInfo.entity(fromEntity, name); - } - - Edm getModel() - { - return modelInfo.getModel(); - } -*/ //---------------------------------------------------------------------------------------------------- @Override diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBMap.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBMap.java index 2695e5fb3..d6c89afe9 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBMap.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBMap.java @@ -3,7 +3,6 @@ import com.genexus.db.service.*; public class DynamoDBMap extends IODataMapName{ -//ver DynamoDBMaps.cs public DynamoDBMap(String name) { super(mapAttributeMap(name)); needsAttributeMap = _needsAttributeMap(name); diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java index 58f4b4bd9..d3a96a9c1 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java @@ -111,7 +111,8 @@ private int _executeQuery(DynamoDBResultSet resultSet) throws SQLException .projectionExpression(String.join(",", query.projection)); if(query.filters.length > 0) { - builder.filterExpression(String.join(" AND ", query.filters)).expressionAttributeValues(values); + builder.filterExpression(String.join(" AND ", query.filters)) + .expressionAttributeValues(values); } if(expressionAttributeNames != null) builder.expressionAttributeNames(expressionAttributeNames); diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java index b65d57012..3cde80af4 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java @@ -6,6 +6,7 @@ import com.genexus.util.NameValuePair; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.DynamoDbException; import software.amazon.awssdk.services.dynamodb.model.QueryRequest; import software.amazon.awssdk.services.dynamodb.model.ScanRequest; @@ -24,123 +25,22 @@ public class DynamoDBResultSet extends ServiceResultSet public DynamoDBResultSet(DynamoDBPreparedStatement stmt) throws SQLException { this.stmt = stmt; -// execute(stmt.query, stmt.parms); } -/* - private static final Pattern FILTER_PATTERN = Pattern.compile("\\((.*) = :(.*)\\)"); - private void execute(DynamoQuery query, Object[] parms) throws SQLException - { - DynamoDbClient client = stmt.getClient(); - - HashMap values = new HashMap(); - - for(Iterator it = query.getVars(); it.hasNext();) - { - VarValue var = it.next(); - values.put(var.name, DynamoDBHelper.ToAttributeValue(var)); - } - - for (Iterator it = query.getAssignAtts(); it.hasNext(); ) - { - NameValuePair asg = it.next(); - String name = trimSharp(asg.name); - String parmName = asg.value.substring(1); -//@todo -// if(!DynamoDBHelper.addAttributeValue(name, values, parms, parmName)) -// throw new SQLException(String.format("Cannot assign attribute value (name: %s)", parmName); - } - String pattern = "\\((.*) = :(.*)\\)"; - HashMap keyCondition = new HashMap<>(); - ArrayList filters = new ArrayList(); - - HashMap expressionAttributeNames = null; - - for (Iterator it = Arrays.stream(query.selectList) - .filter(selItem -> ((DynamoDBMap) selItem).needsAttributeMap()) - .map(selItem -> selItem.getName()).iterator(); it.hasNext(); ) - { - if(expressionAttributeNames == null) - expressionAttributeNames = new HashMap<>(); - String mappedName = it.next(); - String key = "#" + mappedName; - String value = mappedName; - expressionAttributeNames.put(key, value); - } - - for(String keyFilter : query.filters) - { - Matcher match = FILTER_PATTERN.matcher(keyFilter); - if(match.matches() && match.groupCount() > 1) - { - String varName = match.group(2); - String name = trimSharp(match.group(1)); -//@todo -// if(!DynamoDBHelper.addAttributeValue(name, values, parms[varName]as ServiceParameter); -// throw new SQLException(String.format("Cannot assign attribute value (name: %s)", parmName); - keyCondition.put(name, values.get(name)); - } - } - switch (query.getQueryType()) + @Override + public boolean next() throws SQLException + { + try { - case QUERY: + if (iterator.hasNext()) { - if(query instanceof DynamoScan) - { - ScanRequest.Builder builder = ScanRequest.builder() - .tableName(query.tableName) - .projectionExpression(String.join(",", query.projection)); - if(query.filters.length > 0) - { - builder.filterExpression(String.join(" AND ", query.filters)).expressionAttributeValues(values); - } - if(expressionAttributeNames != null) - builder.expressionAttributeNames(expressionAttributeNames); - - iterator = client.scanPaginator(builder.build()) - .stream() - .flatMap(response -> response.items() - .stream() - .map(map -> new HashMap(map))) - .iterator(); - }else - { - QueryRequest.Builder builder = QueryRequest.builder() - .tableName(query.tableName) - .keyConditionExpression(String.join(" AND ", query.filters)) - .expressionAttributeValues(values) - .projectionExpression(String.join(", ", query.projection)) - .indexName(query.getIndex()) - .scanIndexForward(query.isScanIndexForward()); - if(expressionAttributeNames != null) - builder.expressionAttributeNames(expressionAttributeNames); - - iterator = client.queryPaginator(builder.build()) - .stream() - .flatMap(response -> response.items() - .stream() - .map(map -> new HashMap(map))) - .iterator(); - } - break; + currentEntry = (HashMap) iterator.next(); + return true; } - default: throw new UnsupportedOperationException(String.format("Work in progress: %s", query.getQueryType())); } - } - - private static String trimSharp(String name) - { - return name.startsWith("#") ? name.substring(1) : name; - } - */ - - @Override - public boolean next() throws SQLException - { - if(iterator.hasNext()) + catch(DynamoDbException ex) { - currentEntry = (HashMap) iterator.next(); - return true; + throw ex; } return false; } From 1e1aa37e164ba8084ef05a14ddc768999baae9d8 Mon Sep 17 00:00:00 2001 From: Gustavo Brown Date: Tue, 15 Mar 2022 17:55:20 -0300 Subject: [PATCH 05/24] rolled back unintended commit --- .../com/genexus/db/dynamodb/DynamoDBResultSet.java | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java index 3cde80af4..01470484d 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java @@ -6,7 +6,6 @@ import com.genexus.util.NameValuePair; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; -import software.amazon.awssdk.services.dynamodb.model.DynamoDbException; import software.amazon.awssdk.services.dynamodb.model.QueryRequest; import software.amazon.awssdk.services.dynamodb.model.ScanRequest; @@ -30,17 +29,10 @@ public DynamoDBResultSet(DynamoDBPreparedStatement stmt) throws SQLException @Override public boolean next() throws SQLException { - try + if(iterator.hasNext()) { - if (iterator.hasNext()) - { - currentEntry = (HashMap) iterator.next(); - return true; - } - } - catch(DynamoDbException ex) - { - throw ex; + currentEntry = (HashMap) iterator.next(); + return true; } return false; } From 56fe85ba93a48d3532d7de73ba9e9015be3c6208 Mon Sep 17 00:00:00 2001 From: Gustavo Brown Date: Wed, 16 Mar 2022 17:50:21 -0300 Subject: [PATCH 06/24] Optimize imports --- gxdynamodb/pom.xml | 10 ---------- .../genexus/db/dynamodb/DynamoDBConnection.java | 14 ++++---------- .../java/com/genexus/db/dynamodb/DynamoDBMap.java | 2 +- .../com/genexus/db/dynamodb/DynamoDBResultSet.java | 9 +-------- .../java/com/genexus/db/dynamodb/DynamoQuery.java | 3 ++- 5 files changed, 8 insertions(+), 30 deletions(-) diff --git a/gxdynamodb/pom.xml b/gxdynamodb/pom.xml index f0a647eb5..4c7f578d2 100644 --- a/gxdynamodb/pom.xml +++ b/gxdynamodb/pom.xml @@ -25,16 +25,6 @@ - - - software.amazon.awssdk - dynamodb-enhanced - 2.17.136 - software.amazon.awssdk dynamodb diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java index 2a246c08b..0db3070a1 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java @@ -2,22 +2,16 @@ import com.genexus.db.service.ServiceConnection; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.enhanced.dynamodb.DynamoDbEnhancedClient; -import software.amazon.awssdk.services.dynamodb.DynamoDbClientBuilder; -import software.amazon.awssdk.services.dynamodb.model.AttributeValue; -import software.amazon.awssdk.services.dynamodb.model.QueryRequest; -import software.amazon.awssdk.services.dynamodb.model.QueryResponse; +import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.DynamoDbClientBuilder; import java.net.URI; -import java.util.HashMap; -import software.amazon.awssdk.services.dynamodb.model.DynamoDbException; -import software.amazon.awssdk.regions.Region;import java.nio.charset.StandardCharsets; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.*; +import java.util.Enumeration; +import java.util.Properties; import java.util.concurrent.Executor; public class DynamoDBConnection extends ServiceConnection diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBMap.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBMap.java index d6c89afe9..29e16823e 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBMap.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBMap.java @@ -1,6 +1,6 @@ package com.genexus.db.dynamodb; -import com.genexus.db.service.*; +import com.genexus.db.service.IODataMapName; public class DynamoDBMap extends IODataMapName{ public DynamoDBMap(String name) { diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java index 01470484d..1521e8c0f 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java @@ -2,21 +2,14 @@ import com.genexus.db.service.IOServiceContext; import com.genexus.db.service.ServiceResultSet; -import com.genexus.db.service.VarValue; -import com.genexus.util.NameValuePair; -import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; -import software.amazon.awssdk.services.dynamodb.model.QueryRequest; -import software.amazon.awssdk.services.dynamodb.model.ScanRequest; import java.math.BigDecimal; import java.sql.SQLException; import java.sql.Time; import java.sql.Timestamp; import java.time.Instant; -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +import java.util.HashMap; public class DynamoDBResultSet extends ServiceResultSet { diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java index 375139c11..3c68674da 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java @@ -1,5 +1,6 @@ package com.genexus.db.dynamodb; -import com.genexus.db.service.*; + +import com.genexus.db.service.Query; public class DynamoQuery extends Query{ private String index; From 7fc789c3daaec9ebdb709af84698b3aca429d99c Mon Sep 17 00:00:00 2001 From: Gustavo Brown Date: Thu, 17 Mar 2022 02:30:34 -0300 Subject: [PATCH 07/24] Fix error with blank password and local url. Fix lint errors --- .../db/dynamodb/DataStoreHelperDynamoDB.java | 2 +- .../db/dynamodb/DynamoDBConnection.java | 10 ++--- .../genexus/db/dynamodb/DynamoDBHelper.java | 3 +- .../dynamodb/DynamoDBPreparedStatement.java | 5 +-- .../db/dynamodb/DynamoDBResultSet.java | 43 ++++++++----------- .../genexus/db/service/ServiceResultSet.java | 2 +- 6 files changed, 29 insertions(+), 36 deletions(-) diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java index 1c34a4bc6..cbb468772 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java @@ -51,7 +51,7 @@ public Object empty(GXType gxtype) case Text: case Image: case UniqueIdentifier: - case Xml: return ""; + case Xml: case DateAsChar: return ""; case Geography: diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java index 0db3070a1..6373495f6 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java @@ -9,7 +9,6 @@ import java.net.URI; import java.sql.ResultSet; -import java.sql.SQLException; import java.util.Enumeration; import java.util.Properties; import java.util.concurrent.Executor; @@ -27,7 +26,7 @@ public class DynamoDBConnection extends ServiceConnection DynamoDbClient mDynamoDB; Region mRegion = Region.US_EAST_1; - public DynamoDBConnection(String connUrl, Properties initialConnProps) throws SQLException + public DynamoDBConnection(String connUrl, Properties initialConnProps) { super(connUrl, initialConnProps); // After initialization use props variable from super class to manage properties initializeDBConnection(connUrl); @@ -36,9 +35,9 @@ public DynamoDBConnection(String connUrl, Properties initialConnProps) throws SQ private void initializeDBConnection(String connUrl) { String mLocalUrl = null, mClientId = null, mClientSecret = null; - for(Enumeration keys = props.keys(); keys.hasMoreElements(); ) + for(Enumeration keys = props.keys(); keys.hasMoreElements(); ) { - String key = ((String)keys.nextElement()); + String key = (String)keys.nextElement(); String value = props.getProperty(key, key); switch(key.toLowerCase()) { @@ -52,7 +51,8 @@ private void initializeDBConnection(String connUrl) DynamoDbClientBuilder builder = DynamoDbClient.builder().region(mRegion); if(mLocalUrl != null) builder = builder.endpointOverride(URI.create(mLocalUrl)); - if(mClientId != null && mClientSecret != null) + if(mClientId != null && !mClientId.equals("") && + mClientSecret != null && !mClientSecret.equals("")) { AwsBasicCredentials mCredentials = AwsBasicCredentials.create(mClientId, mClientSecret); builder = builder.credentialsProvider(StaticCredentialsProvider.create(mCredentials)); diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java index 2364bac7b..d7b9e7e70 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java @@ -39,7 +39,8 @@ public static AttributeValue toAttributeValue(VarValue var) throws SQLException case NChar: case NVarChar: case NText: - return builder.s(value.toString()).build(); + + // Unused datatypes case UniqueIdentifier: case Xml: case Geography: diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java index d3a96a9c1..c59067b30 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java @@ -79,8 +79,7 @@ private int _executeQuery(DynamoDBResultSet resultSet) throws SQLException expressionAttributeNames = new HashMap<>(); String mappedName = it.next(); String key = "#" + mappedName; - String value = mappedName; - expressionAttributeNames.put(key, value); + expressionAttributeNames.put(key, mappedName); } if(query.getQueryType() != QueryType.QUERY) @@ -173,7 +172,7 @@ private int _executeQuery(DynamoDBResultSet resultSet) throws SQLException client.deleteItem(request); break; } - default: throw new UnsupportedOperationException(String.format("Work in progress: %s", query.getQueryType())); + default: throw new UnsupportedOperationException(String.format("Invalid query type: %s", query.getQueryType())); } return 0; } diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java index 1521e8c0f..bdef9ff83 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java @@ -13,18 +13,18 @@ public class DynamoDBResultSet extends ServiceResultSet { - private DynamoDBPreparedStatement stmt; + private final DynamoDBPreparedStatement stmt; public DynamoDBResultSet(DynamoDBPreparedStatement stmt) throws SQLException { this.stmt = stmt; } @Override - public boolean next() throws SQLException + public boolean next() { if(iterator.hasNext()) { - currentEntry = (HashMap) iterator.next(); + currentEntry = iterator.next(); return true; } return false; @@ -34,7 +34,7 @@ public boolean next() throws SQLException private boolean lastWasNull; @Override - public boolean wasNull() throws SQLException + public boolean wasNull() { return value == null || lastWasNull; } @@ -55,7 +55,7 @@ private long getNumeric(int columnIndex) if (sNumber != null) return Long.parseLong(sNumber); else if (value.bool() != null) - return value.bool().booleanValue() ? 1 : 0; + return value.bool() ? 1 : 0; } lastWasNull = true; return 0; @@ -87,13 +87,13 @@ private Instant getInstant(int columnIndex) throws SQLException } @Override - public T getAs(Class reference, int columnIndex, T defaultValue) throws SQLException + public T getAs(Class reference, int columnIndex, T defaultValue) { throw new UnsupportedOperationException(String.format("Data Type: %s", reference.getName())); } @Override - public String getString(int columnIndex) throws SQLException + public String getString(int columnIndex) { String value = DynamoDBHelper.getString(getAttValue(columnIndex)); lastWasNull = value == null; @@ -101,7 +101,7 @@ public String getString(int columnIndex) throws SQLException } @Override - public boolean getBoolean(int columnIndex) throws SQLException + public boolean getBoolean(int columnIndex) { AttributeValue value = getAttValue(columnIndex); if(value != null) @@ -110,7 +110,7 @@ public boolean getBoolean(int columnIndex) throws SQLException if(boolValue != null) { lastWasNull = false; - return boolValue.booleanValue(); + return boolValue; } } lastWasNull = true; @@ -118,50 +118,43 @@ public boolean getBoolean(int columnIndex) throws SQLException } @Override - public byte getByte(int columnIndex) throws SQLException + public byte getByte(int columnIndex) { return (byte)getNumeric(columnIndex); } @Override - public short getShort(int columnIndex) throws SQLException + public short getShort(int columnIndex) { return (short)getNumeric(columnIndex); } @Override - public int getInt(int columnIndex) throws SQLException + public int getInt(int columnIndex) { return (int)getNumeric(columnIndex); } @Override - public long getLong(int columnIndex) throws SQLException + public long getLong(int columnIndex) { - return (long)getNumeric(columnIndex); + return getNumeric(columnIndex); } @Override - public float getFloat(int columnIndex) throws SQLException + public float getFloat(int columnIndex) { return (float)getDecimal(columnIndex); } @Override - public double getDouble(int columnIndex) throws SQLException + public double getDouble(int columnIndex) { - return (double)getDecimal(columnIndex); + return getDecimal(columnIndex); } @Override - @Deprecated - public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException - { - return getBigDecimal(columnIndex).setScale(scale); - } - - @Override - public BigDecimal getBigDecimal(int columnIndex) throws SQLException + public BigDecimal getBigDecimal(int columnIndex) { AttributeValue value = getAttValue(columnIndex); if(value != null) diff --git a/java/src/main/java/com/genexus/db/service/ServiceResultSet.java b/java/src/main/java/com/genexus/db/service/ServiceResultSet.java index f1f74084f..144f97f7a 100644 --- a/java/src/main/java/com/genexus/db/service/ServiceResultSet.java +++ b/java/src/main/java/com/genexus/db/service/ServiceResultSet.java @@ -104,7 +104,7 @@ public double getDouble(int columnIndex) throws SQLException @Deprecated public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { - return getAs(BigDecimal.class, columnIndex, BigDecimal.ZERO); + return getBigDecimal(columnIndex).setScale(scale); } @Override From 5a75076735e3263e673a8a0a2141b514af9b4782 Mon Sep 17 00:00:00 2001 From: Gustavo Brown Date: Thu, 17 Mar 2022 13:00:24 -0300 Subject: [PATCH 08/24] Fix lint errors --- .../com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java | 6 +++--- .../java/com/genexus/db/dynamodb/DynamoDBConnection.java | 8 +++++--- .../main/java/com/genexus/db/dynamodb/DynamoDBDriver.java | 2 +- .../genexus/db/dynamodb/DynamoDBPreparedStatement.java | 3 --- .../java/com/genexus/db/dynamodb/DynamoDBResultSet.java | 7 +++---- .../main/java/com/genexus/db/dynamodb/DynamoQuery.java | 1 - 6 files changed, 12 insertions(+), 15 deletions(-) diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java index cbb468772..8fd827f03 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java @@ -44,15 +44,15 @@ public Object empty(GXType gxtype) case Clob: case VarChar: case Raw: - case Blob: return ""; - case Boolean: return false; - case Decimal: return 0f; + case Blob: case NText: case Text: case Image: case UniqueIdentifier: case Xml: case DateAsChar: return ""; + case Boolean: return false; + case Decimal: return 0f; case Geography: case Geopoint: diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java index 6373495f6..80ce0cdb4 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBConnection.java @@ -1,6 +1,7 @@ package com.genexus.db.dynamodb; import com.genexus.db.service.ServiceConnection; +import org.apache.commons.lang.StringUtils; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; @@ -16,6 +17,7 @@ public class DynamoDBConnection extends ServiceConnection { private static final String GXDYNAMODB_VERSION = "1.0"; + private static final String GXDYNAMODB_PRODUCT_NAME = "DynamoDB"; private static final String CLIENT_ID = "user"; private static final String CLIENT_SECRET = "password"; @@ -51,8 +53,8 @@ private void initializeDBConnection(String connUrl) DynamoDbClientBuilder builder = DynamoDbClient.builder().region(mRegion); if(mLocalUrl != null) builder = builder.endpointOverride(URI.create(mLocalUrl)); - if(mClientId != null && !mClientId.equals("") && - mClientSecret != null && !mClientSecret.equals("")) + if(StringUtils.isNotEmpty(mClientId) && + StringUtils.isNotEmpty(mClientSecret)) { AwsBasicCredentials mCredentials = AwsBasicCredentials.create(mClientId, mClientSecret); builder = builder.credentialsProvider(StaticCredentialsProvider.create(mCredentials)); @@ -79,7 +81,7 @@ public boolean isClosed() @Override public String getDatabaseProductName() { - return "DynamoDB"; + return GXDYNAMODB_PRODUCT_NAME; } @Override diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBDriver.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBDriver.java index 5fbcf34db..ef716751c 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBDriver.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBDriver.java @@ -28,7 +28,7 @@ public DynamoDBDriver() } @Override - public Connection connect(String url, Properties info) throws SQLException + public Connection connect(String url, Properties info) { if(!acceptsURL(url)) return null; diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java index c59067b30..331587cdc 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java @@ -65,10 +65,7 @@ private int _executeQuery(DynamoDBResultSet resultSet) throws SQLException if(!DynamoDBHelper.addAttributeValue(name, values, query.getParm(parmName))) throw new SQLException(String.format("Cannot assign attribute value (name: %s)", parmName)); } - String pattern = "\\((.*) = :(.*)\\)"; HashMap keyCondition = new HashMap<>(); - ArrayList filters = new ArrayList<>(); - HashMap expressionAttributeNames = null; for (Iterator it = Arrays.stream(query.selectList) diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java index bdef9ff83..741478392 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java @@ -9,7 +9,6 @@ import java.sql.Time; import java.sql.Timestamp; import java.time.Instant; -import java.util.HashMap; public class DynamoDBResultSet extends ServiceResultSet { @@ -75,7 +74,7 @@ private double getDecimal(int columnIndex) return 0; } - private Instant getInstant(int columnIndex) throws SQLException + private Instant getInstant(int columnIndex) { String value = getString(columnIndex); if(value == null) @@ -171,13 +170,13 @@ public BigDecimal getBigDecimal(int columnIndex) } @Override - public java.sql.Date getDate(int columnIndex) throws SQLException + public java.sql.Date getDate(int columnIndex) { return getAs(java.sql.Date.class, columnIndex, new java.sql.Date(0)); } @Override - public Time getTime(int columnIndex) throws SQLException + public Time getTime(int columnIndex) { return getAs(Time.class, columnIndex, new Time(0)); } diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java index 3c68674da..74a20eb80 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java @@ -7,7 +7,6 @@ public class DynamoQuery extends Query{ private boolean scanIndexForward = true; private static final String RANGE_KEY_INDEX = "RangeKey"; - private static final char[] indexTrimChars = new char[] { '(', ')' }; @Override public DynamoQuery orderBy(String index) From 467c4920e9c9924dbcf54802c47302b1a9ff60e6 Mon Sep 17 00:00:00 2001 From: Gustavo Brown Date: Fri, 18 Mar 2022 00:54:42 -0300 Subject: [PATCH 09/24] Allow datetime fields to only contain the date part --- gxdynamodb/pom.xml | 2 +- .../db/dynamodb/DynamoDBResultSet.java | 31 +++++++++++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/gxdynamodb/pom.xml b/gxdynamodb/pom.xml index 4c7f578d2..2c169f1a0 100644 --- a/gxdynamodb/pom.xml +++ b/gxdynamodb/pom.xml @@ -18,7 +18,7 @@ software.amazon.awssdk bom - 2.17.136 + 2.17.151 pom import diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java index 741478392..3f88123d9 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java @@ -1,5 +1,6 @@ package com.genexus.db.dynamodb; +import com.genexus.CommonUtil; import com.genexus.db.service.IOServiceContext; import com.genexus.db.service.ServiceResultSet; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; @@ -9,6 +10,12 @@ import java.sql.Time; import java.sql.Timestamp; import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.TemporalAccessor; public class DynamoDBResultSet extends ServiceResultSet { @@ -74,15 +81,33 @@ private double getDecimal(int columnIndex) return 0; } + private static final DateTimeFormatter ISO_DATE_TIME_OR_DATE = new DateTimeFormatterBuilder() + .parseCaseInsensitive() + .append(DateTimeFormatter.ISO_LOCAL_DATE) + .optionalStart() + .appendLiteral('T') + .append(DateTimeFormatter.ISO_LOCAL_TIME) + .optionalStart() + .appendOffsetId() + .optionalStart() + .appendLiteral('[') + .parseCaseSensitive() + .appendZoneRegionId() + .appendLiteral(']').toFormatter(); + private Instant getInstant(int columnIndex) { String value = getString(columnIndex); if(value == null) { lastWasNull = true; - return Instant.EPOCH; + return CommonUtil.nullDate().toInstant(); } - return Instant.parse(value); + + TemporalAccessor accessor = ISO_DATE_TIME_OR_DATE.parseBest(value, LocalDateTime::from, LocalDate::from); + if(accessor instanceof LocalDateTime) + return ((LocalDateTime) accessor).toInstant(ZoneOffset.UTC); + else return LocalDate.from(accessor).atStartOfDay().toInstant(ZoneOffset.UTC); } @Override @@ -182,7 +207,7 @@ public Time getTime(int columnIndex) } @Override - public Timestamp getTimestamp(int columnIndex) throws SQLException + public Timestamp getTimestamp(int columnIndex) { return java.sql.Timestamp.from(getInstant(columnIndex)); } From 0d68b9a1ffd908152a5ca9b40e82212273867500 Mon Sep 17 00:00:00 2001 From: Gustavo Brown Date: Fri, 18 Mar 2022 16:35:12 -0300 Subject: [PATCH 10/24] Add missing implementation for getDate/1 --- .../main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java index 3f88123d9..b9c8d9173 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java @@ -3,6 +3,7 @@ import com.genexus.CommonUtil; import com.genexus.db.service.IOServiceContext; import com.genexus.db.service.ServiceResultSet; +import org.apache.commons.lang.time.DateUtils; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import java.math.BigDecimal; @@ -197,7 +198,7 @@ public BigDecimal getBigDecimal(int columnIndex) @Override public java.sql.Date getDate(int columnIndex) { - return getAs(java.sql.Date.class, columnIndex, new java.sql.Date(0)); + return java.sql.Date.valueOf(getTimestamp(columnIndex).toLocalDateTime().toLocalDate()); } @Override From 63ea5e15eea23114e0094b0b1a532bbd2e8c396b Mon Sep 17 00:00:00 2001 From: Gustavo Brown Date: Wed, 23 Mar 2022 11:05:12 -0300 Subject: [PATCH 11/24] Add special case for queries that filter with an empty string key --- .../genexus/db/dynamodb/DynamoDBErrors.java | 7 ++++ .../db/dynamodb/DynamoDBResultSet.java | 24 +++++++++++--- .../genexus/db/service/ServiceException.java | 32 ++++++++++++++++--- 3 files changed, 53 insertions(+), 10 deletions(-) create mode 100644 gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBErrors.java diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBErrors.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBErrors.java new file mode 100644 index 000000000..0420a0286 --- /dev/null +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBErrors.java @@ -0,0 +1,7 @@ +package com.genexus.db.dynamodb; + +public class DynamoDBErrors +{ + public static final String ValidationException = "ValidationException"; + public static CharSequence ValidationExceptionMessageKey = "The AttributeValue for a key attribute cannot contain an empty string value."; +} diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java index b9c8d9173..ccc5d2779 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java @@ -2,9 +2,13 @@ import com.genexus.CommonUtil; import com.genexus.db.service.IOServiceContext; +import com.genexus.db.service.ServiceError; +import com.genexus.db.service.ServiceException; import com.genexus.db.service.ServiceResultSet; import org.apache.commons.lang.time.DateUtils; +import software.amazon.awssdk.awscore.exception.AwsErrorDetails; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.DynamoDbException; import java.math.BigDecimal; import java.sql.SQLException; @@ -27,14 +31,24 @@ public DynamoDBResultSet(DynamoDBPreparedStatement stmt) throws SQLException } @Override - public boolean next() + public boolean next() throws SQLException { - if(iterator.hasNext()) + try { - currentEntry = iterator.next(); - return true; + if(iterator.hasNext()) + { + currentEntry = iterator.next(); + return true; + } + return false; + }catch(DynamoDbException e) + { + AwsErrorDetails details = e.awsErrorDetails(); + if(details != null && details.errorCode().equals(DynamoDBErrors.ValidationException) && + details.errorMessage().contains(DynamoDBErrors.ValidationExceptionMessageKey)) + return false; // Handles special case where a string key attribute is filtered with an empty value which is not supported on DynamoDB but should yield a not record found in GX + throw e; } - return false; } private static final IOServiceContext SERVICE_CONTEXT = null; diff --git a/java/src/main/java/com/genexus/db/service/ServiceException.java b/java/src/main/java/com/genexus/db/service/ServiceException.java index 80332c882..24f294419 100644 --- a/java/src/main/java/com/genexus/db/service/ServiceException.java +++ b/java/src/main/java/com/genexus/db/service/ServiceException.java @@ -1,17 +1,29 @@ package com.genexus.db.service; +import java.sql.SQLException; + public class ServiceException extends Error { private String sqlState; private int vendorCode; public ServiceException(String reason, String sqlState, int vendorCode) { - super(reason); - this.sqlState = sqlState; - this.vendorCode = vendorCode; + this(reason, sqlState, vendorCode, null); } - - public String getSQLState() + + public ServiceException(String reason, String sqlState, int vendorCode, Throwable innerException) + { + super(reason, innerException); + this.sqlState = sqlState; + this.vendorCode = vendorCode; + } + + public static SQLException createSQLException(ServiceError serviceError, Throwable e) + { + return new SQLException(createServiceException(serviceError, e)); + } + + public String getSQLState() { return sqlState; } @@ -20,4 +32,14 @@ public int getVendorCode() { return vendorCode; } + + public static ServiceException createServiceException(ServiceError serviceError) + { + return new ServiceException(serviceError.toString(), serviceError.getSqlState(), serviceError.getCode()); + } + + public static ServiceException createServiceException(ServiceError serviceError, Throwable innerException) + { + return new ServiceException(serviceError.toString(), serviceError.getSqlState(), serviceError.getCode(), innerException); + } } From 8dc285b75c60c41e391196bf43868a1da76fe8c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Echag=C3=BCe?= Date: Wed, 6 Apr 2022 14:14:31 -0300 Subject: [PATCH 12/24] Update version to 2.6 --- gxdynamodb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gxdynamodb/pom.xml b/gxdynamodb/pom.xml index 2c169f1a0..e67e0efd6 100644 --- a/gxdynamodb/pom.xml +++ b/gxdynamodb/pom.xml @@ -7,7 +7,7 @@ com.genexus parent - 2.5-SNAPSHOT + 2.6-SNAPSHOT gxdynamodb From 4042b192936b7699edabb4804bbcc0878bf20868 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Echag=C3=BCe?= Date: Wed, 6 Apr 2022 14:15:02 -0300 Subject: [PATCH 13/24] Remove hardcoded dependency version --- gxdynamodb/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gxdynamodb/pom.xml b/gxdynamodb/pom.xml index e67e0efd6..004dc7331 100644 --- a/gxdynamodb/pom.xml +++ b/gxdynamodb/pom.xml @@ -37,7 +37,7 @@ com.genexus gxcommon - 2.5-SNAPSHOT + ${project.version} compile From eea3a3d46126b0e251a96b642afffaf83a16c2bf Mon Sep 17 00:00:00 2001 From: Gustavo Brown Date: Fri, 22 Apr 2022 11:22:44 -0300 Subject: [PATCH 14/24] Add support for binary streams --- .../com/genexus/db/dynamodb/DynamoDBResultSet.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java index ccc5d2779..651ddfe79 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java @@ -10,6 +10,8 @@ import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.DynamoDbException; +import java.io.ByteArrayInputStream; +import java.io.InputStream; import java.math.BigDecimal; import java.sql.SQLException; import java.sql.Time; @@ -227,6 +229,16 @@ public Timestamp getTimestamp(int columnIndex) return java.sql.Timestamp.from(getInstant(columnIndex)); } + @Override + public InputStream getBinaryStream(int columnIndex) throws SQLException + { + AttributeValue value = getAttValue(columnIndex); + if(value != null) + return value.b().asInputStream(); + lastWasNull = true; + return null; + } + // JDK8 @Override public T getObject(int columnIndex, Class type) From 1d3675b8c71e5568f089a2bb23633351764d1c3c Mon Sep 17 00:00:00 2001 From: Gustavo Brown Date: Fri, 22 Apr 2022 12:06:15 -0300 Subject: [PATCH 15/24] Fix NullPointerException when trying to get a binary stream from a record which does not have one --- .../java/com/genexus/db/dynamodb/DynamoDBResultSet.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java index 651ddfe79..d6274acc1 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java @@ -7,6 +7,7 @@ import com.genexus.db.service.ServiceResultSet; import org.apache.commons.lang.time.DateUtils; import software.amazon.awssdk.awscore.exception.AwsErrorDetails; +import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.DynamoDbException; @@ -234,7 +235,11 @@ public InputStream getBinaryStream(int columnIndex) throws SQLException { AttributeValue value = getAttValue(columnIndex); if(value != null) - return value.b().asInputStream(); + { + SdkBytes bytes = value.b(); + if(bytes != null) + return bytes.asInputStream(); + } lastWasNull = true; return null; } From b2053d8f00e319645df09cca8a29f1e538fa89ba Mon Sep 17 00:00:00 2001 From: Gustavo Brown Date: Tue, 26 Apr 2022 18:28:18 -0300 Subject: [PATCH 16/24] Add support for blobs, when duplicate/when none semantics. Fix datetimes --- .../genexus/db/dynamodb/DynamoDBHelper.java | 4 ++ .../dynamodb/DynamoDBPreparedStatement.java | 71 ++++++++++++++----- .../db/dynamodb/DynamoDBResultSet.java | 21 +++++- .../com/genexus/db/dynamodb/DynamoQuery.java | 13 ++-- 4 files changed, 86 insertions(+), 23 deletions(-) diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java index d7b9e7e70..f9e808090 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java @@ -3,6 +3,7 @@ import com.genexus.db.service.VarValue; import json.org.json.JSONArray; import json.org.json.JSONObject; +import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import java.sql.SQLException; @@ -52,6 +53,7 @@ public static AttributeValue toAttributeValue(VarValue var) throws SQLException case Clob: case Raw: case Blob: + return value != null ? builder.b((SdkBytes) value).build() : builder.nul(true).build(); case Undefined: case Image: case DateAsChar: @@ -75,6 +77,8 @@ else if (attValue.hasM()) return new JSONObject(convertToDictionary(attValue.m())).toString(); else if (attValue.hasL()) return new JSONArray(attValue.l().stream().map(DynamoDBHelper::getString).collect(Collectors.toList())).toString(); + else if(attValue.bool() != null) + return attValue.bool().toString(); return null; } diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java index 331587cdc..d28002d19 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java @@ -1,15 +1,17 @@ package com.genexus.db.dynamodb; +import com.genexus.db.Cursor; import com.genexus.db.ServiceCursorBase; import com.genexus.db.driver.GXConnection; -import com.genexus.db.service.IODataMap; -import com.genexus.db.service.QueryType; -import com.genexus.db.service.ServicePreparedStatement; -import com.genexus.db.service.VarValue; +import com.genexus.db.service.*; import com.genexus.util.NameValuePair; +import com.genexus.xml.ws.Service; +import jdk.internal.org.objectweb.asm.tree.TryCatchBlockNode; +import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.*; +import java.io.InputStream; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; @@ -27,7 +29,6 @@ public class DynamoDBPreparedStatement extends ServicePreparedStatement super(con, parms, gxCon); this.query = query; this.cursor = cursor; - query.initializeParms(parms); } @Override @@ -44,17 +45,21 @@ public int executeUpdate() throws SQLException return _executeQuery(null); } - private static final Pattern FILTER_PATTERN = Pattern.compile("\\((.*) = (:.*)\\)"); + private static final Pattern FILTER_PATTERN = Pattern.compile("\\((.*) = :(.*)\\)"); private int _executeQuery(DynamoDBResultSet resultSet) throws SQLException { + query.initializeParms(parms); DynamoDbClient client = getClient(); HashMap values = new HashMap<>(); - for(Iterator it = query.getVars(); it.hasNext();) + if(query.getQueryType() == QueryType.QUERY) { - VarValue var = it.next(); - values.put(var.name, DynamoDBHelper.toAttributeValue(var)); + for (Iterator it = query.getVars().values().iterator(); it.hasNext(); ) + { + VarValue var = it.next(); + values.put(var.name, DynamoDBHelper.toAttributeValue(var)); + } } for (Iterator it = query.getAssignAtts(); it.hasNext(); ) @@ -68,6 +73,13 @@ private int _executeQuery(DynamoDBResultSet resultSet) throws SQLException HashMap keyCondition = new HashMap<>(); HashMap expressionAttributeNames = null; + String keyItemForUpd = query.getPartitionKey(); + if(keyItemForUpd != null && keyItemForUpd.startsWith("#")) + { + expressionAttributeNames = new HashMap<>(); + expressionAttributeNames.put(keyItemForUpd, keyItemForUpd.substring(1)); + } + for (Iterator it = Arrays.stream(query.selectList) .filter(selItem -> ((DynamoDBMap) selItem).needsAttributeMap()) .map(IODataMap::getName).iterator(); it.hasNext(); ) @@ -86,7 +98,7 @@ private int _executeQuery(DynamoDBResultSet resultSet) throws SQLException Matcher match = FILTER_PATTERN.matcher(keyFilter); if (match.matches() && match.groupCount() > 1) { - String varName = match.group(2); + String varName = String.format(":%s", match.group(2)); String name = trimSharp(match.group(1)); if (!DynamoDBHelper.addAttributeValue(name, values, query.getParm(varName))) throw new SQLException(String.format("Cannot assign attribute value (name: %s)", varName)); @@ -143,12 +155,21 @@ private int _executeQuery(DynamoDBResultSet resultSet) throws SQLException } case INS: { - PutItemRequest request = PutItemRequest.builder() + PutItemRequest.Builder builder = PutItemRequest.builder() .tableName(query.tableName) .item(values) - .build(); - client.putItem(request); - break; + .conditionExpression(String.format("attribute_not_exists(%s)", keyItemForUpd)); + if(expressionAttributeNames != null) + builder.expressionAttributeNames(expressionAttributeNames); + PutItemRequest request = builder.build(); + try + { + client.putItem(request); + }catch(ConditionalCheckFailedException recordAlreadyExists) + { + return Cursor.DUPLICATE; + } + break; } case UPD: { @@ -162,11 +183,21 @@ private int _executeQuery(DynamoDBResultSet resultSet) throws SQLException } case DLT: { - DeleteItemRequest request = DeleteItemRequest.builder() + DeleteItemRequest.Builder builder = DeleteItemRequest.builder() .tableName(query.tableName) .key(keyCondition) - .build(); - client.deleteItem(request); + .conditionExpression(String.format("attribute_exists(%s)", keyItemForUpd)); + if(expressionAttributeNames != null) + builder.expressionAttributeNames(expressionAttributeNames); + + DeleteItemRequest request = builder.build(); + try + { + client.deleteItem(request); + }catch(ConditionalCheckFailedException recordNotFound) + { + return Cursor.EOF; + } break; } default: throw new UnsupportedOperationException(String.format("Invalid query type: %s", query.getQueryType())); @@ -200,6 +231,12 @@ DynamoDbClient getClient() throws SQLException return ((DynamoDBConnection)getConnection()).mDynamoDB; } + @Override + public void setBinaryStream(int parameterIndex, InputStream x, int length) throws SQLException + { + parms[parameterIndex-1] = SdkBytes.fromInputStream(x); + } + /// JDK8 @Override public void closeOnCompletion() diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java index d6274acc1..229f4abee 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java @@ -23,6 +23,7 @@ import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; +import java.time.format.DateTimeParseException; import java.time.temporal.TemporalAccessor; public class DynamoDBResultSet extends ServiceResultSet @@ -113,6 +114,8 @@ private double getDecimal(int columnIndex) .appendZoneRegionId() .appendLiteral(']').toFormatter(); + private static DateTimeFormatter US_DATE_TIME_OR_DATE = DateTimeFormatter.ofPattern("M/d/yyyy[ HH:mm:ss]"); + private Instant getInstant(int columnIndex) { String value = getString(columnIndex); @@ -121,8 +124,22 @@ private Instant getInstant(int columnIndex) lastWasNull = true; return CommonUtil.nullDate().toInstant(); } + TemporalAccessor accessor; + + try + { + accessor = ISO_DATE_TIME_OR_DATE.parseBest(value, LocalDateTime::from, LocalDate::from); + }catch(DateTimeParseException dtpe) + { + try + { + accessor = US_DATE_TIME_OR_DATE.parseBest(value, LocalDateTime::from, LocalDate::from); + }catch(Exception e) + { + throw dtpe; + } + } - TemporalAccessor accessor = ISO_DATE_TIME_OR_DATE.parseBest(value, LocalDateTime::from, LocalDate::from); if(accessor instanceof LocalDateTime) return ((LocalDateTime) accessor).toInstant(ZoneOffset.UTC); else return LocalDate.from(accessor).atStartOfDay().toInstant(ZoneOffset.UTC); @@ -215,7 +232,7 @@ public BigDecimal getBigDecimal(int columnIndex) @Override public java.sql.Date getDate(int columnIndex) { - return java.sql.Date.valueOf(getTimestamp(columnIndex).toLocalDateTime().toLocalDate()); + return java.sql.Date.valueOf(getTimestamp(columnIndex).toInstant().atOffset(ZoneOffset.UTC).toLocalDate()); } @Override diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java index 74a20eb80..e6652aa89 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java @@ -7,6 +7,7 @@ public class DynamoQuery extends Query{ private boolean scanIndexForward = true; private static final String RANGE_KEY_INDEX = "RangeKey"; + private String partitionKey; @Override public DynamoQuery orderBy(String index) @@ -22,6 +23,14 @@ public DynamoQuery orderBy(String index) return this; } + public DynamoQuery setKey(String partitionKey) + { + this.partitionKey = partitionKey; + return this; + } + + public String getPartitionKey(){ return partitionKey; } + public DynamoQuery(DataStoreHelperDynamoDB dataStoreHelper) { super(dataStoreHelper); @@ -38,8 +47,4 @@ public void setIndex(String index) { public boolean isScanIndexForward() { return scanIndexForward; } - - public void setScanIndexForward(boolean scanIndexForward) { - this.scanIndexForward = scanIndexForward; - } } From 5cedb4686ecb051935b51d4261622a525adf91f7 Mon Sep 17 00:00:00 2001 From: Gustavo Brown Date: Tue, 26 Apr 2022 18:30:28 -0300 Subject: [PATCH 17/24] add 'as' method to cast a query to its derived type --- java/src/main/java/com/genexus/db/ServiceCursorBase.java | 6 ------ java/src/main/java/com/genexus/db/service/Query.java | 8 ++++++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/java/src/main/java/com/genexus/db/ServiceCursorBase.java b/java/src/main/java/com/genexus/db/ServiceCursorBase.java index 29e4461ff..b3996c838 100644 --- a/java/src/main/java/com/genexus/db/ServiceCursorBase.java +++ b/java/src/main/java/com/genexus/db/ServiceCursorBase.java @@ -195,12 +195,6 @@ void postExecute(AbstractDataStoreProviderBase connectionProvider, AbstractDataS default: throw new RuntimeException("Not implemented"); } - - //if(currentOf) - //{ // Si tengo currentof marco uncommited changes para hacer un rollback dado que en algunos casos - // de error (Bantotal) les estaban quedando locks en la bd - // connectionProvider.getConnection().setUncommitedChanges(); - //} } public boolean next(AbstractDataSource ds) throws SQLException diff --git a/java/src/main/java/com/genexus/db/service/Query.java b/java/src/main/java/com/genexus/db/service/Query.java index b80696509..c814d898c 100644 --- a/java/src/main/java/com/genexus/db/service/Query.java +++ b/java/src/main/java/com/genexus/db/service/Query.java @@ -8,7 +8,7 @@ public class Query implements IQuery { private static final String [] EMPTY_ARR_STRING = new String [0]; - ServiceDataStoreHelper dataStoreHelper; + final ServiceDataStoreHelper dataStoreHelper; public Query(ServiceDataStoreHelper dataStoreHelper) { this.dataStoreHelper = dataStoreHelper; @@ -24,7 +24,7 @@ public Query(ServiceDataStoreHelper dataStoreHelper) public IODataMap[] selectList = new IODataMap[0]; private final HashMap mVarValues = new HashMap<>(); - public Iterator getVars() { return mVarValues.values().iterator(); } + public HashMap getVars() { return mVarValues; } public VarValue getParm(String parmName) { @@ -99,6 +99,10 @@ public Query setParmType(int parmId, GXType gxType) parmTypes.put(parmId, gxType); return this; } + public T as(Class reference) + { + return reference.cast(this); + } public void initializeParms(Object[] parms) { From 966a2d3a17838ea1dced64d8606feb5dc4d6aa99 Mon Sep 17 00:00:00 2001 From: Gustavo Brown Date: Tue, 26 Apr 2022 18:47:52 -0300 Subject: [PATCH 18/24] fix imports --- .../db/dynamodb/DynamoDBPreparedStatement.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java index d28002d19..fe2ca821b 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java @@ -3,10 +3,11 @@ import com.genexus.db.Cursor; import com.genexus.db.ServiceCursorBase; import com.genexus.db.driver.GXConnection; -import com.genexus.db.service.*; +import com.genexus.db.service.IODataMap; +import com.genexus.db.service.QueryType; +import com.genexus.db.service.ServicePreparedStatement; +import com.genexus.db.service.VarValue; import com.genexus.util.NameValuePair; -import com.genexus.xml.ws.Service; -import jdk.internal.org.objectweb.asm.tree.TryCatchBlockNode; import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.*; @@ -15,7 +16,10 @@ import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.*; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; From a3a0136f9738b62cdea7f8b1afd14f18aab75c18 Mon Sep 17 00:00:00 2001 From: Gustavo Brown Date: Mon, 9 May 2022 18:00:50 -0300 Subject: [PATCH 19/24] Improve Query/Scan inference. Fix lint errors --- .../genexus/db/dynamodb/DynamoDBErrors.java | 2 +- .../dynamodb/DynamoDBPreparedStatement.java | 47 +++++++++++++------ .../db/dynamodb/DynamoDBResultSet.java | 6 +-- .../com/genexus/db/dynamodb/DynamoQuery.java | 22 +++++++++ .../java/com/genexus/db/service/Query.java | 4 +- 5 files changed, 59 insertions(+), 22 deletions(-) diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBErrors.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBErrors.java index 0420a0286..619b17205 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBErrors.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBErrors.java @@ -3,5 +3,5 @@ public class DynamoDBErrors { public static final String ValidationException = "ValidationException"; - public static CharSequence ValidationExceptionMessageKey = "The AttributeValue for a key attribute cannot contain an empty string value."; + public static final CharSequence ValidationExceptionMessageKey = "The AttributeValue for a key attribute cannot contain an empty string value."; } diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java index fe2ca821b..35baf3040 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBPreparedStatement.java @@ -16,12 +16,10 @@ import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; public class DynamoDBPreparedStatement extends ServicePreparedStatement { @@ -50,6 +48,7 @@ public int executeUpdate() throws SQLException } private static final Pattern FILTER_PATTERN = Pattern.compile("\\((.*) = :(.*)\\)"); + private static final Pattern VAR_PATTERN = Pattern.compile(".*(:.*)\\).*"); private int _executeQuery(DynamoDBResultSet resultSet) throws SQLException { query.initializeParms(parms); @@ -59,9 +58,8 @@ private int _executeQuery(DynamoDBResultSet resultSet) throws SQLException if(query.getQueryType() == QueryType.QUERY) { - for (Iterator it = query.getVars().values().iterator(); it.hasNext(); ) + for (VarValue var : query.getVars().values()) { - VarValue var = it.next(); values.put(var.name, DynamoDBHelper.toAttributeValue(var)); } } @@ -97,7 +95,7 @@ private int _executeQuery(DynamoDBResultSet resultSet) throws SQLException if(query.getQueryType() != QueryType.QUERY) { - for (String keyFilter : query.filters) + for (String keyFilter : query.getAllFilters().collect(Collectors.toList())) { Matcher match = FILTER_PATTERN.matcher(keyFilter); if (match.matches() && match.groupCount() > 1) @@ -115,17 +113,34 @@ private int _executeQuery(DynamoDBResultSet resultSet) throws SQLException { case QUERY: { + boolean issueScan = query instanceof DynamoScan; + if (!issueScan) + { // Check whether a query has to be demoted to scan due to empty parameters + for (String keyFilter : query.keyFilters) + { + Matcher match = VAR_PATTERN.matcher(keyFilter); + if (match.matches()) + { + String varName = match.group(1); + VarValue varValue = query.getParm(varName); + if (varValue != null && varValue.value.toString().isEmpty()) + { + issueScan = true; + break; + } + } + } + } + Iterator> iterator; - if(query instanceof DynamoScan) + if(issueScan) { ScanRequest.Builder builder = ScanRequest.builder() .tableName(query.tableName) .projectionExpression(String.join(",", query.projection)); - if(query.filters.length > 0) - { - builder.filterExpression(String.join(" AND ", query.filters)) - .expressionAttributeValues(values); - } + String filterString = query.getAllFilters().collect(Collectors.joining(" AND ")); + if(!filterString.isEmpty()) + builder.filterExpression(filterString).expressionAttributeValues(values); if(expressionAttributeNames != null) builder.expressionAttributeNames(expressionAttributeNames); @@ -139,11 +154,13 @@ private int _executeQuery(DynamoDBResultSet resultSet) throws SQLException { QueryRequest.Builder builder = QueryRequest.builder() .tableName(query.tableName) - .keyConditionExpression(String.join(" AND ", query.filters)) + .keyConditionExpression(String.join(" AND ", query.keyFilters)) .expressionAttributeValues(values) .projectionExpression(String.join(", ", query.projection)) .indexName(query.getIndex()) .scanIndexForward(query.isScanIndexForward()); + if(query.filters.length > 0) + builder.filterExpression(String.join(" AND ", query.filters)); if(expressionAttributeNames != null) builder.expressionAttributeNames(expressionAttributeNames); @@ -236,7 +253,7 @@ DynamoDbClient getClient() throws SQLException } @Override - public void setBinaryStream(int parameterIndex, InputStream x, int length) throws SQLException + public void setBinaryStream(int parameterIndex, InputStream x, int length) { parms[parameterIndex-1] = SdkBytes.fromInputStream(x); } diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java index 229f4abee..74f16e15a 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java @@ -2,16 +2,12 @@ import com.genexus.CommonUtil; import com.genexus.db.service.IOServiceContext; -import com.genexus.db.service.ServiceError; -import com.genexus.db.service.ServiceException; import com.genexus.db.service.ServiceResultSet; -import org.apache.commons.lang.time.DateUtils; import software.amazon.awssdk.awscore.exception.AwsErrorDetails; import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.DynamoDbException; -import java.io.ByteArrayInputStream; import java.io.InputStream; import java.math.BigDecimal; import java.sql.SQLException; @@ -114,7 +110,7 @@ private double getDecimal(int columnIndex) .appendZoneRegionId() .appendLiteral(']').toFormatter(); - private static DateTimeFormatter US_DATE_TIME_OR_DATE = DateTimeFormatter.ofPattern("M/d/yyyy[ HH:mm:ss]"); + private static final DateTimeFormatter US_DATE_TIME_OR_DATE = DateTimeFormatter.ofPattern("M/d/yyyy[ HH:mm:ss]"); private Instant getInstant(int columnIndex) { diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java index e6652aa89..314dbca0d 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoQuery.java @@ -2,12 +2,23 @@ import com.genexus.db.service.Query; +import java.util.Arrays; +import java.util.stream.Stream; + public class DynamoQuery extends Query{ private String index; private boolean scanIndexForward = true; private static final String RANGE_KEY_INDEX = "RangeKey"; private String partitionKey; + public String[] keyFilters = EMPTY_ARR_STRING; + + @Override + public DynamoQuery For(String tableName) + { + super.For(tableName); + return this; + } @Override public DynamoQuery orderBy(String index) @@ -29,6 +40,12 @@ public DynamoQuery setKey(String partitionKey) return this; } + public DynamoQuery keyFilter(String[] keyFilters) + { + this.keyFilters = keyFilters; + return this; + } + public String getPartitionKey(){ return partitionKey; } public DynamoQuery(DataStoreHelperDynamoDB dataStoreHelper) @@ -47,4 +64,9 @@ public void setIndex(String index) { public boolean isScanIndexForward() { return scanIndexForward; } + + public Stream getAllFilters() + { + return Stream.concat(Arrays.stream(keyFilters), Arrays.stream(filters)); + } } diff --git a/java/src/main/java/com/genexus/db/service/Query.java b/java/src/main/java/com/genexus/db/service/Query.java index c814d898c..9f794f6a3 100644 --- a/java/src/main/java/com/genexus/db/service/Query.java +++ b/java/src/main/java/com/genexus/db/service/Query.java @@ -3,11 +3,13 @@ import com.genexus.util.NameValuePair; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; +import java.util.stream.Stream; public class Query implements IQuery { - private static final String [] EMPTY_ARR_STRING = new String [0]; + protected static final String [] EMPTY_ARR_STRING = new String [0]; final ServiceDataStoreHelper dataStoreHelper; public Query(ServiceDataStoreHelper dataStoreHelper) { From d187d494a272e2b23ee9bcbce0ae94d4f2ffc3ab Mon Sep 17 00:00:00 2001 From: Gustavo Brown Date: Thu, 30 Jun 2022 14:57:49 -0300 Subject: [PATCH 20/24] Fix datetime fields. Fix update queries --- gxdynamodb/pom.xml | 2 +- .../genexus/db/dynamodb/DynamoDBHelper.java | 19 +++-- .../dynamodb/DynamoDBPreparedStatement.java | 74 ++++++++++++++----- .../db/dynamodb/DynamoDBResultSet.java | 44 ++++++++--- 4 files changed, 103 insertions(+), 36 deletions(-) diff --git a/gxdynamodb/pom.xml b/gxdynamodb/pom.xml index 004dc7331..46dd8dfe6 100644 --- a/gxdynamodb/pom.xml +++ b/gxdynamodb/pom.xml @@ -7,7 +7,7 @@ com.genexus parent - 2.6-SNAPSHOT + 2.7-SNAPSHOT gxdynamodb diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java index f9e808090..92bb315aa 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java @@ -7,6 +7,8 @@ import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.ZoneOffset; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -16,7 +18,11 @@ public class DynamoDBHelper { public static AttributeValue toAttributeValue(VarValue var) throws SQLException { + if(var == null) + return null; Object value = var.value; + if(value == null) + return null; AttributeValue.Builder builder = AttributeValue.builder(); switch (var.type) { @@ -27,9 +33,10 @@ public static AttributeValue toAttributeValue(VarValue var) throws SQLException case Decimal: return builder.n(value.toString()).build(); case Date: + return builder.s(((java.sql.Date) value).toLocalDate().toString()).build(); case DateTime: case DateTime2: - return builder.s(value.toString()).build(); + return builder.s(((Timestamp) value).toLocalDateTime().atOffset(ZoneOffset.UTC).toString()).build(); case Boolean: case Byte: return builder.bool((Boolean) value).build(); @@ -53,7 +60,7 @@ public static AttributeValue toAttributeValue(VarValue var) throws SQLException case Clob: case Raw: case Blob: - return value != null ? builder.b((SdkBytes) value).build() : builder.nul(true).build(); + return builder.b((SdkBytes) value).build(); case Undefined: case Image: case DateAsChar: @@ -73,12 +80,12 @@ else if (!attValue.ns().isEmpty()) return setToString(attValue.ns()); else if (!attValue.ss().isEmpty()) return setToString(attValue.ss()); + else if(attValue.bool() != null) + return attValue.bool().toString(); else if (attValue.hasM()) return new JSONObject(convertToDictionary(attValue.m())).toString(); else if (attValue.hasL()) return new JSONArray(attValue.l().stream().map(DynamoDBHelper::getString).collect(Collectors.toList())).toString(); - else if(attValue.bool() != null) - return attValue.bool().toString(); return null; } @@ -105,9 +112,7 @@ public static boolean addAttributeValue(String parmName, HashMap values = new HashMap<>(); @@ -64,22 +65,44 @@ private int _executeQuery(DynamoDBResultSet resultSet) throws SQLException } } + HashMap keyCondition = new HashMap<>(); + HashMap expressionAttributeNames = null; + HashSet mappedNames = null; + for (Iterator it = query.getAssignAtts(); it.hasNext(); ) { NameValuePair asg = it.next(); - String name = trimSharp(asg.name); + String name = asg.name; + if(asg.name.startsWith("#")) + { + name = trimSharp(asg.name); + if (!isInsert) + { + if(expressionAttributeNames == null) + { + expressionAttributeNames = new HashMap<>(); + mappedNames = new HashSet<>(); + } + expressionAttributeNames.put(asg.name, name); + mappedNames.add(name); + } + } String parmName = asg.value; - if(!DynamoDBHelper.addAttributeValue(name, values, query.getParm(parmName))) + if(!DynamoDBHelper.addAttributeValue(isInsert ? name : ":" + name, values, query.getParm(parmName))) throw new SQLException(String.format("Cannot assign attribute value (name: %s)", parmName)); } - HashMap keyCondition = new HashMap<>(); - HashMap expressionAttributeNames = null; String keyItemForUpd = query.getPartitionKey(); if(keyItemForUpd != null && keyItemForUpd.startsWith("#")) { - expressionAttributeNames = new HashMap<>(); - expressionAttributeNames.put(keyItemForUpd, keyItemForUpd.substring(1)); + if(expressionAttributeNames == null) + { + expressionAttributeNames = new HashMap<>(); + mappedNames = new HashSet<>(); + } + String keyName = keyItemForUpd.substring(1); + expressionAttributeNames.put(keyItemForUpd, keyName); + mappedNames.add(keyName); } for (Iterator it = Arrays.stream(query.selectList) @@ -87,10 +110,14 @@ private int _executeQuery(DynamoDBResultSet resultSet) throws SQLException .map(IODataMap::getName).iterator(); it.hasNext(); ) { if(expressionAttributeNames == null) + { expressionAttributeNames = new HashMap<>(); + mappedNames = new HashSet<>(); + } String mappedName = it.next(); String key = "#" + mappedName; expressionAttributeNames.put(key, mappedName); + mappedNames.add(mappedName); } if(query.getQueryType() != QueryType.QUERY) @@ -102,9 +129,10 @@ private int _executeQuery(DynamoDBResultSet resultSet) throws SQLException { String varName = String.format(":%s", match.group(2)); String name = trimSharp(match.group(1)); - if (!DynamoDBHelper.addAttributeValue(name, values, query.getParm(varName))) + AttributeValue value = DynamoDBHelper.toAttributeValue(query.getParm(varName)); + if(value == null) throw new SQLException(String.format("Cannot assign attribute value (name: %s)", varName)); - keyCondition.put(name, values.get(name)); + keyCondition.put(name, value); } } } @@ -197,9 +225,18 @@ private int _executeQuery(DynamoDBResultSet resultSet) throws SQLException UpdateItemRequest request = UpdateItemRequest.builder() .tableName(query.tableName) .key(keyCondition) - .attributeUpdates(toAttributeUpdates(keyCondition, values)) + .updateExpression(toAttributeUpdates(keyCondition, values, mappedNames)) + .conditionExpression(String.format("attribute_exists(%s)", keyItemForUpd)) + .expressionAttributeNames(expressionAttributeNames) + .expressionAttributeValues(values) .build(); - client.updateItem(request); + try + { + client.updateItem(request); + }catch(ConditionalCheckFailedException recordNotFound) + { + return Cursor.EOF; + } break; } case DLT: @@ -226,20 +263,21 @@ private int _executeQuery(DynamoDBResultSet resultSet) throws SQLException return 0; } - private HashMap toAttributeUpdates(HashMap keyConditions, HashMap values) + private String toAttributeUpdates(HashMap keyConditions, HashMap values, HashSet mappedNames) { - HashMap updates = new HashMap<>(); + StringBuilder updateExpression = new StringBuilder(); for(Map.Entry item : values.entrySet()) { - if (!keyConditions.containsKey(item.getKey()) && !item.getKey().startsWith("AV")) + String keyName = item.getKey().substring(1); + if (!keyConditions.containsKey(keyName) && !keyName.startsWith("AV")) { - updates.put(item.getKey(), AttributeValueUpdate.builder() - .value(item.getValue()) - .action(AttributeAction.PUT) - .build()); + if (mappedNames != null && mappedNames.contains(keyName)) + keyName = "#" + keyName; + updateExpression.append(updateExpression.length() == 0 ? "SET " : ", "); + updateExpression.append(keyName).append(" = ").append(item.getKey()); } } - return updates; + return updateExpression.toString(); } private static String trimSharp(String name) diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java index 74f16e15a..64739890e 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBResultSet.java @@ -1,6 +1,7 @@ package com.genexus.db.dynamodb; import com.genexus.CommonUtil; +import com.genexus.ModelContext; import com.genexus.db.service.IOServiceContext; import com.genexus.db.service.ServiceResultSet; import software.amazon.awssdk.awscore.exception.AwsErrorDetails; @@ -21,6 +22,7 @@ import java.time.format.DateTimeFormatterBuilder; import java.time.format.DateTimeParseException; import java.time.temporal.TemporalAccessor; +import java.util.TimeZone; public class DynamoDBResultSet extends ServiceResultSet { @@ -31,7 +33,7 @@ public DynamoDBResultSet(DynamoDBPreparedStatement stmt) throws SQLException } @Override - public boolean next() throws SQLException + public boolean next() { try { @@ -100,7 +102,12 @@ private double getDecimal(int columnIndex) .parseCaseInsensitive() .append(DateTimeFormatter.ISO_LOCAL_DATE) .optionalStart() + .optionalStart() .appendLiteral('T') + .optionalEnd() + .optionalStart() + .appendLiteral(' ') + .optionalEnd() .append(DateTimeFormatter.ISO_LOCAL_TIME) .optionalStart() .appendOffsetId() @@ -110,34 +117,51 @@ private double getDecimal(int columnIndex) .appendZoneRegionId() .appendLiteral(']').toFormatter(); - private static final DateTimeFormatter US_DATE_TIME_OR_DATE = DateTimeFormatter.ofPattern("M/d/yyyy[ HH:mm:ss]"); + private static final DateTimeFormatter [] DATE_TIME_FORMATTERS = new DateTimeFormatter[] + { + DateTimeFormatter.ofPattern("M/d/yyyy[ H:mm:ss]"), + DateTimeFormatter.ofPattern("M/d/yyyy[ h:mm:ss a]"), + DateTimeFormatter.ofPattern("yyyy-M-d[ H:mm:ss.S]"), + DateTimeFormatter.ofPattern("yyyy-M-d[ H:mm:ss.S a]") + }; private Instant getInstant(int columnIndex) { String value = getString(columnIndex); - if(value == null) + if(value == null || value.trim().isEmpty()) { lastWasNull = true; return CommonUtil.nullDate().toInstant(); } - TemporalAccessor accessor; + + TemporalAccessor accessor = null; try { accessor = ISO_DATE_TIME_OR_DATE.parseBest(value, LocalDateTime::from, LocalDate::from); }catch(DateTimeParseException dtpe) { - try + for(DateTimeFormatter dateTimeFormatter:DATE_TIME_FORMATTERS) { - accessor = US_DATE_TIME_OR_DATE.parseBest(value, LocalDateTime::from, LocalDate::from); - }catch(Exception e) + try + { + accessor = dateTimeFormatter.parseBest(value, LocalDateTime::from, LocalDate::from); + break; + }catch(Exception ignored){ } + } + if(accessor == null) { - throw dtpe; + return CommonUtil.resetTime(CommonUtil.nullDate()).toInstant(); } } + if(accessor instanceof LocalDateTime) - return ((LocalDateTime) accessor).toInstant(ZoneOffset.UTC); + { + ModelContext ctx = ModelContext.getModelContext(); + TimeZone tz = ctx != null ? ctx.getClientTimeZone() : TimeZone.getDefault(); + return ((LocalDateTime) accessor).atZone(tz.toZoneId()).toInstant(); + } else return LocalDate.from(accessor).atStartOfDay().toInstant(ZoneOffset.UTC); } @@ -244,7 +268,7 @@ public Timestamp getTimestamp(int columnIndex) } @Override - public InputStream getBinaryStream(int columnIndex) throws SQLException + public InputStream getBinaryStream(int columnIndex) { AttributeValue value = getAttValue(columnIndex); if(value != null) From b2a7ff43c71683678b1a17809f31651a1f092c42 Mon Sep 17 00:00:00 2001 From: Gustavo Brown Date: Thu, 7 Jul 2022 20:13:13 -0300 Subject: [PATCH 21/24] Fix possible input types for Date and DateTime in toAttributeValue --- .../com/genexus/db/dynamodb/DynamoDBHelper.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java index 92bb315aa..a92e86782 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java @@ -8,6 +8,8 @@ import java.sql.SQLException; import java.sql.Timestamp; +import java.text.SimpleDateFormat; +import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.util.HashMap; import java.util.List; @@ -16,6 +18,7 @@ public class DynamoDBHelper { + private static final SimpleDateFormat ISO_DATE_FORMATTER = new SimpleDateFormat("yyyy-MM-dd"); public static AttributeValue toAttributeValue(VarValue var) throws SQLException { if(var == null) @@ -33,10 +36,16 @@ public static AttributeValue toAttributeValue(VarValue var) throws SQLException case Decimal: return builder.n(value.toString()).build(); case Date: - return builder.s(((java.sql.Date) value).toLocalDate().toString()).build(); + if(value instanceof java.util.Date) + return builder.s(ISO_DATE_FORMATTER.format(value)).build(); + else return builder.s(((java.sql.Date) value).toLocalDate().toString()).build(); case DateTime: case DateTime2: - return builder.s(((Timestamp) value).toLocalDateTime().atOffset(ZoneOffset.UTC).toString()).build(); + OffsetDateTime offsetDateTime; + if(value instanceof java.util.Date) + offsetDateTime = ((java.util.Date)value).toInstant().atOffset(ZoneOffset.UTC); + else offsetDateTime = ((Timestamp) value).toLocalDateTime().atOffset(ZoneOffset.UTC); + return builder.s(offsetDateTime.toString()).build(); case Boolean: case Byte: return builder.bool((Boolean) value).build(); From 1718a924bc620d751743a419f6ad48dec47018c5 Mon Sep 17 00:00:00 2001 From: Gustavo Brown Date: Mon, 11 Jul 2022 14:50:40 -0300 Subject: [PATCH 22/24] Change DateTime->String conversion. It was shifting offsets twice to reach UTC --- .../java/com/genexus/db/dynamodb/DynamoDBHelper.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java index a92e86782..4b88010f1 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java @@ -41,11 +41,11 @@ public static AttributeValue toAttributeValue(VarValue var) throws SQLException else return builder.s(((java.sql.Date) value).toLocalDate().toString()).build(); case DateTime: case DateTime2: - OffsetDateTime offsetDateTime; - if(value instanceof java.util.Date) - offsetDateTime = ((java.util.Date)value).toInstant().atOffset(ZoneOffset.UTC); - else offsetDateTime = ((Timestamp) value).toLocalDateTime().atOffset(ZoneOffset.UTC); - return builder.s(offsetDateTime.toString()).build(); + Timestamp valueTs; + if(value instanceof java.sql.Timestamp) + valueTs = (java.sql.Timestamp)value; + else valueTs = new java.sql.Timestamp(((java.util.Date) value).getTime()); + return builder.s(valueTs.toLocalDateTime().atOffset(ZoneOffset.UTC).toString()).build(); case Boolean: case Byte: return builder.bool((Boolean) value).build(); From b7a33403b0a3cbe2ff650a8e89628c0ee2b9a565 Mon Sep 17 00:00:00 2001 From: Gustavo Brown Date: Mon, 11 Jul 2022 16:49:13 -0300 Subject: [PATCH 23/24] Simplify empty datetime parameters so there is no need manage java.util.Date and java.sql.Timestamp separaterly --- .../com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java | 7 +++++-- .../java/com/genexus/db/dynamodb/DynamoDBHelper.java | 9 ++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java index 8fd827f03..c40e88f3c 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DataStoreHelperDynamoDB.java @@ -8,6 +8,9 @@ import com.genexus.db.service.IQuery; import com.genexus.db.service.ServiceDataStoreHelper; +import java.sql.Date; +import java.sql.Timestamp; + public class DataStoreHelperDynamoDB extends ServiceDataStoreHelper { public DynamoQuery newQuery() @@ -32,9 +35,9 @@ public Object empty(GXType gxtype) case Int16: case Int32: case Int64: return 0; - case Date: + case Date: return new Date(CommonUtil.nullDate().getTime()); case DateTime: - case DateTime2: return CommonUtil.nullDate(); + case DateTime2: return new Timestamp(CommonUtil.nullDate().getTime()); case Byte: case NChar: case NClob: diff --git a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java index 4b88010f1..ca34af579 100644 --- a/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java +++ b/gxdynamodb/src/main/java/com/genexus/db/dynamodb/DynamoDBHelper.java @@ -36,15 +36,10 @@ public static AttributeValue toAttributeValue(VarValue var) throws SQLException case Decimal: return builder.n(value.toString()).build(); case Date: - if(value instanceof java.util.Date) - return builder.s(ISO_DATE_FORMATTER.format(value)).build(); - else return builder.s(((java.sql.Date) value).toLocalDate().toString()).build(); + return builder.s(ISO_DATE_FORMATTER.format(value)).build(); case DateTime: case DateTime2: - Timestamp valueTs; - if(value instanceof java.sql.Timestamp) - valueTs = (java.sql.Timestamp)value; - else valueTs = new java.sql.Timestamp(((java.util.Date) value).getTime()); + Timestamp valueTs = (java.sql.Timestamp)value; return builder.s(valueTs.toLocalDateTime().atOffset(ZoneOffset.UTC).toString()).build(); case Boolean: case Byte: From d5be2e86d9599548bd3a64e68261934b467cc981 Mon Sep 17 00:00:00 2001 From: Gustavo Brown Date: Thu, 1 Sep 2022 10:54:10 -0300 Subject: [PATCH 24/24] Update build workflow - #596 --- gxdynamodb/pom.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gxdynamodb/pom.xml b/gxdynamodb/pom.xml index 46dd8dfe6..23687c610 100644 --- a/gxdynamodb/pom.xml +++ b/gxdynamodb/pom.xml @@ -4,11 +4,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - - com.genexus - parent - 2.7-SNAPSHOT - + + com.genexus + parent + ${revision}${changelist} + gxdynamodb GeneXus DynamoDB