From c50641efaa14f0f6e382a8bdf50dc15db1cd417a Mon Sep 17 00:00:00 2001 From: Deepak Dixit Date: Sun, 3 Dec 2023 22:03:33 +0530 Subject: [PATCH] Master (#11) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix regression with partitioned tables in PostgreSQL PostgreSQL JDBC Driver introduced separating type for partitioned table from 40.2.12 pgjdbc/pgjdbc#1708 * Add subscreensItem.menuInclude to menu data (#600) * Fixed the problem that moqui cannot be deployed as non-root webapp in Tomcat * Fixed a runtime error if Currency is BTC * Fixed a runtime error if Currency is BTC * Fixed the retries of Elastic Client * Add subscreensItem.menuInclude to menu data * Docker-Image-Pull Feature (#553) * Improvement: In Moqui-Multi-Instance added a Async-Pull-Image-Feature which pulls the image by Using Docker-Engine-Api from multiple registry like AWS, Azure, Docker-Hub * Update AUTHORS * Added: added the generic way to process cmd , by adding an extra field(authTokenPass) inside the entity InstanceImage , now user has separate field for cmd and password so all types of registries is easily configurable * Update ServerEntities.xml by adding description * Library updates, including Jetty 10.0.13 to 10.015 which had reported vulnerabilities; there are lots of dependencies updated in this set, see diff for full details * In ScreenRenderImpl change addFormFieldValue() and related methods to handle first, second, and last rows, for qvt and other client rendered output that needs full data for a form in a map/object * In root build.gradle change gitStatusAll task to be more tolerant of repos with no master branch * Add text-area.@autogrow attribute, supported only in qvt for now * Update various libraries including Groovy to 3.0.19 (which has some minor non-backward compatible changes single 3.0.10 with odd boolean behavior in rare cases, adjusted for in the framework long ago but should be watched for in custom code), Jetty to 10.0.16, H2 database, SLF4J, SnakeYAML, Apache Commons Lang3 * In build.gradle gitStatusAll task also handle upstream remotes with no master branch * Currency (#614) * Use moqui.basic.Uom entity to determine currency formatting and rounding details * Add currency-hide-symbol attribute as a complement to currency-unit-field, displaying the value without the currency symbol * Update authors file * Add and Handle Hmac Sha256 with timestamp * A couple of minor bug fixes in the EntityAutoServiceRunner and ContextJavaUtil (#618) * In EntityAutoServiceRunner, remove unwanted break statement to ensure support for multiple PK fields with wildcard (*). * In ContextJavaUtil, add missing future keyword. * Updated authors file. * Allow for 10 second threshold in nowTimestamp * In L10nFacadeImpl.formatCurrency() use disableAuthz() for entity find on Uom; small change to currency formatting test to pass with current OOTB settings * In addons.xml, added new moqui-sso component. * Add AutoCloseable extension to EntityListIterator for use with try with resources, thanks to Deepak Dixit for the suggestion * BugFix EntityListIterator not closed in NotificationMessageImpl#getNotifyUserIds * Implemented withCloseable/try-with-resources where needed in the code… (#625) * Implemented withCloseable/try-with-resources where needed in the code to ensure proper closure of the entity list iterator resource * Fixed withCloseable syntax --------- Co-authored-by: David E. Jones * Change EntityDataWriterImpl to use groovy CompileStatic, without this a compile error was missed in the changes for PR #625, fix small issues from this --------- Co-authored-by: Yao Chunlin Co-authored-by: David E. Jones Co-authored-by: Wei Zhang Co-authored-by: Rohit pawar <72196393+rohitpawar2811@users.noreply.github.com> Co-authored-by: Jens Hardings Co-authored-by: acetousk Co-authored-by: Ayman Abi Abdallah --- .../org/moqui/impl/EntitySyncServices.xml | 7 +--- .../context/NotificationMessageImpl.groovy | 11 ++---- .../impl/entity/EntityDataDocument.groovy | 12 ++---- .../impl/entity/EntityDataWriterImpl.groovy | 27 +++++++------ .../moqui/impl/entity/EntityFindBase.groovy | 39 +++++++------------ .../impl/entity/EntityListIteratorImpl.java | 1 + .../src/test/groovy/EntityNoSqlCrud.groovy | 10 ++--- 7 files changed, 40 insertions(+), 67 deletions(-) diff --git a/framework/service/org/moqui/impl/EntitySyncServices.xml b/framework/service/org/moqui/impl/EntitySyncServices.xml index bb0d8cad1..098bdf39d 100644 --- a/framework/service/org/moqui/impl/EntitySyncServices.xml +++ b/framework/service/org/moqui/impl/EntitySyncServices.xml @@ -236,13 +236,10 @@ along with this software (see the LICENSE.md file). If not, see // ec.logger.warn("=========== get#EntitySyncData entityName=${entryMap.entityName} count=${currentCount} find=${find}") if (currentCount > 0) { - EntityListIterator resultEli = find.iterator() - try { + find.iterator().withCloseable ({resultEli -> int levels = entryMap.dependents ? 2 : 0 resultEli.writeXmlText((Writer) entityWriter, null, levels) - } finally { - resultEli.close() - } + }) } } diff --git a/framework/src/main/groovy/org/moqui/impl/context/NotificationMessageImpl.groovy b/framework/src/main/groovy/org/moqui/impl/context/NotificationMessageImpl.groovy index 63bc08178..05433737b 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/NotificationMessageImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/context/NotificationMessageImpl.groovy @@ -20,10 +20,8 @@ import org.moqui.BaseArtifactException import org.moqui.Moqui import org.moqui.context.ExecutionContext import org.moqui.context.NotificationMessage -import org.moqui.context.NotificationMessage.NotificationType import org.moqui.entity.EntityFacade import org.moqui.entity.EntityList -import org.moqui.entity.EntityListIterator import org.moqui.entity.EntityValue import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -93,10 +91,9 @@ class NotificationMessageImpl implements NotificationMessage, Externalizable { // notify by group, skipping users already notified if (userGroupId) { - EntityListIterator eli = ef.find("moqui.security.UserGroupMember") + ef.find("moqui.security.UserGroupMember") .conditionDate("fromDate", "thruDate", new Timestamp(System.currentTimeMillis())) - .condition("userGroupId", userGroupId).disableAuthz().iterator() - try { + .condition("userGroupId", userGroupId).disableAuthz().iterator().withCloseable ({eli -> EntityValue nextValue while ((nextValue = (EntityValue) eli.next()) != null) { String userId = (String) nextValue.userId @@ -104,9 +101,7 @@ class NotificationMessageImpl implements NotificationMessage, Externalizable { checkedUserIds.add(userId) if (checkUserNotify(userId, ef)) notifyUserIds.add(userId) } - } finally { - eli.close(); - } + }) } // add all users subscribed to all messages on the topic diff --git a/framework/src/main/groovy/org/moqui/impl/entity/EntityDataDocument.groovy b/framework/src/main/groovy/org/moqui/impl/entity/EntityDataDocument.groovy index a5be31a91..66bcaab30 100644 --- a/framework/src/main/groovy/org/moqui/impl/entity/EntityDataDocument.groovy +++ b/framework/src/main/groovy/org/moqui/impl/entity/EntityDataDocument.groovy @@ -301,8 +301,7 @@ class EntityDataDocument { // do the one big query String lastDocId = null int docCount = 0 - EntityListIterator mainEli = mainFind.iterator() - try { + try (EntityListIterator mainEli = mainFind.iterator()) { logger.info("Feed dataDocumentId ${dataDocumentId} query complete (cursor opened) in ${System.currentTimeMillis() - startTimeMillis}ms") EntityValue ev while ((ev = (EntityValue) mainEli.next()) != null) { @@ -345,7 +344,6 @@ class EntityDataDocument { efi.ecfi.serviceFacade.sync().name(feedReceiveServiceName).parameter("documentList", documentMapList).call() } } finally { - mainEli.close() logger.info("Feed dataDocumentId ${dataDocumentId} feed complete and cursor closed in ${System.currentTimeMillis() - startTimeMillis}ms") } @@ -366,16 +364,14 @@ class EntityDataDocument { ArrayList documentMapList = ddi.hasAllPrimaryPks ? null : new ArrayList() // do the one big query - EntityListIterator mainEli = mainFind.iterator() - try { + + mainFind.iterator().withCloseable ({mainEli-> EntityValue ev while ((ev = (EntityValue) mainEli.next()) != null) { // logger.warn("=========== DataDocument query result for ${dataDocumentId}: ${ev}") mergeValueToDocMap(ev, ddi, documentMapMap, documentMapList, docTsString) } - } finally { - mainEli.close() - } + }) // make the actual list and return it if (ddi.hasAllPrimaryPks) { diff --git a/framework/src/main/groovy/org/moqui/impl/entity/EntityDataWriterImpl.groovy b/framework/src/main/groovy/org/moqui/impl/entity/EntityDataWriterImpl.groovy index e3872d702..0dee3676c 100644 --- a/framework/src/main/groovy/org/moqui/impl/entity/EntityDataWriterImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/entity/EntityDataWriterImpl.groovy @@ -14,6 +14,7 @@ package org.moqui.impl.entity import groovy.json.JsonBuilder +import groovy.transform.CompileStatic import org.moqui.entity.EntityValue import org.moqui.util.ObjectUtilities @@ -35,6 +36,7 @@ import java.time.format.DateTimeFormatter import java.util.zip.ZipEntry import java.util.zip.ZipOutputStream +@CompileStatic class EntityDataWriterImpl implements EntityDataWriter { private final static Logger logger = LoggerFactory.getLogger(EntityDataWriterImpl.class) @@ -170,9 +172,9 @@ class EntityDataWriterImpl implements EntityDataWriter { EntityDefinition ed = efi.getEntityDefinition(en) boolean useMaster = masterName != null && masterName.length() > 0 && ed.getMasterDefinition(masterName) != null EntityFind ef = makeEntityFind(en) - EntityListIterator eli = ef.iterator() - try { + + try (EntityListIterator eli = ef.iterator()) { if (!eli.hasNext()) continue String filename = path + '/' + en + '.' + fileType.name().toLowerCase() @@ -200,8 +202,6 @@ class EntityDataWriterImpl implements EntityDataWriter { } finally { pw.close() } - } finally { - eli.close() } } } catch (Throwable t) { @@ -248,8 +248,7 @@ class EntityDataWriterImpl implements EntityDataWriter { EntityDefinition ed = efi.getEntityDefinition(en) boolean useMaster = masterName != null && masterName.length() > 0 && ed.getMasterDefinition(masterName) != null EntityFind ef = makeEntityFind(en) - EntityListIterator eli = ef.iterator() - try { + try (EntityListIterator eli = ef.iterator()) { if (!eli.hasNext()) continue String filenameBase = tableColumnNames ? ed.getTableName() : en @@ -274,8 +273,6 @@ class EntityDataWriterImpl implements EntityDataWriter { } finally { out.closeEntry() } - } finally { - eli.close() } } } finally { @@ -289,7 +286,13 @@ class EntityDataWriterImpl implements EntityDataWriter { int writer(Writer writer) { if (dependentLevels > 0) efi.createAllAutoReverseManyRelationships() - LinkedHashSet activeEntityNames = skipEntityNames.size() > 0 ? entityNames - skipEntityNames : entityNames + LinkedHashSet activeEntityNames + if (skipEntityNames.size() == 0) { + activeEntityNames = entityNames + } else { + activeEntityNames = new LinkedHashSet<>(entityNames) + activeEntityNames.removeAll(skipEntityNames) + } EntityDefinition singleEd = null if (activeEntityNames.size() == 1) singleEd = efi.getEntityDefinition(activeEntityNames.first()) @@ -305,15 +308,11 @@ class EntityDataWriterImpl implements EntityDataWriter { for (String en in activeEntityNames) { EntityDefinition ed = efi.getEntityDefinition(en) boolean useMaster = masterName != null && masterName.length() > 0 && ed.getMasterDefinition(masterName) != null - EntityFind ef = makeEntityFind(en) - EntityListIterator eli = ef.iterator() - try { + try (EntityListIterator eli = makeEntityFind(en).iterator()) { EntityValue ev while ((ev = eli.next()) != null) { valuesWritten+= writeValue(ev, writer, useMaster) } - } finally { - eli.close() } } diff --git a/framework/src/main/groovy/org/moqui/impl/entity/EntityFindBase.groovy b/framework/src/main/groovy/org/moqui/impl/entity/EntityFindBase.groovy index 3d5d25c5f..31d015a3c 100644 --- a/framework/src/main/groovy/org/moqui/impl/entity/EntityFindBase.groovy +++ b/framework/src/main/groovy/org/moqui/impl/entity/EntityFindBase.groovy @@ -1149,19 +1149,18 @@ abstract class EntityFindBase implements EntityFind { } // call the abstract method - EntityListIterator eli - try { eli = iteratorExtended(queryWhereCondition, havingCondition, orderByExpanded, fieldInfoArray, fieldOptionsArray) } + try (EntityListIterator eli = iteratorExtended(queryWhereCondition, havingCondition, orderByExpanded, fieldInfoArray, fieldOptionsArray)) { + MNode databaseNode = this.efi.getDatabaseNode(ed.getEntityGroupName()) + if (limit != null && databaseNode != null && "cursor".equals(databaseNode.attribute("offset-style"))) { + el = (EntityListImpl) eli.getPartialList(offset != null ? offset : 0, limit, false) + } else { + el = (EntityListImpl) eli.getCompleteList(false); + } + } catch (SQLException e) { throw new EntitySqlException(makeErrorMsg("Error finding list of", LIST_ERROR, queryWhereCondition, ed, ec), e) } catch (ArtifactAuthorizationException e) { throw e } catch (Exception e) { throw new EntityException(makeErrorMsg("Error finding list of", LIST_ERROR, queryWhereCondition, ed, ec), e) } - MNode databaseNode = this.efi.getDatabaseNode(ed.getEntityGroupName()) - if (limit != null && databaseNode != null && "cursor".equals(databaseNode.attribute("offset-style"))) { - el = (EntityListImpl) eli.getPartialList(offset != null ? offset : 0, limit, true) - } else { - el = (EntityListImpl) eli.getCompleteList(true) - } - // register lock after because we can't before, don't know which records will be returned if (forUpdate && !isViewEntity && efi.ecfi.transactionFacade.getUseLockTrack()) { int elSize = el.size() @@ -1446,9 +1445,7 @@ abstract class EntityFindBase implements EntityFind { this.useCache(false) long totalUpdated = 0 - EntityListIterator eli = (EntityListIterator) null - try { - eli = iterator() + iterator().withCloseable ({eli -> EntityValue value while ((value = eli.next()) != null) { value.putAll(fieldsToSet) @@ -1458,9 +1455,7 @@ abstract class EntityFindBase implements EntityFind { totalUpdated++ } } - } finally { - if (eli != null) eli.close() - } + }) return totalUpdated } @@ -1495,33 +1490,27 @@ abstract class EntityFindBase implements EntityFind { } } else { this.resultSetConcurrency(ResultSet.CONCUR_UPDATABLE) - EntityListIterator eli = (EntityListIterator) null - try { - eli = iterator() + iterator().withCloseable ({eli-> + while (eli.next() != null) { // no longer need to clear cache, eli.remove() does that eli.remove() totalDeleted++ } - } finally { - if (eli != null) eli.close() - } + }) } return totalDeleted } @Override void extract(SimpleEtl etl) { - EntityListIterator eli = iterator() - try { + try (EntityListIterator eli = iterator()) { EntityValue ev while ((ev = eli.next()) != null) { etl.processEntry(ev) } } catch (StopException e) { logger.warn("EntityFind extract stopped on: " + (e.getCause()?.toString() ?: e.toString())) - } finally { - eli.close() } } diff --git a/framework/src/main/groovy/org/moqui/impl/entity/EntityListIteratorImpl.java b/framework/src/main/groovy/org/moqui/impl/entity/EntityListIteratorImpl.java index fccd72de6..f0c120292 100644 --- a/framework/src/main/groovy/org/moqui/impl/entity/EntityListIteratorImpl.java +++ b/framework/src/main/groovy/org/moqui/impl/entity/EntityListIteratorImpl.java @@ -285,6 +285,7 @@ public EntityValueBase currentEntityValueBase() { } catch (SQLException e) { throw new EntityException("Error getting all results", e); } finally { + //TODO: Remove closeAfter with respect to try-with-resource implementation if (closeAfter) close(); } } diff --git a/framework/src/test/groovy/EntityNoSqlCrud.groovy b/framework/src/test/groovy/EntityNoSqlCrud.groovy index 667e5db8d..262af5ec4 100644 --- a/framework/src/test/groovy/EntityNoSqlCrud.groovy +++ b/framework/src/test/groovy/EntityNoSqlCrud.groovy @@ -121,12 +121,11 @@ class EntityNoSqlCrud extends Specification { def "ELI find TestNoSqlEntity"() { when: - EntityListIterator eli EntityList partialEl = null EntityValue first = null - try { - eli = ec.entity.find("moqui.test.TestNoSqlEntity") - .orderBy("-testNumberInteger").iterator() + try (EntityListIterator eli = ec.entity.find("moqui.test.TestNoSqlEntity") + .orderBy("-testNumberInteger").iterator()) { + partialEl = eli.getPartialList(0, 100, false) @@ -134,10 +133,7 @@ class EntityNoSqlCrud extends Specification { first = eli.next() } catch (Exception e) { logger.error("partialEl error", e) - } finally { - if (eli != null) eli.close() } - // logger.warn("partialEl.size() ${partialEl.size()} first value ${first}") then: