From 8adc66f7e4c6377e8af8071dbbd4559842dae011 Mon Sep 17 00:00:00 2001 From: Pablo Castelo Date: Wed, 29 May 2024 10:47:01 +0200 Subject: [PATCH 1/4] Check if the output reach the LuceneIndex limit for a node property, if that's the case it will be stored as binary --- .../aecu/core/history/HistoryUtil.java | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/de/valtech/aecu/core/history/HistoryUtil.java b/core/src/main/java/de/valtech/aecu/core/history/HistoryUtil.java index 3f16c533..69d9c8bd 100644 --- a/core/src/main/java/de/valtech/aecu/core/history/HistoryUtil.java +++ b/core/src/main/java/de/valtech/aecu/core/history/HistoryUtil.java @@ -18,6 +18,9 @@ */ package de.valtech.aecu.core.history; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Calendar; import java.util.GregorianCalendar; @@ -50,6 +53,12 @@ import de.valtech.aecu.api.service.HistoryEntry.STATE; import de.valtech.aecu.core.service.HistoryEntryImpl; +import javax.jcr.Binary; +import javax.jcr.RepositoryException; +import javax.jcr.Session; +import javax.jcr.Value; +import javax.jcr.ValueFactory; + /** * Reads and writes history entries. * @@ -77,6 +86,9 @@ public class HistoryUtil { protected static final String ATTR_START = "start"; protected static final String ATTR_END = "end"; private static final String NAME_INDEX = "oak:index"; + // This size is limited by the LuceneDocumentMaker to be able to read the property and create the new index + // The limit is 102400 but just to be in the safe size, is set to a bit lower number + private static final int MAXIMUN_PROPERTY_SIZE = 100000; private Random random = new Random(); @@ -358,7 +370,19 @@ private void saveExecutionResultInHistory(ExecutionResult result, String path, R values.put(ATTR_RUN_STATE, result.getState().name()); values.put(ATTR_PATH, result.getPath()); if (StringUtils.isNotBlank(result.getOutput())) { - values.put(ATTR_RUN_OUTPUT, result.getOutput()); + if (result.getOutput().getBytes(StandardCharsets.UTF_8).length < MAXIMUN_PROPERTY_SIZE) { + values.put(ATTR_RUN_OUTPUT, result.getOutput()); + } else { + try { + ValueFactory factory = resolver.adaptTo(Session.class).getValueFactory(); + InputStream is = new ByteArrayInputStream(result.getOutput().getBytes()); + Binary binary = factory.createBinary(is); + Value value = factory.createValue(binary); + values.put(ATTR_RUN_OUTPUT, value); + } catch (RepositoryException e) { + LOG.error("Not able to save the output of the script as binary on the History node [{}]", entry.getPath()); + } + } } if (StringUtils.isNotBlank(result.getResult())) { values.put(ATTR_RUN_RESULT, result.getResult()); From 78a91cbb9bb5c937852b912742a4e98ce9258aeb Mon Sep 17 00:00:00 2001 From: Pablo Castelo Date: Wed, 29 May 2024 21:42:11 +0200 Subject: [PATCH 2/4] Store the value as binary when the size is too big --- .../java/de/valtech/aecu/core/history/HistoryUtil.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/de/valtech/aecu/core/history/HistoryUtil.java b/core/src/main/java/de/valtech/aecu/core/history/HistoryUtil.java index 69d9c8bd..b62ac812 100644 --- a/core/src/main/java/de/valtech/aecu/core/history/HistoryUtil.java +++ b/core/src/main/java/de/valtech/aecu/core/history/HistoryUtil.java @@ -54,9 +54,9 @@ import de.valtech.aecu.core.service.HistoryEntryImpl; import javax.jcr.Binary; +import javax.jcr.Node; import javax.jcr.RepositoryException; import javax.jcr.Session; -import javax.jcr.Value; import javax.jcr.ValueFactory; /** @@ -374,11 +374,13 @@ private void saveExecutionResultInHistory(ExecutionResult result, String path, R values.put(ATTR_RUN_OUTPUT, result.getOutput()); } else { try { - ValueFactory factory = resolver.adaptTo(Session.class).getValueFactory(); + Node node = entry.adaptTo(Node.class); + Session session = node.getSession(); + ValueFactory factory = session.getValueFactory(); InputStream is = new ByteArrayInputStream(result.getOutput().getBytes()); Binary binary = factory.createBinary(is); - Value value = factory.createValue(binary); - values.put(ATTR_RUN_OUTPUT, value); + node.setProperty(ATTR_RUN_OUTPUT, binary); + session.save(); } catch (RepositoryException e) { LOG.error("Not able to save the output of the script as binary on the History node [{}]", entry.getPath()); } From 4456f1cb75c6d5016a5897777daae12ffbaaf4d1 Mon Sep 17 00:00:00 2001 From: Pablo Castelo Date: Wed, 29 May 2024 21:43:48 +0200 Subject: [PATCH 3/4] Fix typo --- .../src/main/java/de/valtech/aecu/core/history/HistoryUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/de/valtech/aecu/core/history/HistoryUtil.java b/core/src/main/java/de/valtech/aecu/core/history/HistoryUtil.java index b62ac812..86320db7 100644 --- a/core/src/main/java/de/valtech/aecu/core/history/HistoryUtil.java +++ b/core/src/main/java/de/valtech/aecu/core/history/HistoryUtil.java @@ -87,7 +87,7 @@ public class HistoryUtil { protected static final String ATTR_END = "end"; private static final String NAME_INDEX = "oak:index"; // This size is limited by the LuceneDocumentMaker to be able to read the property and create the new index - // The limit is 102400 but just to be in the safe size, is set to a bit lower number + // The limit is 102400 but just to be in the safe side, is set to a bit lower number private static final int MAXIMUN_PROPERTY_SIZE = 100000; private Random random = new Random(); From 0e5cb32af29095454eae1a5ab4a43c1959d54df5 Mon Sep 17 00:00:00 2001 From: Pablo Castelo Date: Wed, 29 May 2024 23:29:58 +0200 Subject: [PATCH 4/4] Fix typo, store binary in a different property --- .../java/de/valtech/aecu/core/history/HistoryUtil.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/de/valtech/aecu/core/history/HistoryUtil.java b/core/src/main/java/de/valtech/aecu/core/history/HistoryUtil.java index 86320db7..57a9978e 100644 --- a/core/src/main/java/de/valtech/aecu/core/history/HistoryUtil.java +++ b/core/src/main/java/de/valtech/aecu/core/history/HistoryUtil.java @@ -78,6 +78,7 @@ public class HistoryUtil { public static final String ATTR_PATH = "path"; protected static final String ATTR_RUN_OUTPUT = "runOutput"; + protected static final String ATTR_RUN_OUTPUT_FULL = "runOutputFull"; protected static final String ATTR_RUN_STATE = "runState"; protected static final String ATTR_RUN_RESULT = "runResult"; protected static final String ATTR_RUN_TIME = "runTime"; @@ -88,7 +89,7 @@ public class HistoryUtil { private static final String NAME_INDEX = "oak:index"; // This size is limited by the LuceneDocumentMaker to be able to read the property and create the new index // The limit is 102400 but just to be in the safe side, is set to a bit lower number - private static final int MAXIMUN_PROPERTY_SIZE = 100000; + private static final int MAXIMUM_PROPERTY_SIZE = 100000; private Random random = new Random(); @@ -370,16 +371,18 @@ private void saveExecutionResultInHistory(ExecutionResult result, String path, R values.put(ATTR_RUN_STATE, result.getState().name()); values.put(ATTR_PATH, result.getPath()); if (StringUtils.isNotBlank(result.getOutput())) { - if (result.getOutput().getBytes(StandardCharsets.UTF_8).length < MAXIMUN_PROPERTY_SIZE) { + if (result.getOutput().getBytes(StandardCharsets.UTF_8).length < MAXIMUM_PROPERTY_SIZE) { values.put(ATTR_RUN_OUTPUT, result.getOutput()); } else { + values.put(ATTR_RUN_OUTPUT, "Output data too big, full data is stored as a binary in runOutputFull"); + LOG.info("Script result is bigger than 100 000 bytes. Full data can be found as a binary in property runOutputFull for path {}", path ); try { Node node = entry.adaptTo(Node.class); Session session = node.getSession(); ValueFactory factory = session.getValueFactory(); InputStream is = new ByteArrayInputStream(result.getOutput().getBytes()); Binary binary = factory.createBinary(is); - node.setProperty(ATTR_RUN_OUTPUT, binary); + node.setProperty(ATTR_RUN_OUTPUT_FULL, binary); session.save(); } catch (RepositoryException e) { LOG.error("Not able to save the output of the script as binary on the History node [{}]", entry.getPath());