From 05c76542c88f92b796a4d4c7367529e052225215 Mon Sep 17 00:00:00 2001 From: Nassim Boutekedjiret Date: Wed, 5 Jun 2024 18:18:04 +0200 Subject: [PATCH] Issue #203: Refactor-Extensions-code-to-check-protocol-efficiently * Reverted untested changes --- .../resources/log4j2.component.properties | 4 +- .../common/helpers/MetricsHubConstants.java | 2 +- .../engine/strategy/AbstractStrategy.java | 2 +- .../source/compute/ComputeProcessor.java | 118 +++++++++--------- 4 files changed, 59 insertions(+), 67 deletions(-) diff --git a/metricshub-agent/src/main/resources/log4j2.component.properties b/metricshub-agent/src/main/resources/log4j2.component.properties index 3b8272503..c66aced76 100644 --- a/metricshub-agent/src/main/resources/log4j2.component.properties +++ b/metricshub-agent/src/main/resources/log4j2.component.properties @@ -1,3 +1 @@ -log4j2.isThreadContextMapInheritable=true -log4j2.asyncLoggerRingBufferSize=32768 -log4j2.asyncLoggerWaitStrategy=Sleep \ No newline at end of file +log4j2.isThreadContextMapInheritable=true \ No newline at end of file diff --git a/metricshub-engine/src/main/java/org/sentrysoftware/metricshub/engine/common/helpers/MetricsHubConstants.java b/metricshub-engine/src/main/java/org/sentrysoftware/metricshub/engine/common/helpers/MetricsHubConstants.java index 6c9a3f7e5..ca3f5a581 100644 --- a/metricshub-engine/src/main/java/org/sentrysoftware/metricshub/engine/common/helpers/MetricsHubConstants.java +++ b/metricshub-engine/src/main/java/org/sentrysoftware/metricshub/engine/common/helpers/MetricsHubConstants.java @@ -170,7 +170,7 @@ public class MetricsHubConstants { /** * Thread Timeout */ - public static final long THREAD_TIMEOUT = 2 * 60L; // 15 minutes + public static final long THREAD_TIMEOUT = 15 * 60L; // 15 minutes // MetricsHub / OpenTelemetry mappings diff --git a/metricshub-engine/src/main/java/org/sentrysoftware/metricshub/engine/strategy/AbstractStrategy.java b/metricshub-engine/src/main/java/org/sentrysoftware/metricshub/engine/strategy/AbstractStrategy.java index 6fdcf07a5..3b731a63c 100644 --- a/metricshub-engine/src/main/java/org/sentrysoftware/metricshub/engine/strategy/AbstractStrategy.java +++ b/metricshub-engine/src/main/java/org/sentrysoftware/metricshub/engine/strategy/AbstractStrategy.java @@ -150,7 +150,7 @@ protected void processSourcesAndComputes( .build() .run(() -> runSource(connectorId, attributes, source, previousSourceTable)); - if (sourceTable == null || (sourceTable.getTable() == null && sourceTable.getRawData() == null)) { + if (sourceTable == null) { log.warn( "Hostname {} - Received null source table for Source key {} - Connector {} - Monitor {}.", hostname, diff --git a/metricshub-engine/src/main/java/org/sentrysoftware/metricshub/engine/strategy/source/compute/ComputeProcessor.java b/metricshub-engine/src/main/java/org/sentrysoftware/metricshub/engine/strategy/source/compute/ComputeProcessor.java index 313c2aafe..1bd2662a7 100644 --- a/metricshub-engine/src/main/java/org/sentrysoftware/metricshub/engine/strategy/source/compute/ComputeProcessor.java +++ b/metricshub-engine/src/main/java/org/sentrysoftware/metricshub/engine/strategy/source/compute/ComputeProcessor.java @@ -213,7 +213,7 @@ public void process(@SpanAttribute("compute.definition") final ArrayTranslate ar final List> resultTable = new ArrayList<>(); List resultRow; - for (List row : extractSourceTableListOfList()) { + for (List row : sourceTable.getTable()) { if (columnIndex >= row.size()) { log.warn( "Hostname {} - The index of the column is {} but the row size is {}, the translate computation cannot be performed.", @@ -244,17 +244,7 @@ public void process(@SpanAttribute("compute.definition") final ArrayTranslate ar } sourceTable.setTable(resultTable); - sourceTable.setRawData(SourceTable.tableToCsv(extractSourceTableListOfList(), TABLE_SEP, false)); - } - - /** - * Find the source table list of list from the sourceTable instance. - * - * @return The source table as list of list - */ - private List> extractSourceTableListOfList() { - final List> result = sourceTable.getTable(); - return result == null ? new ArrayList<>() : result; + sourceTable.setRawData(SourceTable.tableToCsv(sourceTable.getTable(), TABLE_SEP, false)); } @Override @@ -288,7 +278,7 @@ public void process(@SpanAttribute("compute.definition") final And and) { int colOperand2 = getColumnIndex(operand2); - for (List line : extractSourceTableListOfList()) { + for (List line : sourceTable.getTable()) { try { if (columnIndex < line.size()) { // Set the column value of the 'line' at 'columnIndex' to the result of a bitwise 'AND' operation @@ -313,7 +303,7 @@ public void process(@SpanAttribute("compute.definition") final And and) { } } - sourceTable.setRawData(SourceTable.tableToCsv(extractSourceTableListOfList(), TABLE_SEP, false)); + sourceTable.setRawData(SourceTable.tableToCsv(sourceTable.getTable(), TABLE_SEP, false)); } @Override @@ -390,7 +380,7 @@ public void process(@SpanAttribute("compute.definition") final Awk awk) { } final String input = (sourceTable.getRawData() == null || sourceTable.getRawData().isEmpty()) - ? SourceTable.tableToCsv(extractSourceTableListOfList(), TABLE_SEP, true) + ? SourceTable.tableToCsv(sourceTable.getTable(), TABLE_SEP, true) : sourceTable.getRawData(); final String computeKey = String.format(LOG_COMPUTE_KEY_SUFFIX_TEMPLATE, sourceKey, this.index); @@ -481,7 +471,8 @@ static boolean checkConvert(final Convert convert) { * @param columnIndex The column number */ void convertHex2Dec(final Integer columnIndex) { - extractSourceTableListOfList() + sourceTable + .getTable() .forEach(row -> { if (columnIndex < row.size()) { final String value = row.get(columnIndex).replace("0x", EMPTY).replace(":", EMPTY).replaceAll("\\s*", EMPTY); @@ -498,7 +489,7 @@ void convertHex2Dec(final Integer columnIndex) { columnIndex ); }); - sourceTable.setRawData(SourceTable.tableToCsv(extractSourceTableListOfList(), TABLE_SEP, false)); + sourceTable.setRawData(SourceTable.tableToCsv(sourceTable.getTable(), TABLE_SEP, false)); } /** @@ -507,7 +498,8 @@ void convertHex2Dec(final Integer columnIndex) { * @param columnIndex The column number */ void convertArray2SimpleStatus(final Integer columnIndex) { - extractSourceTableListOfList() + sourceTable + .getTable() .forEach(row -> { if (columnIndex < row.size()) { final String value = PslUtils.nthArg(row.get(columnIndex), "1-", VERTICAL_BAR, NEW_LINE); @@ -521,7 +513,7 @@ void convertArray2SimpleStatus(final Integer columnIndex) { columnIndex ); }); - sourceTable.setRawData(SourceTable.tableToCsv(extractSourceTableListOfList(), TABLE_SEP, false)); + sourceTable.setRawData(SourceTable.tableToCsv(sourceTable.getTable(), TABLE_SEP, false)); } /** @@ -600,12 +592,12 @@ public void process(@SpanAttribute("compute.definition") final DuplicateColumn d // for each list in the list, duplicate the column of the given index int columnIndex = duplicateColumn.getColumn() - 1; - for (List elementList : extractSourceTableListOfList()) { + for (List elementList : sourceTable.getTable()) { if (columnIndex >= 0 && columnIndex < elementList.size()) { elementList.add(columnIndex, elementList.get(columnIndex)); } } - sourceTable.setRawData(SourceTable.tableToCsv(extractSourceTableListOfList(), TABLE_SEP, false)); + sourceTable.setRawData(SourceTable.tableToCsv(sourceTable.getTable(), TABLE_SEP, false)); } @Override @@ -644,7 +636,8 @@ private void processAbstractMatchingLines(final AbstractMatchingLines abstractMa : null; // If there are both a regex and a valueList, both are applied, one after the other. - final List> filteredTable = extractSourceTableListOfList() + final List> filteredTable = sourceTable + .getTable() .stream() .filter(line -> columnIndex < line.size() && @@ -654,7 +647,7 @@ private void processAbstractMatchingLines(final AbstractMatchingLines abstractMa .collect(Collectors.toList()); sourceTable.setTable(filteredTable); - sourceTable.setRawData(SourceTable.tableToCsv(extractSourceTableListOfList(), TABLE_SEP, false)); + sourceTable.setRawData(SourceTable.tableToCsv(sourceTable.getTable(), TABLE_SEP, false)); } } @@ -695,8 +688,8 @@ private boolean isConsistentMatchingLinesInfo(AbstractMatchingLines abstractMatc abstractMatchingLines.getColumn() != null && abstractMatchingLines.getColumn() > 0 && sourceTable != null && - extractSourceTableListOfList() != null && - !extractSourceTableListOfList().isEmpty() + sourceTable.getTable() != null && + !sourceTable.getTable().isEmpty() ); // CHECKSTYLE:ON } @@ -786,7 +779,7 @@ public void process(@SpanAttribute("compute.definition") final Extract extract) String text; List> resultTable = new ArrayList<>(); List resultRow; - for (List row : extractSourceTableListOfList()) { + for (List row : sourceTable.getTable()) { if (columnIndex >= row.size()) { log.warn("Hostname {} - Invalid column index: {}. The table remains unchanged.", hostname, column); return; @@ -807,7 +800,7 @@ public void process(@SpanAttribute("compute.definition") final Extract extract) } sourceTable.setTable(resultTable); - sourceTable.setRawData(SourceTable.tableToCsv(extractSourceTableListOfList(), TABLE_SEP, false)); + sourceTable.setRawData(SourceTable.tableToCsv(sourceTable.getTable(), TABLE_SEP, false)); } @Override @@ -838,7 +831,7 @@ public void process( columnIndex--; - for (final List line : extractSourceTableListOfList()) { + for (final List line : sourceTable.getTable()) { if (columnIndex < line.size()) { final String columnValue = line.get(columnIndex); @@ -861,7 +854,7 @@ public void process( } } } - sourceTable.setRawData(SourceTable.tableToCsv(extractSourceTableListOfList(), TABLE_SEP, false)); + sourceTable.setRawData(SourceTable.tableToCsv(sourceTable.getTable(), TABLE_SEP, false)); } /** @@ -977,7 +970,7 @@ public void process(@SpanAttribute("compute.definition") final KeepColumns keepC List> resultTable = new ArrayList<>(); List resultRow; columnNumbers = columnNumbers.stream().filter(Objects::nonNull).sorted().collect(Collectors.toList()); - for (List row : extractSourceTableListOfList()) { + for (List row : sourceTable.getTable()) { resultRow = new ArrayList<>(); for (Integer columnIndex : columnNumbers) { if (columnIndex < 1 || columnIndex > row.size()) { @@ -998,7 +991,7 @@ public void process(@SpanAttribute("compute.definition") final KeepColumns keepC } sourceTable.setTable(resultTable); - sourceTable.setRawData(SourceTable.tableToCsv(extractSourceTableListOfList(), TABLE_SEP, false)); + sourceTable.setRawData(SourceTable.tableToCsv(sourceTable.getTable(), TABLE_SEP, false)); } /** @@ -1110,7 +1103,7 @@ public void process(@SpanAttribute("compute.definition") final PerBitTranslation return; } - for (List line : extractSourceTableListOfList()) { + for (List line : sourceTable.getTable()) { if (columnIndex < line.size()) { long valueToBeReplacedLong; @@ -1132,7 +1125,7 @@ public void process(@SpanAttribute("compute.definition") final PerBitTranslation line.set(columnIndex, columnResult); } } - sourceTable.setRawData(SourceTable.tableToCsv(extractSourceTableListOfList(), TABLE_SEP, false)); + sourceTable.setRawData(SourceTable.tableToCsv(sourceTable.getTable(), TABLE_SEP, false)); } /** @@ -1175,11 +1168,12 @@ public void process(@SpanAttribute("compute.definition") final Replace replace) if (COLUMN_PATTERN.matcher(replacement).matches()) { final int replacementColumnIndex = getColumnIndex(replacement); - if (!extractSourceTableListOfList().isEmpty()) { + if (!sourceTable.getTable().isEmpty()) { // If strToReplace is like "$n", the strToReplace is actually the content of the column n. if (COLUMN_PATTERN.matcher(strToReplace).matches()) { final int strToReplaceColumnIndex = getColumnIndex(strToReplace); - extractSourceTableListOfList() + sourceTable + .getTable() .forEach(line -> { if (validateSizeAndIndices(line.size(), columnIndex, replacementColumnIndex, strToReplaceColumnIndex)) { line.set( @@ -1189,7 +1183,8 @@ public void process(@SpanAttribute("compute.definition") final Replace replace) } }); } else { - extractSourceTableListOfList() + sourceTable + .getTable() .forEach(line -> { if (validateSizeAndIndices(line.size(), columnIndex, replacementColumnIndex)) { line.set(columnIndex, line.get(columnIndex).replace(strToReplace, line.get(replacementColumnIndex))); @@ -1201,8 +1196,9 @@ public void process(@SpanAttribute("compute.definition") final Replace replace) // If strToReplace is like "$n", the strToReplace is actually the content of the column n. if (COLUMN_PATTERN.matcher(strToReplace).matches()) { final int strToReplaceColumnIndex = getColumnIndex(strToReplace); - if (!extractSourceTableListOfList().isEmpty()) { - extractSourceTableListOfList() + if (!sourceTable.getTable().isEmpty()) { + sourceTable + .getTable() .forEach(line -> { if (validateSizeAndIndices(line.size(), columnIndex, strToReplaceColumnIndex)) { line.set(columnIndex, line.get(columnIndex).replace(line.get(strToReplaceColumnIndex), replacement)); @@ -1210,7 +1206,8 @@ public void process(@SpanAttribute("compute.definition") final Replace replace) }); } } else { - extractSourceTableListOfList() + sourceTable + .getTable() .forEach(line -> { if (validateSizeAndIndices(line.size(), columnIndex)) { line.set(columnIndex, line.get(columnIndex).replace(strToReplace, replacement)); @@ -1220,9 +1217,9 @@ public void process(@SpanAttribute("compute.definition") final Replace replace) } sourceTable.setTable( - SourceTable.csvToTable(SourceTable.tableToCsv(extractSourceTableListOfList(), TABLE_SEP, false), TABLE_SEP) + SourceTable.csvToTable(SourceTable.tableToCsv(sourceTable.getTable(), TABLE_SEP, false), TABLE_SEP) ); - sourceTable.setRawData(SourceTable.tableToCsv(extractSourceTableListOfList(), TABLE_SEP, false)); + sourceTable.setRawData(SourceTable.tableToCsv(sourceTable.getTable(), TABLE_SEP, false)); } /** @@ -1344,7 +1341,8 @@ void performSubstring( final Function startFunction = getValueFunction(startColumnIndex); final Function endFunction = getValueFunction(endColumnIndex); - extractSourceTableListOfList() + sourceTable + .getTable() .forEach(row -> { if (columnIndex < row.size()) { final String columnValue = row.get(columnIndex); @@ -1383,7 +1381,7 @@ void performSubstring( log.warn("Hostname {} - Cannot perform substring on row {} on column index {}", hostname, row, columnIndex); }); - sourceTable.setRawData(SourceTable.tableToCsv(extractSourceTableListOfList(), TABLE_SEP, false)); + sourceTable.setRawData(SourceTable.tableToCsv(sourceTable.getTable(), TABLE_SEP, false)); } /** @@ -1471,7 +1469,7 @@ public void process(@SpanAttribute("compute.definition") final Translate transla final String defaultTranslation = translations.get(DEFAULT); - for (List line : extractSourceTableListOfList()) { + for (List line : sourceTable.getTable()) { if (columnIndex < line.size()) { final String valueToBeReplaced = line.get(columnIndex).toLowerCase(); final String newValue = translations.getOrDefault(valueToBeReplaced, defaultTranslation); @@ -1493,11 +1491,11 @@ public void process(@SpanAttribute("compute.definition") final Translate transla if (needSerialization) { sourceTable.setTable( - SourceTable.csvToTable(SourceTable.tableToCsv(extractSourceTableListOfList(), TABLE_SEP, false), TABLE_SEP) + SourceTable.csvToTable(SourceTable.tableToCsv(sourceTable.getTable(), TABLE_SEP, false), TABLE_SEP) ); } - sourceTable.setRawData(SourceTable.tableToCsv(extractSourceTableListOfList(), TABLE_SEP, false)); + sourceTable.setRawData(SourceTable.tableToCsv(sourceTable.getTable(), TABLE_SEP, false)); } /** @@ -1521,7 +1519,7 @@ public void process(@SpanAttribute("compute.definition") final Xml2Csv xml2csv) if (xmlResult != null && !xmlResult.isEmpty()) { sourceTable.setTable(xmlResult); - sourceTable.setRawData(SourceTable.tableToCsv(extractSourceTableListOfList(), TABLE_SEP, false)); + sourceTable.setRawData(SourceTable.tableToCsv(sourceTable.getTable(), TABLE_SEP, false)); } } catch (Exception e) { logComputeError( @@ -1598,11 +1596,11 @@ private void performMathComputeOnTable( final String operand2, final int operand2Index ) { - for (List line : extractSourceTableListOfList()) { + for (List line : sourceTable.getTable()) { performMathComputeOnLine(computeOperation, columnIndex, operand2, operand2Index, line); } - sourceTable.setRawData(SourceTable.tableToCsv(extractSourceTableListOfList(), TABLE_SEP, false)); + sourceTable.setRawData(SourceTable.tableToCsv(sourceTable.getTable(), TABLE_SEP, false)); } /** @@ -1694,12 +1692,12 @@ private void processAbstractConcat(final AbstractConcat abstractConcat) { abstractConcat.getColumn() != null && abstractConcat.getColumn() > 0 && sourceTable != null && - extractSourceTableListOfList() != null && - !extractSourceTableListOfList().isEmpty(); + sourceTable.getTable() != null && + !sourceTable.getTable().isEmpty(); if (firstChecks) { // Case 1 : concatenation with an exiting column - if (abstractConcat.getColumn() <= extractSourceTableListOfList().get(0).size()) { + if (abstractConcat.getColumn() <= sourceTable.getTable().get(0).size()) { int columnIndex = abstractConcat.getColumn() - 1; String concatString = abstractConcat.getValue(); @@ -1708,30 +1706,26 @@ private void processAbstractConcat(final AbstractConcat abstractConcat) { Matcher matcher = COLUMN_PATTERN.matcher(concatString); if (matcher.matches()) { int concatColumnIndex = Integer.parseInt(matcher.group(1)) - 1; - extractSourceTableListOfList() - .forEach(line -> concatColumns(line, columnIndex, concatColumnIndex, abstractConcat)); + sourceTable.getTable().forEach(line -> concatColumns(line, columnIndex, concatColumnIndex, abstractConcat)); } else { - extractSourceTableListOfList().forEach(line -> concatString(line, columnIndex, abstractConcat)); + sourceTable.getTable().forEach(line -> concatString(line, columnIndex, abstractConcat)); // Serialize and deserialize // in case the String to concat contains a ';' // so that a new column is created. if (concatString.contains(TABLE_SEP)) { sourceTable.setTable( - SourceTable.csvToTable( - SourceTable.tableToCsv(extractSourceTableListOfList(), TABLE_SEP, false), - TABLE_SEP - ) + SourceTable.csvToTable(SourceTable.tableToCsv(sourceTable.getTable(), TABLE_SEP, false), TABLE_SEP) ); } } - } else if (abstractConcat.getColumn() == extractSourceTableListOfList().get(0).size() + 1) { + } else if (abstractConcat.getColumn() == sourceTable.getTable().get(0).size() + 1) { // Case 2 : concatenation with non existing column // add at the end of the list (or at the beginning if the list is empty) - extractSourceTableListOfList().forEach(line -> line.add(abstractConcat.getValue())); + sourceTable.getTable().forEach(line -> line.add(abstractConcat.getValue())); } - sourceTable.setRawData(SourceTable.tableToCsv(extractSourceTableListOfList(), TABLE_SEP, false)); + sourceTable.setRawData(SourceTable.tableToCsv(sourceTable.getTable(), TABLE_SEP, false)); } }