Skip to content

Commit

Permalink
ERTool Updates (recompile required) + DataPower (Analytics.xml) + AP…
Browse files Browse the repository at this point in the history 
…I Management v5 (postmortem.xml) Updates

ERTool Updates:
- Enhancement to file writing (FileOutputStream -> OutputStream), resolves Issue #12

DataPower (Analytics.xml) Updates:
- (Update) Moved backtrace and related info to reload event prominently at the top (above best practice suggestions)
- (New) Added formula to track Raid Disk Failure in Hardware status provider

API Management (postmortem.xml) Updates:
- (Update) Fixed extraction of date in topology-config.json formulas
- (New) Added tracking of CLI commands and various upgrade messages related to an upgrade event
  • Loading branch information
@dmicale
dmicale committed Feb 23, 2020
1 parent 580e2a7 commit b9a98b1
Show file tree
Hide file tree
Showing 4 changed files with 88 additions and 53 deletions.
33 changes: 22 additions & 11 deletions framework/Analytics.xml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<Analytics version="DataPower_4.0.1" date="Feb 21 2020 5:30PM EST">
<Analytics version="DataPower_4.0.2" date="Feb 22 2020 8:30AM EST">
<Formula CollapseResult="true" FormulaID="IsHardwareDevice">
<Expression LogLevel="Warning" NextOperation="And">
<Section Name="LicenseStatus" XPath="/Status/LicenseStatus//Feature"/>
Expand Down Expand Up @@ -1896,16 +1896,16 @@
</Formula>
<!-- end of internal issue which caused device reboot -->

<Formula Categories="memory;failure">
<Formula Categories="memory;failure" TopPositionResult="true">
<Name>Backtrace Available</Name>
<DisplayMessage>Backtrace Available (Requires Internal Decoding)</DisplayMessage>
<Expression LogLevel="Error">
<Expression LogLevel="Critical">
<Section Name="&lt;ExtendedBacktrace@datapower.ibm.com&gt;" Base64="true" MIME="true" LineReturn="true"/>
</Expression>
<Expression LogLevel="Error">
<Expression LogLevel="Critical">
<Section Name="&lt;Backtrace@datapower.ibm.com&gt;" Base64="true" MIME="true" LineReturn="true"/>
</Expression>
<Expression LogLevel="Error">
<Expression LogLevel="Critical">
<Section Name="&lt;PreviousBacktrace@datapower.ibm.com&gt;" Base64="true" MIME="true" LineReturn="true"/>
</Expression>
</Formula>
Expand Down Expand Up @@ -2229,6 +2229,17 @@
</Expression>
</Formula>

<Formula Categories="hardware">
<Name>Raid Disk Failure</Name>
<DisplayMessage>{/Hardware/diagraiddiskfailure/text()}</DisplayMessage>
<Expression LogLevel="Critical">
<Section Name="Hardware" XPath="/Hardware/diagraiddiskfailure/@status"/>
<RegExp>*</RegExp>
<Condition FieldPosition="0" Operation="EqualsTo" Value="error"/>
<Condition FieldPosition="0" Operation="EqualsTo" Value="critical"/>
</Expression>
</Formula>

<!-- detect DiagnosticsLog version of crypto engine status -->
<Formula Categories="configuration;failure;hardware" SortCondition="Timestamp" SortMethod="descending">
<Name>{Condition:SectionName} {Condition:Timestamp} Reports crypto accelerator is not detected, contact support.</Name>
Expand Down Expand Up @@ -3037,11 +3048,11 @@
</Formula>

<!-- report backtrace transaction ids below failure notification so user knows if its a reload or a watchdog -->
<Formula Categories="failure" SortCondition="Timestamp" SortMethod="descending" SortOption="EEE MMM dd HH:mm:ss yyyy">
<Formula Categories="failure" TopPositionResult="true" SortCondition="Timestamp" SortMethod="descending" SortOption="EEE MMM dd HH:mm:ss yyyy">
<Name>[BACKTRACE] Recorded on {Condition:Timestamp}</Name>
<DisplayMessage>Transaction ID: {Condition:TransID}, Seconds Up: {Condition:SecsUp}, SigNum: {Condition:SigNum}</DisplayMessage>
<URL description="Backtraces Provide Transaction ID, in case of Watchdog Events, there may be multiple backtraces in a 30 second timeframe, these transaction IDs may not relate to the reload: ">TBD Technote</URL>
<Expression LogLevel="Error">
<Expression LogLevel="Critical">
<Section Name="Backtrace" Wildcard="True" XPath="/Root/text()"/>
<RegExp>*</RegExp>
<Condition FieldPosition="1" RegGroup="All" ConditionName="Timestamp" Operation="NotEqualTo" Value="" NextOperation="And">0x[0-9a-fA-F]*\: ([a-zA-Z]{3} [a-zA-Z]{3}\s+\d+\s\d{2}:\d{2}:\d{2}\s\d{4})</Condition>
Expand All @@ -3051,11 +3062,11 @@
</Expression>
</Formula>

<Formula Categories="failure" SortCondition="Timestamp" SortMethod="descending">
<Formula Categories="failure" TopPositionResult="true" SortCondition="Timestamp" SortMethod="descending">
<Name>[BACKTRACE] Recorded on {Condition:Timestamp}</Name>
<DisplayMessage>Transaction ID: {Condition:TransID}, Seconds Up: {Condition:SecsUp}, SigNum: {Condition:SigNum}</DisplayMessage>
<URL description="Backtraces Provide Transaction ID, in case of Watchdog Events, there may be multiple backtraces in a 30 second timeframe, these transaction IDs may not relate to the reload: ">TBD Technote</URL>
<Expression LogLevel="Error">
<Expression LogLevel="Critical">
<Section Name="Backtrace" Wildcard="True" XPath="/Root/text()"/>
<RegExp>*</RegExp>
<Condition FieldPosition="1" RegGroup="All" ConditionName="Timestamp" Operation="NotEqualTo" Value="" NextOperation="And">Thread 0x[0-9a-fA-F]*\: (\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)</Condition>
Expand Down Expand Up @@ -3431,7 +3442,7 @@
<!-- end MQ Config search for retry interval -->

<!-- Backtrace search for objects related to reloads -->
<Formula Categories="failure">
<Formula Categories="failure" TopPositionResult="true">
<Name>Backtrace on {Condition:Timestamp} Contained Configuration Object Information: Component={Condition:Component}, Domain={Condition:Domain}, Config={Condition:Config}</Name>
<DisplayMessage>Component={Condition:Component}, Domain={Condition:Domain}, Config={Condition:Config} should be reviewed further as it could be related to a reload of the appliance.</DisplayMessage>
<URL description="Backtrace generated during reload notes domain and configuration object: ">https://www.ibm.com/support/pages/node/498213</URL>
Expand All @@ -3446,7 +3457,7 @@
</Formula>

<!-- 6.0 backtrace search -->
<Formula Categories="failure">
<Formula Categories="failure" TopPositionResult="true">
<Name>Backtrace on {Condition:Timestamp} Contained Configuration Object Information: Component={Condition:Component}, Domain={Condition:Domain}, Config={Condition:Config}</Name>
<DisplayMessage>Component={Condition:Component}, Domain={Condition:Domain}, Config={Condition:Config} should be reviewed further as it could be related to a reload of the appliance.</DisplayMessage>
<URL description="Backtrace generated during reload notes domain and configuration object: ">https://www.ibm.com/support/pages/node/498213</URL>
Expand Down
38 changes: 30 additions & 8 deletions framework/postmortem.xml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
<!-- This supports Cast Iron / API Management / API Connect post mortems -->
<Analytics version="APIConnect_1.0.0.6" date="Oct 1 2018 9:30AM EST">
<!-- This supports Cast Iron / API Management (v5) / API Connect (v5) post mortems -->
<Analytics version="APIConnect_1.0.0.8" date="Feb 22 2020 11:00PM EST">
<!-- keeping this at the top as it serves as a wildcard to cache all files in the archive, makes tool output more efficient, does not display this result formula to user -->
<Formula Categories="general">
<Expression LogLevel="Info">
Expand Down Expand Up @@ -306,8 +306,8 @@
<RegExp>*</RegExp>
<!-- gets latest date result of the topology config -->
<Condition FieldPosition="1" RegGroup="All" ConditionName="Structure" Operation="NotEqualTo" Value="" NextOperation="And">\"date\"((?![\s\S]*\"date\"+)[0-9a-zA-Z\s\:_\"\-\+\[\],.\(\){\/\\\-\=}\@!#$%^&amp;*]+)</Condition>
<Condition FieldPosition="1" InValue="{Condition:Structure}" RegGroup="All" ConditionName="Date" Operation="NotEqualTo" Value="" NextOperation="And">\:\s\"([a-zA-Z]{3}\s[a-zA-Z]{3}\s[0-9]{1,2}\s[0-9]{4}\s[0-9]{1,2}\:[0-9]{1,2}\:[0-9]{1,2}\s[A-Z]{1,3}\+[0-9]{1,4}\s\([a-zA-Z]{1,3}\))\"</Condition>
<Condition FieldPosition="1" InValue="{Condition:Structure}" RegGroup="All" ConditionName="GatewayClusters" Operation="NotEqualTo" Value="" NextOperation="And">(\"gatewayClusters\"\: \[\s+\{[0-9a-zA-Z\s\:_\"\-\+\[\],.\\\/\={\}\(\)]+)\"managementClusters\"</Condition>
<Condition FieldPosition="1" InValue="{Condition:Structure}" RegGroup="AllResult" ConditionName="Date" Operation="NotEqualTo" Value="" NextOperation="And">\:\s\"([a-zA-Z]{3}\s[a-zA-Z]{3}\s[0-9]{1,2}\s[0-9]{4}\s[0-9]{1,2}\:[0-9]{1,2}\:[0-9]{1,2}.*\s[A-Za-z0-9]+\))\"</Condition>
<Condition FieldPosition="1" InValue="{Condition:Structure}" RegGroup="AllResult" ConditionName="GatewayClusters" Operation="NotEqualTo" Value="" NextOperation="And">(\"gatewayClusters\"\: \[\s+\{[0-9a-zA-Z\s\:_\"\-\+\[\],.\\\/\={\}\(\)]+)\"managementClusters\"</Condition>
<Condition FieldPosition="1" InValue="{Condition:GatewayClusters}" RegGroup="AllResult" ConditionName="ClusterID" Operation="NotEqualTo" Value="" NextOperation="And">\"_id\":\s\"([a-zA-Z0-9.\-_ ]+)\"</Condition>
<Condition FieldPosition="1" InValue="{Condition:GatewayClusters}" RegGroup="AllResult" ConditionName="ClusterName" Operation="NotEqualTo" Value="" NextOperation="And">\"name\":\s\"([a-zA-Z0-9.\-_ ]+)\"</Condition>
<Condition FieldPosition="1" InValue="{Condition:GatewayClusters}" RegGroup="AllResult" ConditionName="HostName" Operation="NotEqualTo" Value="" NextOperation="And">\"hostname\":\s\"([a-zA-Z0-9.\-_]+)\"</Condition>
Expand All @@ -324,8 +324,8 @@
<RegExp>*</RegExp>
<!-- gets latest date result of the topology config -->
<Condition FieldPosition="1" RegGroup="All" ConditionName="Structure" Operation="NotEqualTo" Value="" NextOperation="And">\"date\"((?![\s\S]*\"date\"+)[0-9a-zA-Z\s\:_\"\-\+\[\],.\(\){\/\\\-\=}\@!#$%^&amp;*]+)</Condition>
<Condition FieldPosition="1" InValue="{Condition:Structure}" RegGroup="All" ConditionName="Date" Operation="NotEqualTo" Value="" NextOperation="And">\:\s\"([a-zA-Z]{3}\s[a-zA-Z]{3}\s[0-9]{1,2}\s[0-9]{4}\s[0-9]{1,2}\:[0-9]{1,2}\:[0-9]{1,2}\s[A-Z]{1,3}\+[0-9]{1,4}\s\([a-zA-Z]{1,3}\))\"</Condition>
<Condition FieldPosition="1" InValue="{Condition:Structure}" RegGroup="All" ConditionName="ManagementClusters" Operation="NotEqualTo" Value="" NextOperation="And">(\"managementClusters\"\: \[\s+\{[0-9a-zA-Z\s\:_\"\-\+\[\],.\\\/\={\}\(\)]+)\"servers\"</Condition>
<Condition FieldPosition="1" InValue="{Condition:Structure}" RegGroup="AllResult" ConditionName="Date" Operation="NotEqualTo" Value="" NextOperation="And">\:\s\"([a-zA-Z]{3}\s[a-zA-Z]{3}\s[0-9]{1,2}\s[0-9]{4}\s[0-9]{1,2}\:[0-9]{1,2}\:[0-9]{1,2}.*\s[A-Za-z0-9]+\))\"</Condition>
<Condition FieldPosition="1" InValue="{Condition:Structure}" RegGroup="AllResult" ConditionName="ManagementClusters" Operation="NotEqualTo" Value="" NextOperation="And">(\"managementClusters\"\: \[\s+\{[0-9a-zA-Z\s\:_\"\-\+\[\],.\\\/\={\}\(\)]+)\"servers\"</Condition>
<Condition FieldPosition="1" InValue="{Condition:ManagementClusters}" RegGroup="AllResult" ConditionName="ClusterID" Operation="NotEqualTo" Value="" NextOperation="And">\"_id\":\s\"([a-zA-Z0-9.\-_ ]+)\"</Condition>
<Condition FieldPosition="1" InValue="{Condition:ManagementClusters}" RegGroup="AllResult" ConditionName="ClusterName" Operation="NotEqualTo" Value="" NextOperation="And">\"name\":\s\"([a-zA-Z0-9.\-_ ]+)\"</Condition>
<Condition FieldPosition="1" InValue="{Condition:ManagementClusters}" RegGroup="AllResult" ConditionName="HostName" Operation="NotEqualTo" Value="" NextOperation="And">\"hostname\":\s\"([a-zA-Z0-9.\-_]+)\"</Condition>
Expand All @@ -341,8 +341,8 @@
<RegExp>*</RegExp>
<!-- gets latest date result of the topology config -->
<Condition FieldPosition="1" RegGroup="All" ConditionName="Structure" Operation="NotEqualTo" Value="" NextOperation="And">\"date\"((?![\s\S]*\"date\"+)[0-9a-zA-Z\s\:_\"\-\+\[\],.\(\){\/\\\-\=}\@!#$%^&amp;*]+)</Condition>
<Condition FieldPosition="1" InValue="{Condition:Structure}" RegGroup="All" ConditionName="Date" Operation="NotEqualTo" Value="" NextOperation="And">\:\s\"([a-zA-Z]{3}\s[a-zA-Z]{3}\s[0-9]{1,2}\s[0-9]{4}\s[0-9]{1,2}\:[0-9]{1,2}\:[0-9]{1,2}\s[A-Z]{1,3}\+[0-9]{1,4}\s\([a-zA-Z]{1,3}\))\"</Condition>
<Condition FieldPosition="1" InValue="{Condition:Structure}" RegGroup="All" ConditionName="ServersStructure" Operation="NotEqualTo" Value="" NextOperation="And">(\"servers\"\: \[\s+\{[0-9a-zA-Z\s\:_\"\-\+\[\],.\\\/\={\}\(\)]+)\"dbhosts\"</Condition>
<Condition FieldPosition="1" InValue="{Condition:Structure}" RegGroup="AllResult" ConditionName="Date" Operation="NotEqualTo" Value="" NextOperation="And">\:\s\"([a-zA-Z]{3}\s[a-zA-Z]{3}\s[0-9]{1,2}\s[0-9]{4}\s[0-9]{1,2}\:[0-9]{1,2}\:[0-9]{1,2}.*\s[A-Za-z0-9]+\))\"</Condition>
<Condition FieldPosition="1" InValue="{Condition:Structure}" RegGroup="AllResult" ConditionName="ServersStructure" Operation="NotEqualTo" Value="" NextOperation="And">(\"servers\"\: \[\s+\{[0-9a-zA-Z\s\:_\"\-\+\[\],.\\\/\={\}\(\)]+)\"dbhosts\"</Condition>
<Condition FieldPosition="1" InValue="{Condition:ServersStructure}" RegGroup="AllResult" ConditionName="ServerName" Operation="NotEqualTo" Value="" NextOperation="And">\"name\":\s\"([a-zA-Z0-9.\-_ ]+)\"\,\W+\"type\":\s\"([a-zA-Z0-9.\-_]+)\"</Condition>
<Condition FieldPosition="2" InValue="{Condition:ServersStructure}" RegGroup="AllResult" ConditionName="ServerType" Operation="NotEqualTo" Value="" NextOperation="And">\"name\":\s\"([a-zA-Z0-9.\-_ ]+)\"\,\W+\"type\":\s\"([a-zA-Z0-9.\-_]+)\"</Condition>
<Condition FieldPosition="1" InValue="{Condition:ServersStructure}" RegGroup="AllResult" ConditionName="IPAddress" Operation="NotEqualTo" Value="" NextOperation="And">\"ipAddress\":\s\"([a-zA-Z0-9.\-_]+)\"</Condition>
Expand Down Expand Up @@ -489,4 +489,26 @@
<Condition FieldPosition="4" RegGroup="All" ConditionName="Error" Operation="NotEqualTo" Value=""/>
</Expression>
</Formula>

<Formula Categories="configuration" SortCondition="Timestamp" SortMethod="Descending">
<Name>{Condition:SectionName} Upgrade/CLI commands</Name>
<DisplayMessage>{Condition:SectionName} {Condition:Timestamp}||{Condition:UpgradeMsg}</DisplayMessage>
<Expression LogLevel="Info">
<Section Name="messages" Wildcard="True" XPath="/Root/text()"/>
<RegExp>\b([0-9-]+ [0-9:.]+) [0-9a-zA-Z_\.\-]+\: cnlog\[[0-9]+\]\: \[[0-9a-fA-Fx]+\] INFO (upgrade|cli) (.*)\b</RegExp>
<Condition FieldPosition="1" RegGroup="All" ConditionName="Timestamp" Operation="NotEqualTo" Value="" NextOperation="And"/>
<Condition FieldPosition="3" RegGroup="All" ConditionName="UpgradeMsg" Operation="NotEqualTo" Value=""/>
</Expression>
<Expression LogLevel="Info" ExpressionGroup="ResetSearch">
<Section Name="var/log/informix/online.log" Wildcard="True" XPath="/Root/text()"/>
<RegExp>([0-9]{1,2}\/[0-9]{1,2}\/[0-9]{1,2} [0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2})\s\s(Conversion from version [a-zA-Z0-9.]+ Started|\(5\) connection rejected - no calls allowed for sqlexec|Cannot convert server.(.*)|The Master Daemon Died|oninit\:(.*))</RegExp>
<Condition FieldPosition="1" RegGroup="AllResult" ConditionName="TempTimestamp" Operation="NotEqualTo" Value="" NextOperation="And"/>
<Condition FieldPosition="1" RegGroup="AllResult" InValue="{Condition:TempTimestamp}" ConditionName="Month" Operation="NotEqualTo" Value="" NextOperation="And">\b([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{1,2}) ([0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2})\b</Condition>
<Condition FieldPosition="2" RegGroup="AllResult" InValue="{Condition:TempTimestamp}" ConditionName="Day" Operation="NotEqualTo" Value="" NextOperation="And">\b([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{1,2}) ([0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2})\b</Condition>
<Condition FieldPosition="3" RegGroup="AllResult" InValue="{Condition:TempTimestamp}" ConditionName="Year" Operation="NotEqualTo" Value="" NextOperation="And">\b([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{1,2}) ([0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2})\b</Condition>
<Condition FieldPosition="4" RegGroup="AllResult" InValue="{Condition:TempTimestamp}" ConditionName="Time" Operation="NotEqualTo" Value="" NextOperation="And">\b([0-9]{1,2})\/([0-9]{1,2})\/([0-9]{1,2}) ([0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2})\b</Condition>
<Condition FieldPosition="-2" RegGroup="AllResult" InValue="20{Condition:Year}-{Condition:Month}-{Condition:Day} {Condition:Time}" ConditionName="Timestamp" Operation="Append" Value="" NextOperation="And"/>
<Condition FieldPosition="2" RegGroup="AllResult" ConditionName="UpgradeMsg" Operation="NotEqualTo" Value=""/>
</Expression>
</Formula>
</Analytics>
11 changes: 7 additions & 4 deletions framework/src/com/ibm/datapower/er/Analytics/AnalyticsFunctions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@
package com.ibm.datapower.er.Analytics;

import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
Expand Down Expand Up @@ -195,10 +197,11 @@ public static String generateFileFromContent(DocumentSection section) {
}
if (nl != null && nl.getLength() > 0) {
try {
FileOutputStream fso = new FileOutputStream(dstFile);
OutputStream s = Files.newOutputStream(dstFile.toPath(), StandardOpenOption.CREATE_NEW,
StandardOpenOption.WRITE);
byte[] data = nl.item(0).getTextContent().getBytes();
fso.write(data, 0, data.length);
fso.close();
s.write(data, 0, data.length);
s.close();
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
Expand Down
59 changes: 29 additions & 30 deletions framework/src/com/ibm/datapower/er/PartsProcessorsHTML.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,10 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.nio.file.Files;
import java.nio.file.StandardOpenOption;
import java.util.Properties;

import com.ibm.datapower.er.mgmt.Base64;
Expand Down Expand Up @@ -152,36 +155,32 @@ public static String binaryBody(IPartInfo mimePart, PrintWriter writer, String o
extension = ".pcap";
}

File temp = File.createTempFile(filename, extension, directory);
String fullfile = temp.getAbsolutePath();
try
{
FileOutputStream fos = new FileOutputStream(fullfile);
BufferedReader bis = new BufferedReader(new InputStreamReader(in));
String sep = System.getProperty("line.separator");

while((nextLine = bis.readLine()) != null){
if ( !base64Encode ) // if base64Encode is true we leave the base64 encoding, otherwise we remove it
{
byte[] bytes = Base64.Decode(nextLine);
fos.write(bytes);

if ( lineReturn )
fos.write(sep.getBytes());
}
else
{
fos.write(nextLine.getBytes());

if ( lineReturn )
fos.write(sep.getBytes());
}
}
fos.close();
}catch(Exception ex)
{
temp.delete();
}
File temp = File.createTempFile(filename, extension, directory);
try {

OutputStream s = Files.newOutputStream(temp.toPath(), StandardOpenOption.APPEND);
BufferedReader bis = new BufferedReader(new InputStreamReader(in));
String sep = System.getProperty("line.separator");

while ((nextLine = bis.readLine()) != null) {
if (!base64Encode) // if base64Encode is true we leave the base64 encoding, otherwise we remove it
{
byte[] bytes = Base64.Decode(nextLine);
s.write(bytes, 0, bytes.length);

if (lineReturn)
s.write(sep.getBytes(), 0, sep.getBytes().length);
} else {
s.write(nextLine.getBytes(), 0, nextLine.getBytes().length);

if (lineReturn)
s.write(sep.getBytes(), 0, sep.getBytes().length);
}
}
s.close();
} catch (Exception ex) {
temp.delete();
}
//return the filename for the link
return temp.getName();
// return temp.getAbsolutePath();
Expand Down

0 comments on commit b9a98b1

Please sign in to comment.