Merge branch 'METRON-1572' into METRON-1533-NEW

apache · Jun 5, 2018 · 957a942 · 957a942
2 parents e60c3fc + 571e85d
commit 957a942
Show file tree

Hide file tree

Showing 114 changed files with 1,494 additions and 580 deletions.
diff --git a/KEYS b/KEYS
@@ -984,3 +984,62 @@ vkd/cbkuZuCthrOJq3gOfaRXS46EZ4rLAvoQKUiw1ZznjjYzI6ErJc1iD10B4/CD
 7wQCgib6Orvh57hVFTFWa116lpkU+rV3RUFC8RI=
 =MZyO
 -----END PGP PUBLIC KEY BLOCK-----
+pub   rsa4096 2018-05-15 [SC]
+      CEB9FD4AF1ABCFBF860538813CA8AFC41CEEA43A
+uid           [ultimate] Justin Leet (CODE SIGNING KEY) <leet@apache.org>
+sig 3        3CA8AFC41CEEA43A 2018-05-15  Justin Leet (CODE SIGNING KEY) <leet@apache.org>
+sub   rsa4096 2018-05-15 [E]
+sig          3CA8AFC41CEEA43A 2018-05-15  Justin Leet (CODE SIGNING KEY) <leet@apache.org>
+
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBFr7NJQBEADEvgaMw/ftxVxkymNyykvt3RRYitlyigOudIddBHMp6BCTCyol
+9sYQ3PfyeXQs85E4pEyIvJy2svRQZ18vvN/pVP2UKRjlbIgbDTSRwtt2DlUA31re
+DqLHm8JAACEe7KTDO4Nck3ALlrRerSjA5cOpa5fKQ7dgbaJQLl7sUxYaKekjIhFy
+sD4jv51C2ofVGPBjoMfWzOm9Ogmo1GyLRL3WlN/Tmzm3QJSKbCUay4hbu2X9KLCG
+oUwAj/imZbKIM6MNyi0DTG7nj+utnEieiYHrtx24jEC1nMzjbyb3niQpxRDhR7lf
+Kl/70fcOtBSooVRK0trntxIIoLyvmJ++Rm9riPzycvISnjuv2lH62/knErnL/Vkp
+SS/FpiHjwFTqoUIEH8AE5iYQAzGskpVlyZCAlrLKq+YzApVxVnJpiHHAnGxutdr9
+wcep3iu0mqiQok38jbojN2eYkhdmyQ2o5Tq2+l2p8zhefADGn5LqgNjmiq4EYMoU
+hCuC0b3BrSASmnVib0uOwGbIX5iKbvzvm4tOKM/MPf23K7LBi7Fj3/eY06czEExB
+HU0ufwGyMuHpas+OuOoq36pzxHtomemmNZwXqyq7g4Wvmr1s/vKpgVYMUluX3yyb
+Xr/a1YpD70Kwmb7N0DX1vmW9pVszoPhFOI7LXFZ7mIpFBVuCZvSr0ghIQQARAQAB
+tDBKdXN0aW4gTGVldCAoQ09ERSBTSUdOSU5HIEtFWSkgPGxlZXRAYXBhY2hlLm9y
+Zz6JAk4EEwEKADgWIQTOuf1K8avPv4YFOIE8qK/EHO6kOgUCWvs0lAIbAwULCQgH
+AwUVCgkICwUWAgMBAAIeAQIXgAAKCRA8qK/EHO6kOotbD/4jP0ITIgkImYE+T84F
+DSPOyjWQ7pAyrK3hBledUqjsM8CaW9ouo7BCHPDi6Ty/5QZ6YMc/YcU39SvQ46pl
+J3UO0/E+fDh2qjoOuLibq8MQp5gFE9X97VRYpfH9nTsy3fpuWXLRZAJR3KT5Nutu
+RAZSzzPBLtNZCliO6J4ozh/SGztFtj18NKgxXzDUn8cCSmBnpmzYpOeoS11hiGmj
+0NKvM1sw3KlIUXgfbZe+aphg1R7TqtUNcnHLtWE+qPHT0tEDMrJ2+/jLvpgEnzmf
+iARQBI7z9MhW24vSOPQOpvQ8i3u2s1UMeiee0vOlIPkNfdJVY2ligBcgL+fePjvT
+GwxArv1TlJTYGjklJwo7WBq1sya6w1rY4A1TwBMs8QPRL9jIE8W931KAEzI75R9k
+HpuhVQcgdahVMf5rX2SLGLh+OjY/xykosgMSr1FM/d35JAbDtE5+NklUPwuYFdXj
+57O4FMANSjUhg2ACq6wrslHXDUeTqRoNHEko3TwdHGt0YZuMVSPfN7SG0ixs+8ZR
+m/GcSoUmoj7lEPtEJ0oP6yGd2oRDVrBbpLlnLuJxfBbAnyLSKs/sCytTEHewnLlS
+Iewf1+v6a8SeCfxBEDD+BcyAYQlW1oH5lcDnoRQhaE9VC0OUi/JoJmMlRKl1i7IB
+9iqOaMOZ5bljcY1SGjyDJoNmm7kCDQRa+zSUARAA4xEBFNERKKKqvVtnnyPA1Bst
+tsYxnculEQg0mDGmdHO31cL5sMuxcUHxUWuC1Siemck2upzFqPfB2fDA6I5kTf5F
+/mX6eoqKvNA+fxWVup+qK866RIA7oSkUxEstCb/XeQgfbelw/g2W7FCazLvUDdaz
+TniZnxrJWuURtZ9kw0SGMwtG7kkKILr0iF+NTA0yo35KkBJNup7ge0Si6n0ZGgf5
+FQeTbqHFgb08BDtd181gyk3Otb87mBuvPxtKM2kZ1tpWwxXGV51LaVp3+xtk1VVU
+kCzV9j0h3hyCP0APuHnkh5HViUiVSKd6G1TM3b0CVDEEKvExAeaP5c9/mFhSyrYo
+w/iN8Ki8uT56fu0pfGG8Ld+zh/MwsfJccxMhnYgq9bgdMsCBmtagepWhGrYQ+Oj3
++CXHIK+lui8Z+KTGe4TAqBVscb9Qlw2FrCzukFIXZc2luG1Ugu3qI2IuoTc/c+LZ
+v9z0GSK8MeZ2hQ0nUTcIIJSID7ElHOxjEh+1Z2hlh9Z7VJMcoSZ8dbjElXp5TAPQ
+237BuasmqtH1XxHRCqCazDZYEibJRMEmVVpkEI8AElcbTHkbMbXF6PR24IU3Lu8e
+p8Q3XCP8jKSlI4E+nAceNxKnmrSLPEhdDgWXgTXGoz4+yDJRRaH1v/M7Pwm7K2Cg
+pMB9pp5dGsU7JIJLE0sAEQEAAYkCNgQYAQoAIBYhBM65/Urxq8+/hgU4gTyor8Qc
+7qQ6BQJa+zSUAhsMAAoJEDyor8Qc7qQ6KXYP/0SeNE+Y/0Wk77diUH8xUcwAErmu
+zeyfsfNuWJ1qN0kqrlwK5Q5aGSnVkLGDn8frKtxVEEqnPYBu00o4b2xn5Wx2mjDJ
+Uhd9+jHZLE2xFoYu2MAZ7Latudl8YayAiPM/OwriUlNfrs3Kw2uXccEdfXGWUZXv
+r+tbh1XvWgivd6jZg0pZhFT1E4kQZ3xbD2KDQylSnh/rsX7Pdov+AbFCqNtQCtfa
+wLok3qMOOaY4SKjmrqo+RkuxssWnnu9rfx44WnNe+0/+p2JXmFsWxoUE0NhVgBsR
+S9ngFZCUtNrbYoOzIQCrW5PP40dVQ8O6Z0ogBTxpHhYhhJxppMD+D0Z2SmUywX6S
+C8YQC1J5a7fy95kCZUJtqN2E8cfs5Fe8APV8od7g6pCBkl4W5U6FDmxzeMc4/fnL
+2tJZlf6/F2sFp8NpZy5xFFhfYQTx29aotqI8ZYL+Ap/NrcxgJHVlzhhPWaDQPNvG
+voLQ+ML+V7mIoQHdopQFxByUhTncvkf/GKH/g19jaPOkZrDpWK4NtpUHKy+liZ1V
+YAiuKTxjtA+/Yp60CluyauFX2SFFQJmxlyArbAuUXT59QVKvZvsXJc4PLXwTlBj+
+GOiGOVqmwEZRnk5qI4WO51lM2MJ0VfNBpZGPAI+KNUTYdBQhVBnWy1tf8bS6W5NY
+hjlTBT6jB6Df3Vma
+=FS7f
+-----END PGP PUBLIC KEY BLOCK-----
diff --git a/Upgrading.md b/Upgrading.md
@@ -19,7 +19,7 @@ limitations under the License.
 This document constitutes a per-version listing of changes of
 configuration which are non-backwards compatible.
 
-## 0.4.2 to 0.4.3
+## 0.4.2 to 0.5.0
 
 ### [METRON-941: native PaloAlto parser corrupts message when having a comma in the payload](https://issues.apache.org/jira/browse/METRON-941)
 While modifying the PaloAlto log parser to support logs from newer

diff --git a/dev-utilities/release-utils/metron-rc-check b/dev-utilities/release-utils/metron-rc-check
@@ -22,7 +22,7 @@ function help {
   echo "usage: ${0}"
   echo "    -v/--version=<version>   The version of the metron release. [Required]"
   echo "    -c/--candidate=<RC#>      Defines the Release Candidate. [Required]"
-  echo "    -b/--bro=<bro version>   The version of the bro kafka plugin. [Required]"
+  echo "    -b/--bro=<bro version>   The version of the bro kafka plugin. [Optional]"
   echo "    -h/--help                Usage information."
   echo " "
   echo "example: "
@@ -119,21 +119,22 @@ else
   exit 1
 fi
 
-if [ -z "$BRO" ]; then
-	echo "Missing -b/--bro which is required"
-	exit 1
-fi
-
-if [[ "$BRO" =~ ^[0-9]{1,2}\.[0-9]{1,2}\.[0-9]{1,2} ]]; then
-  BRO_VERSION="$BRO"
-else
-  echo "[ERROR] $BRO may not be a valid version number"
-  exit 1
+# validating the bro plugin is not required
+if [ -n "$BRO" ]; then
+  if [[ "$BRO" =~ ^[0-9]{1,2}\.[0-9]{1,2}\.[0-9]{1,2} ]]; then
+    BRO_VERSION="$BRO"
+  else
+    echo "[ERROR] $BRO may not be a valid version number"
+    exit 1
+  fi
 fi
 
 echo "Metron Version $METRON_VERSION"
 echo "Release Candidate $RC"
-echo "Bro Plugin Version $BRO_VERSION"
+
+if [ -n "$BRO" ]; then
+  echo "Bro Plugin Version $BRO_VERSION"
+fi
 
 METRON_RC_DIST="$METRON_DIST$METRON_VERSION-$UPPER_RC"
 echo "Metron RC Distribution Root is $METRON_RC_DIST"
@@ -158,8 +159,7 @@ echo "Working directory $WORK"
 KEYS="$METRON_RC_DIST/KEYS"
 METRON_ASSEMBLY="$METRON_RC_DIST/apache-metron-$METRON_VERSION-$RC.tar.gz"
 METRON_ASSEMBLY_SIG="$METRON_ASSEMBLY.asc"
-METRON_KAFKA_BRO_ASSEMBLY="$METRON_RC_DIST/apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz"
-METRON_KAFKA_BRO_ASSEMBLY_ASC="$METRON_KAFKA_BRO_ASSEMBLY.asc"
+
 
 echo "Downloading $KEYS"
 if ! wget -P "$WORK" "$KEYS" ; then
@@ -172,20 +172,29 @@ if ! wget -P "$WORK" "$METRON_ASSEMBLY" ; then
   echo "[ERROR] Failed to download $METRON_ASSEMBLY"
   exit 1
 fi
+
 echo "Downloading $METRON_ASSEMBLY_SIG"
 if ! wget -P "$WORK" "$METRON_ASSEMBLY_SIG" ; then
   echo "[ERROR] Failed to download $METRON_ASSEMBLY_SIG"
   exit 1
 fi
-echo "Downloading $METRON_KAFKA_BRO_ASSEMBLY"
-if ! wget -P "$WORK" "$METRON_KAFKA_BRO_ASSEMBLY" ; then
-  echo "[ERROR] Failed to download $METRON_KAFKA_BRO_ASSEMBLY"
-  exit 1
-fi
-echo "Downloading $METRON_KAFKA_BRO_ASSEMBLY_ASC"
-if ! wget -P "$WORK" "$METRON_KAFKA_BRO_ASSEMBLY_ASC" ; then
-  echo "[ERROR] Failed to download $METRON_KAFKA_BRO_ASSEMBLY_ASC"
-  exit 1
+
+if [ -n "$BRO" ]; then
+
+  METRON_KAFKA_BRO_ASSEMBLY="$METRON_RC_DIST/apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz"
+  METRON_KAFKA_BRO_ASSEMBLY_ASC="$METRON_KAFKA_BRO_ASSEMBLY.asc"
+
+  echo "Downloading $METRON_KAFKA_BRO_ASSEMBLY"
+  if ! wget -P "$WORK" "$METRON_KAFKA_BRO_ASSEMBLY" ; then
+    echo "[ERROR] Failed to download $METRON_KAFKA_BRO_ASSEMBLY"
+    exit 1
+  fi
+
+  echo "Downloading $METRON_KAFKA_BRO_ASSEMBLY_ASC"
+  if ! wget -P "$WORK" "$METRON_KAFKA_BRO_ASSEMBLY_ASC" ; then
+    echo "[ERROR] Failed to download $METRON_KAFKA_BRO_ASSEMBLY_ASC"
+    exit 1
+  fi
 fi
 
 cd "$WORK" || exit 1
@@ -202,10 +211,19 @@ if ! gpg --verify ./"apache-metron-$METRON_VERSION-$RC.tar.gz.asc" "apache-metro
   exit 1
 fi
 
-echo "Verifying Bro Kafka Plugin Assembly"
-if ! gpg --verify ./"apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz.asc" "apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz" ; then
-  echo "[ERROR] failed to verify Bro Kafka Plugin Assembly"
-  exit 1
+if [ -n "$BRO" ]; then
+
+  echo "Verifying Bro Kafka Plugin Assembly"
+  if ! gpg --verify ./"apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz.asc" "apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz" ; then
+    echo "[ERROR] failed to verify Bro Kafka Plugin Assembly"
+    exit 1
+  fi
+
+  if ! tar -xzf "apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz" ; then
+    echo "[ERROR] failed to unpack  Bro Kafka Plugin Assembly"
+    exit 1
+  fi
+
 fi
 
 echo "Unpacking Assemblies"
@@ -214,11 +232,6 @@ if ! tar -xzf "apache-metron-$METRON_VERSION-$RC.tar.gz" ; then
   exit 1
 fi
 
-if ! tar -xzf "apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz" ; then
-  echo "[ERROR] failed to unpack  Bro Kafka Plugin Assembly"
-  exit 1
-fi
-
 echo ""
 echo ""
 read -p "  run test suite [install, unit tests, integration tests, ui tests, licenses, rpm build]? [yN] " -n 1 -r

diff --git a/dev-utilities/release-utils/validate-jira-for-release b/dev-utilities/release-utils/validate-jira-for-release
@@ -137,35 +137,52 @@ for i in "$@"; do
   esac
 done
 
-WORKDIR="~/tmp"
 
 # ensure all required values are set
 if [ -z "$VERSION" ]; then
-	echo "Missing -v/--version is is required"
-	exit 1
+  help
+  echo "Missing -v/--version is is required"
+  exit 1
 fi
 if [ -z "$START" ]; then
-	echo "Missing -s/--start which is required"
-	exit 1
+  help
+  echo "Missing -s/--start which is required"
+  exit 1
 fi
 if [ -z "$END" ]; then
-	echo "Missing -e/--end which is required"
-	exit 1
+  help
+  echo "Missing -e/--end which is required"
+  exit 1
 fi
 if [ -z "$REPO" ]; then
-	echo "Missing -r/--repo which is required"
-	exit 1
+  help
+  echo "Missing -r/--repo which is required"
+  exit 1
 fi
 if [ -z "$BRANCH" ]; then
-	echo "Missing -b/--branch which is required"
-	exit 1
+  help
+  echo "Missing -b/--branch which is required"
+  exit 1
 fi
 
-# clone the metron repo and fetch all tags
-git clone $REPO "metron-$VERSION"
+TMPDIR="$HOME/tmp"
+WORKDIR="$TMPDIR/metron-$VERSION"
+
+# warn the user if the working directory exists
+if [ -d "$WORKDIR" ]; then
+  read -p "  directory exists [$WORKDIR].  overwrite existing repo? [yN] " -n 1 -r
+  echo
+  if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+    exit 1
+  fi
+fi
+
+# fetch the repo and all tags
+rm -rf "$WORKDIR"
+git clone $REPO "$WORKDIR"
+cd "$WORKDIR"
 git checkout $BRANCH
-cd "$WORKDIR/metron-$VERSION"
-git fetch --all --tags
+git fetch --tags
 
 # find all JIRAs that have been committed since the last release
 GET_JIRAS="git log $START..$END --oneline | grep -E -o 'METRON[- ]*[0-9]+'"
@@ -177,21 +194,21 @@ printf "$FORMAT_STR" "JIRA" "STATUS" "FIX VERSION" "ASSIGNEE" "FIX"
 # for each JIRA since the last release tag...
 eval $GET_JIRAS | while read JIRA ; do
 
-	# fetch the JIRA content
-	URL="https://issues.apache.org/jira/si/jira.issueviews:issue-xml/$JIRA/$JIRA.xml"
-        CONTENT=`curl -s $URL`
+  # fetch the JIRA content
+  URL="https://issues.apache.org/jira/si/jira.issueviews:issue-xml/$JIRA/$JIRA.xml"
+  CONTENT=`curl -s $URL`
 
-	# painfully extract some fields
-	STATUS=`echo "$CONTENT" | grep "<status[^>]*>" | sed 's/^.*<status[^>]*>//' | sed 's/<.status>.*$//'`
-	ASSIGNEE=`echo "$CONTENT" | grep "<assignee[^>]*>" | sed 's/^.*<assignee[^>]*>//' | sed 's/<.assignee>.*$//'`
-	FIXV=`echo "$CONTENT" | grep "<fixVersion[^>]*>" | sed 's/^.*<fixVersion[^>]*>//' | sed 's/<.fixVersion>.*$//'`
+  # painfully extract some fields
+  STATUS=`echo "$CONTENT" | grep "<status[^>]*>" | sed 's/^.*<status[^>]*>//' | sed 's/<.status>.*$//'`
+  ASSIGNEE=`echo "$CONTENT" | grep "<assignee[^>]*>" | sed 's/^.*<assignee[^>]*>//' | sed 's/<.assignee>.*$//'`
+  FIXV=`echo "$CONTENT" | grep "<fixVersion[^>]*>" | sed 's/^.*<fixVersion[^>]*>//' | sed 's/<.fixVersion>.*$//'`
 
-	# the link is only populated, if there is something to fix
-	LINK=""
-	if [ "$FIXV" != "$VERSION" ] || [ "$STATUS" != "Done" ]; then
-		LINK="https://issues.apache.org/jira/browse/$JIRA"
-	fi
+  # the link is only populated, if there is something to fix
+  LINK=""
+  if [ "$FIXV" != "$VERSION" ] || [ "$STATUS" != "Done" ]; then
+    LINK="https://issues.apache.org/jira/browse/$JIRA"
+  fi
 
-	# show the JIRA
-      	printf "$FORMAT_STR" "$JIRA" "$STATUS" "$FIXV" "$ASSIGNEE" "$LINK"
+  # show the JIRA
+  printf "$FORMAT_STR" "$JIRA" "$STATUS" "$FIXV" "$ASSIGNEE" "$LINK"
 done
diff --git a/metron-analytics/metron-maas-common/pom.xml b/metron-analytics/metron-maas-common/pom.xml
@@ -18,7 +18,7 @@
   <parent>
     <groupId>org.apache.metron</groupId>
     <artifactId>metron-analytics</artifactId>
-    <version>0.4.3</version>
+    <version>0.5.0</version>
   </parent>
   <artifactId>metron-maas-common</artifactId>
   <url>https://metron.apache.org/</url>

diff --git a/metron-analytics/metron-maas-service/pom.xml b/metron-analytics/metron-maas-service/pom.xml
@@ -18,7 +18,7 @@
   <parent>
     <groupId>org.apache.metron</groupId>
     <artifactId>metron-analytics</artifactId>
-    <version>0.4.3</version>
+    <version>0.5.0</version>
   </parent>
   <artifactId>metron-maas-service</artifactId>
   <url>https://metron.apache.org/</url>

diff --git a/metron-analytics/metron-profiler-client/pom.xml b/metron-analytics/metron-profiler-client/pom.xml
@@ -18,7 +18,7 @@
     <parent>
         <groupId>org.apache.metron</groupId>
         <artifactId>metron-analytics</artifactId>
-        <version>0.4.3</version>
+        <version>0.5.0</version>
     </parent>
     <artifactId>metron-profiler-client</artifactId>
     <url>https://metron.apache.org/</url>

diff --git a/...ofiler-client/src/test/java/org/apache/metron/profiler/client/stellar/GetProfileTest.java b/...ofiler-client/src/test/java/org/apache/metron/profiler/client/stellar/GetProfileTest.java
@@ -24,6 +24,7 @@
 import org.apache.metron.hbase.mock.MockHBaseTableProvider;
 import org.apache.metron.profiler.client.ProfileWriter;
 import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
 import org.apache.metron.stellar.dsl.functions.resolver.SimpleFunctionResolver;
 import org.apache.metron.stellar.dsl.functions.resolver.SingletonFunctionResolver;
 import org.apache.metron.profiler.ProfileMeasurement;
@@ -266,7 +267,7 @@ public void testWithTwoGroups() {
   /**
    * Initialization should fail if the required context values are missing.
    */
-  @Test(expected = IllegalStateException.class)
+  @Test(expected = ParseException.class)
   public void testMissingContext() {
     Context empty = Context.EMPTY_CONTEXT();