From 58eff702b6e3f490d1110adcbbef087238aa3d91 Mon Sep 17 00:00:00 2001 From: merrimanr Date: Tue, 1 Aug 2017 15:10:10 -0500 Subject: [PATCH 1/2] initial commit --- .../docker/rpm-docker/SPECS/metron.spec | 3 + metron-interface/metron-config/.gitignore | 7 + metron-interface/metron-config/package.json | 10 +- .../metron-config/proxy.conf.json | 4 +- .../metron-config/src/app/_blockly.scss | 675 ++++++++++++++++++ .../metron-config/src/app/app.module.ts | 4 +- .../general-settings.module.ts | 3 +- .../app/model/stellar-function-description.ts | 4 +- .../sensor-parser-config.component.html | 2 +- .../sensor-blockly-editor.component.html | 52 ++ .../sensor-blockly-editor.component.scss | 83 +++ .../sensor-blockly-editor.component.ts | 81 +++ .../sensor-blockly-editor.module.ts | 32 + .../sensor-threat-triage.component.html | 6 + .../sensor-threat-triage.component.ts | 21 + .../sensor-threat-triage.module.ts | 4 +- .../src/app/service/blockly.service.ts | 39 + .../src/app/service/stellar.service.ts | 6 + .../blockly-editor.component.html | 21 + .../blockly-editor.component.scss | 41 ++ .../blockly-editor.component.spec.ts | 0 .../blockly-editor.component.ts | 203 ++++++ .../blockly-editor/blockly-editor.module.ts | 13 + .../src/assets/blockly/stellar_blocks.js | 287 ++++++++ .../src/assets/blockly/stellar_generator.js | 67 ++ metron-interface/metron-config/src/index.html | 6 + .../metron-config/src/styles.scss | 1 + .../model/StellarFunctionDescription.java | 56 +- metron-interface/metron-rest/pom.xml | 5 + .../rest/controller/BlocklyController.java | 48 ++ .../rest/controller/StellarController.java | 7 + .../metron/rest/service/BlocklyService.java | 25 + .../metron/rest/service/StellarService.java | 2 + .../rest/service/impl/StellarServiceImpl.java | 56 +- .../rest/service/impl/blockly/Block.java | 131 ++++ .../service/impl/blockly/BlocklyCompiler.java | 366 ++++++++++ .../impl/blockly/BlocklyServiceImpl.java | 91 +++ .../rest/service/impl/blockly/Field.java | 59 ++ .../rest/service/impl/blockly/Mutation.java | 42 ++ .../rest/service/impl/blockly/Value.java | 64 ++ .../metron/rest/service/impl/blockly/Xml.java | 72 ++ .../impl/blockly/BlocklyServiceImplTest.java | 83 +++ .../stellar/common/StellarCompiler.java | 2 +- 43 files changed, 2743 insertions(+), 41 deletions(-) create mode 100644 metron-interface/metron-config/src/app/_blockly.scss create mode 100644 metron-interface/metron-config/src/app/sensors/sensor-threat-triage/rule-blockly/sensor-blockly-editor.component.html create mode 100644 metron-interface/metron-config/src/app/sensors/sensor-threat-triage/rule-blockly/sensor-blockly-editor.component.scss create mode 100644 metron-interface/metron-config/src/app/sensors/sensor-threat-triage/rule-blockly/sensor-blockly-editor.component.ts create mode 100644 metron-interface/metron-config/src/app/sensors/sensor-threat-triage/rule-blockly/sensor-blockly-editor.module.ts create mode 100644 metron-interface/metron-config/src/app/service/blockly.service.ts create mode 100644 metron-interface/metron-config/src/app/shared/blockly-editor/blockly-editor.component.html create mode 100644 metron-interface/metron-config/src/app/shared/blockly-editor/blockly-editor.component.scss create mode 100644 metron-interface/metron-config/src/app/shared/blockly-editor/blockly-editor.component.spec.ts create mode 100644 metron-interface/metron-config/src/app/shared/blockly-editor/blockly-editor.component.ts create mode 100644 metron-interface/metron-config/src/app/shared/blockly-editor/blockly-editor.module.ts create mode 100644 metron-interface/metron-config/src/assets/blockly/stellar_blocks.js create mode 100644 metron-interface/metron-config/src/assets/blockly/stellar_generator.js create mode 100644 metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/BlocklyController.java create mode 100644 metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/BlocklyService.java create mode 100644 metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/Block.java create mode 100644 metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/BlocklyCompiler.java create mode 100644 metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/BlocklyServiceImpl.java create mode 100644 metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/Field.java create mode 100644 metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/Mutation.java create mode 100644 metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/Value.java create mode 100644 metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/Xml.java create mode 100644 metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/blockly/BlocklyServiceImplTest.java diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec index 94c7e05207..f3f427769d 100644 --- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec +++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec @@ -413,6 +413,9 @@ This package installs the Metron Management UI %{metron_home} %attr(0644,root,root) %{metron_home}/web/management-ui/assets/ace/*.js %attr(0644,root,root) %{metron_home}/web/management-ui/assets/ace/LICENSE %attr(0644,root,root) %{metron_home}/web/management-ui/assets/ace/snippets/*.js +%attr(0644,root,root) %{metron_home}/web/management-ui/assets/blockly/*.js +%attr(0644,root,root) %{metron_home}/web/management-ui/assets/blockly/media/* +%attr(0644,root,root) %{metron_home}/web/management-ui/assets/blockly/msg/js/*.js %attr(0644,root,root) %{metron_home}/web/management-ui/assets/fonts/Roboto/LICENSE.txt %attr(0644,root,root) %{metron_home}/web/management-ui/assets/fonts/Roboto/*.ttf %attr(0644,root,root) %{metron_home}/web/management-ui/assets/images/* diff --git a/metron-interface/metron-config/.gitignore b/metron-interface/metron-config/.gitignore index 4be186c4cc..16eff9a25b 100644 --- a/metron-interface/metron-config/.gitignore +++ b/metron-interface/metron-config/.gitignore @@ -37,6 +37,13 @@ testem.log /src/assets/ace/theme-monokai.js /src/assets/ace/worker-json.js +#blockly +/src/assets/blockly/media +/src/assets/blockly/msg +/src/assets/blockly/blockly_compressed.js +/src/assets/blockly/blocks_compressed.js +/src/assets/blockly/javascript_compressed.js + #System Files .DS_Store Thumbs.db diff --git a/metron-interface/metron-config/package.json b/metron-interface/metron-config/package.json index 393e492f36..d662075db4 100644 --- a/metron-interface/metron-config/package.json +++ b/metron-interface/metron-config/package.json @@ -3,7 +3,8 @@ "version": "0.4.1", "license": "Apache 2.0", "config": { - "node_ace": "node_modules/ace-builds/src-min-noconflict/" + "node_ace": "node_modules/ace-builds/src-min-noconflict/", + "node_blockly": "node_modules/blockly" }, "angular-cli": {}, "scripts": { @@ -16,6 +17,9 @@ "e2e-all": "./node_modules/.bin/protractor --suite=all", "copy-ace": "cp -f $npm_package_config_node_ace/ext-language_tools.js $npm_package_config_node_ace/mode-json.js $npm_package_config_node_ace/theme-monokai.js $npm_package_config_node_ace/worker-json.js src/assets/ace", "copy-ace-snippets": "cp -f $npm_package_config_node_ace/snippets/text.js $npm_package_config_node_ace/snippets/json.js src/assets/ace/snippets", + "copy-blockly": "cp -f $npm_package_config_node_blockly/blockly_compressed.js $npm_package_config_node_blockly/blocks_compressed.js $npm_package_config_node_blockly/javascript_compressed.js src/assets/blockly", + "copy-blockly-msg": "mkdir -p src/assets/blockly/msg/js && cp -f $npm_package_config_node_blockly/msg/js/en.js src/assets/blockly/msg/js", + "copy-blockly-media": "mkdir -p src/assets/blockly/media && cp -f $npm_package_config_node_blockly/media/* src/assets/blockly/media", "postinstall": "npm run copy-ace & npm run copy-ace-snippets" }, "private": true, @@ -60,6 +64,8 @@ "protractor": "4.0.5", "ts-node": "1.2.1", "tslint": "3.13.0", - "typescript": "~2.0.3" + "typescript": "~2.0.3", + "blockly": "^1.0.0" } } + diff --git a/metron-interface/metron-config/proxy.conf.json b/metron-interface/metron-config/proxy.conf.json index 29466ccc00..612bd67420 100644 --- a/metron-interface/metron-config/proxy.conf.json +++ b/metron-interface/metron-config/proxy.conf.json @@ -1,10 +1,10 @@ { "/api/v1": { - "target": "http://localhost:8080", + "target": "http://node1:8082", "secure": false }, "/logout": { - "target": "http://localhost:8080", + "target": "http://node1:8082", "secure": false } } diff --git a/metron-interface/metron-config/src/app/_blockly.scss b/metron-interface/metron-config/src/app/_blockly.scss new file mode 100644 index 0000000000..6f5afe3041 --- /dev/null +++ b/metron-interface/metron-config/src/app/_blockly.scss @@ -0,0 +1,675 @@ + +//.blocklyDiv +//{ +// width: 400px; +//} + +.injectionDiv +{ + height: 100%; + position: relative; + background-color: #0b4451; +} + +.blocklySvg { + background-color: #333333; + outline: none; + overflow: hidden; /* IE overflows by default. */ + display: block; +} + +.blocklyWidgetDiv { + display: none; + position: absolute; + z-index: 99999; /* big value for bootstrap3 compatibility */ +} + + + +.blocklyNonSelectable { + user-select: none; + -moz-user-select: none; + -webkit-user-select: none; + -ms-user-select: none; +} + +.blocklyTooltipDiv { + background-color: #ffffc7; + border: 1px solid #ddc; + //box-shadow: 4px 4px 20px 1px rgba(000.15); + color: #000; + display: none; + font-family: sans-serif; + font-size: 9pt; + opacity: 0.9; + padding: 2px; + position: absolute; + z-index: 100000; /* big value for bootstrap3 compatibility */ +} + +.blocklyResizeSE { + cursor: se-resize; + fill: #aaa; +} + +.blocklyResizeSW { + cursor: sw-resize; + fill: #aaa; +} + +.blocklyResizeLine { + stroke: #888; + stroke-width: 1; +} + +.blocklyHighlightedConnectionPath { + fill: none; + stroke: #fc3; + stroke-width: 4px; +} + +.blocklyPathLight { + fill: none; + stroke-linecap: round; + stroke-width: 1; +} + +.blocklySelected>.blocklyPath { + stroke: #fc3; + stroke-width: 3px; +} + +.blocklySelected>.blocklyPathLight { + display: none; +} + +.blocklyDragging>.blocklyPath +.blocklyDragging>.blocklyPathLight { + fill-opacity: .8; + stroke-opacity: .8; +} + +.blocklyDragging>.blocklyPathDark { + display: none; +} + +.blocklyDisabled>.blocklyPath { + fill-opacity: .5; + stroke-opacity: .5; +} + +.blocklyDisabled>.blocklyPathLight +.blocklyDisabled>.blocklyPathDark { + display: none; +} + +.blocklyText { + cursor: default; + fill: #fff; + font-family: sans-serif; + font-size: 11pt; +} + +.blocklyNonEditableText>text { + pointer-events: none; +} + +.blocklyNonEditableText>rect +.blocklyEditableText>rect { + fill: #fff; + fill-opacity: .6; +} + +.blocklyNonEditableText>text +.blocklyEditableText>text { + fill: #000; +} + +.blocklyEditableText:hover>rect { + stroke: #fff; + stroke-width: 2; +} + +.blocklyBubbleText { + fill: #000; +} + +.blocklyFlyoutButton { + fill: #888; + cursor: default; +} + +.blocklyFlyoutButtonShadow { + fill: #666; +} + +.blocklyFlyoutButton:hover { + fill: #aaa; +} + +.blocklyFlyoutLabel { + cursor: default; +} + +.blocklyFlyoutLabelBackground { + opacity: 0; +} + +.blocklyFlyoutLabelText { + fill: #000; +} + +.blocklyFlyoutLabelText:hover { + fill: #aaa; +} + +/* + Dont allow users to select text. It gets annoying when trying to + drag a block and selected text moves instead. +*/ +.blocklySvg text { + user-select: none; + -moz-user-select: none; + -webkit-user-select: none; + cursor: inherit; +} + +.blocklyHidden { + display: none; +} + +.blocklyFieldDropdown:not(.blocklyHidden) { + display: block; +} + +.blocklyIconGroup { + cursor: default; +} + +.blocklyIconGroup:not(:hover) +.blocklyIconGroupReadonly { + opacity: .6; +} + +.blocklyIconShape { + fill: #00f; + stroke: #fff; + stroke-width: 1px; +} + +.blocklyIconSymbol { + fill: #fff; +} + +.blocklyMinimalBody { + margin: 0; + padding: 0; +} + +.blocklyCommentTextarea { + background-color: #ffc; + border: 0; + margin: 0; + padding: 2px; + resize: none; +} + +.blocklyHtmlInput { + border: none; + border-radius: 4px; + font-family: sans-serif; + height: 100%; + margin: 0; + outline: none; + padding: 0 1px; + width: 100% +} + +.blocklyMainBackground { + stroke-width: 2; + stroke: #4d4d4d; /* Equates to #ddd due to border being off-pixel. */ +} + +.blocklyMutatorBackground { + fill: #fff; + stroke: #ddd; + stroke-width: 1; +} + +.blocklyFlyoutBackground { + fill: #ddd; + fill-opacity: .8; +} + +.blocklyScrollbarBackground { + opacity: 0; +} + +.blocklyScrollbarHandle { + fill: #ccc; +} + +.blocklyScrollbarBackground:hover+.blocklyScrollbarHandle +.blocklyScrollbarHandle:hover { + fill: #bbb; +} + +.blocklyZoom>image { + opacity: .4; +} + +.blocklyZoom>image:hover { + opacity: .6; +} + +.blocklyZoom>image:active { + opacity: .8; +} + +/* Darken flyout scrollbars due to being on a grey background. */ +/* By contrast workspace scrollbars are on a white background. */ +.blocklyFlyout .blocklyScrollbarHandle { + fill: #bbb; +} + +.blocklyFlyout .blocklyScrollbarBackground:hover+.blocklyScrollbarHandle +.blocklyFlyout .blocklyScrollbarHandle:hover { + fill: #aaa; +} + +.blocklyInvalidInput { + background: #faa; +} + +.blocklyAngleCircle { + stroke: #444; + stroke-width: 1; + fill: #ddd; + fill-opacity: .8; +} + +.blocklyAngleMarks { + stroke: #444; + stroke-width: 1; +} + +.blocklyAngleGauge { + fill: #f88; + fill-opacity: .8; +} + +.blocklyAngleLine { + stroke: #f00; + stroke-width: 2; + stroke-linecap: round; +} + +.blocklyContextMenu { + border-radius: 4px; +} + +.blocklyDropdownMenu { + padding: 0 !important; +} + +/* Override the default Closure URL. */ +.blocklyWidgetDiv .goog-option-selected .goog-menuitem-checkbox +.blocklyWidgetDiv .goog-option-selected .goog-menuitem-icon { + background: url(/assets/blockly/media/sprites.png) no-repeat -48px -16px !important; +} + +/* Category tree in Toolbox. */ +.blocklyToolboxDiv { + background-color: #0b4451; + overflow-x: visible; + overflow-y: auto; + position: absolute; +} + +.blocklyTreeRoot { + padding: 4px 0; +} + +.blocklyTreeRoot:focus { + outline: none; +} + +.blocklyTreeRow { + height: 22px; + line-height: 22px; + margin-bottom: 3px; + padding-right: 8px; + white-space: nowrap; +} + +.blocklyHorizontalTree { + float: left; + margin: 1px 5px 8px 0; +} + +.blocklyHorizontalTreeRtl { + float: right; + margin: 1px 0 8px 5px; +} + +.blocklyToolboxDiv[dir="RTL"] .blocklyTreeRow { + margin-left: 8px; +} + +.blocklyTreeRow:not(.blocklyTreeSelected):hover { + background-color: #e4e4e4; +} + +.blocklyTreeSeparator { + border-bottom: solid #e5e5e5 1px; + height: 0; + margin: 5px 0; +} + +.blocklyTreeSeparatorHorizontal { + border-right: solid #e5e5e5 1px; + width: 0; + padding: 5px 0; + margin: 0 5px; +} + + +.blocklyTreeIcon { + background-image: url(/assets/blockly/media/sprites.png); + height: 16px; + vertical-align: middle; + width: 16px; +} + +.blocklyTreeIconClosedLtr { + background-position: -32px -1px; +} + +.blocklyTreeIconClosedRtl { + background-position: 0px -1px; +} + +.blocklyTreeIconOpen { + background-position: -16px -1px; +} + +.blocklyTreeSelected>.blocklyTreeIconClosedLtr { + background-position: -32px -17px; +} + +.blocklyTreeSelected>.blocklyTreeIconClosedRtl { + background-position: 0px -17px; +} + +.blocklyTreeSelected>.blocklyTreeIconOpen { + background-position: -16px -17px; +} + +.blocklyTreeIconNone +.blocklyTreeSelected>.blocklyTreeIconNone { + background-position: -48px -1px; +} + +.blocklyTreeLabel { + cursor: default; + font-size: 14px; + padding: 0 3px; + vertical-align: middle; +} + +.blocklyTreeSelected .blocklyTreeLabel { + color: #fff; +} + +/* Copied from: goog/css/colorpicker-simplegrid.css */ +/* + * Copyright 2007 The Closure Library Authors. All Rights Reserved. + * + * Use of this source code is governed by the Apache License Version 2.0. + * See the COPYING file for details. + */ + +/* Author: pupius@google.com (Daniel Pupius) */ + +/* + Styles to make the colorpicker look like the old gmail color picker + NOTE: without CSS scoping this will override styles defined in palette.css +*/ +.blocklyWidgetDiv .goog-palette { + outline: none; + cursor: default; +} + +.blocklyWidgetDiv .goog-palette-table { + border: 1px solid #666; + border-collapse: collapse; +} + +.blocklyWidgetDiv .goog-palette-cell { + height: 13px; + width: 15px; + margin: 0; + border: 0; + text-align: center; + vertical-align: middle; + border-right: 1px solid #666; + font-size: 1px; +} + +.blocklyWidgetDiv .goog-palette-colorswatch { + position: relative; + height: 13px; + width: 15px; + border: 1px solid #666; +} + +.blocklyWidgetDiv .goog-palette-cell-hover .goog-palette-colorswatch { + border: 1px solid #FFF; +} + +.blocklyWidgetDiv .goog-palette-cell-selected .goog-palette-colorswatch { + border: 1px solid #000; + color: #fff; +} + +/* Copied from: goog/css/menu.css */ +/* + * Copyright 2009 The Closure Library Authors. All Rights Reserved. + * + * Use of this source code is governed by the Apache License Version 2.0. + * See the COPYING file for details. + */ + +/** + * Standard styling for menus created by goog.ui.MenuRenderer. + * + * @author attila@google.com (Attila Bodis) + */ + +.blocklyWidgetDiv .goog-menu { + background: #fff; + border-color: #ccc #666 #666 #ccc; + border-style: solid; + border-width: 1px; + cursor: default; + font: normal 13px Arial sans-serif; + margin: 0; + outline: none; + padding: 4px 0; + position: absolute; + overflow-y: auto; + overflow-x: hidden; + max-height: 100%; + z-index: 20000; /* Arbitrary but some apps depend on it... */ +} + +/* Copied from: goog/css/menuitem.css */ +/* + * Copyright 2009 The Closure Library Authors. All Rights Reserved. + * + * Use of this source code is governed by the Apache License Version 2.0. + * See the COPYING file for details. + */ + +/** + * Standard styling for menus created by goog.ui.MenuItemRenderer. + * + * @author attila@google.com (Attila Bodis) + */ + +/** + * State: resting. + * + * NOTE(mleibmanchrishenry): + * The RTL support in Closure is provided via two mechanisms -- "rtl" CSS + * classes and BiDi flipping done by the CSS compiler. Closure supports RTL + * with or without the use of the CSS compiler. In order for them not + * to conflict with each other the "rtl" CSS classes need to have the #noflip + * annotation. The non-rtl counterparts should ideally have them as well but + * since .goog-menuitem existed without .goog-menuitem-rtl for so long before + * being added there is a risk of people having templates where they are not + * rendering the .goog-menuitem-rtl class when in RTL and instead rely solely + * on the BiDi flipping by the CSS compiler. Thats why were not adding the + * #noflip to .goog-menuitem. + */ +.blocklyWidgetDiv .goog-menuitem { + color: #000; + font: normal 13px Arial sans-serif; + list-style: none; + margin: 0; + /* 28px on the left for icon or checkbox; 7em on the right for shortcut. */ + padding: 4px 7em 4px 28px; + white-space: nowrap; +} + +/* BiDi override for the resting state. */ +/* #noflip */ +.blocklyWidgetDiv .goog-menuitem.goog-menuitem-rtl { + /* Flip left/right padding for BiDi. */ + padding-left: 7em; + padding-right: 28px; +} + +/* If a menu doesnt have checkable items or items with icons remove padding. */ +.blocklyWidgetDiv .goog-menu-nocheckbox .goog-menuitem +.blocklyWidgetDiv .goog-menu-noicon .goog-menuitem { + padding-left: 12px; +} + +/* + * If a menu doesnt have items with shortcuts leave just enough room for + * submenu arrows if they are rendered. + */ +.blocklyWidgetDiv .goog-menu-noaccel .goog-menuitem { + padding-right: 20px; +} + +.blocklyWidgetDiv .goog-menuitem-content { + color: #000; + font: normal 13px Arial sans-serif; +} + +/* State: disabled. */ +.blocklyWidgetDiv .goog-menuitem-disabled .goog-menuitem-accel +.blocklyWidgetDiv .goog-menuitem-disabled .goog-menuitem-content { + color: #ccc !important; +} + +.blocklyWidgetDiv .goog-menuitem-disabled .goog-menuitem-icon { + opacity: 0.3; + -moz-opacity: 0.3; + filter: alpha(opacity=30); +} + +/* State: hover. */ +.blocklyWidgetDiv .goog-menuitem-highlight +.blocklyWidgetDiv .goog-menuitem-hover { + background-color: #d6e9f8; + /* Use an explicit top and bottom border so that the selection is visible + * in high contrast mode. */ + border-color: #d6e9f8; + border-style: dotted; + border-width: 1px 0; + padding-bottom: 3px; + padding-top: 3px; +} + +/* State: selected/checked. */ +.blocklyWidgetDiv .goog-menuitem-checkbox +.blocklyWidgetDiv .goog-menuitem-icon { + background-repeat: no-repeat; + height: 16px; + left: 6px; + position: absolute; + right: auto; + vertical-align: middle; + width: 16px; +} + +/* BiDi override for the selected/checked state. */ +/* #noflip */ +.blocklyWidgetDiv .goog-menuitem-rtl .goog-menuitem-checkbox +.blocklyWidgetDiv .goog-menuitem-rtl .goog-menuitem-icon { + /* Flip left/right positioning. */ + left: auto; + right: 6px; +} + +.blocklyWidgetDiv .goog-option-selected .goog-menuitem-checkbox +.blocklyWidgetDiv .goog-option-selected .goog-menuitem-icon { + /* Client apps may override the URL at which they serve the sprite. */ + background: url(//ssl.gstatic.com/editor/editortoolbar.png) no-repeat -512px 0; +} + +/* Keyboard shortcut ("accelerator") style. */ +.blocklyWidgetDiv .goog-menuitem-accel { + color: #999; + /* Keyboard shortcuts are untranslated; always left-to-right. */ + /* #noflip */ + direction: ltr; + left: auto; + padding: 0 6px; + position: absolute; + right: 0; + text-align: right; +} + +/* BiDi override for shortcut style. */ +/* #noflip */ +.blocklyWidgetDiv .goog-menuitem-rtl .goog-menuitem-accel { + /* Flip left/right positioning and text alignment. */ + left: 0; + right: auto; + text-align: left; +} + +/* Mnemonic styles. */ +.blocklyWidgetDiv .goog-menuitem-mnemonic-hint { + text-decoration: underline; +} + +.blocklyWidgetDiv .goog-menuitem-mnemonic-separator { + color: #999; + font-size: 12px; + padding-left: 4px; +} + +/* Copied from: goog/css/menuseparator.css */ +/* + * Copyright 2009 The Closure Library Authors. All Rights Reserved. + * + * Use of this source code is governed by the Apache License Version 2.0. + * See the COPYING file for details. + */ + +/** + * Standard styling for menus created by goog.ui.MenuSeparatorRenderer. + * + * @author attila@google.com (Attila Bodis) + */ + +.blocklyWidgetDiv .goog-menuseparator { + border-top: 1px solid #ccc; + margin: 4px 0; + padding: 0; +} \ No newline at end of file diff --git a/metron-interface/metron-config/src/app/app.module.ts b/metron-interface/metron-config/src/app/app.module.ts index 25e4acd339..68a76ac7b1 100644 --- a/metron-interface/metron-config/src/app/app.module.ts +++ b/metron-interface/metron-config/src/app/app.module.ts @@ -44,6 +44,7 @@ import {StormService} from './service/storm.service'; import {SensorParserConfigHistoryService} from './service/sensor-parser-config-history.service'; import {SensorIndexingConfigService} from './service/sensor-indexing-config.service'; import {HdfsService} from './service/hdfs.service'; +import {BlocklyService} from "./service/blockly.service"; @NgModule({ @@ -53,7 +54,8 @@ import {HdfsService} from './service/hdfs.service'; providers: [ AuthenticationService, AuthGuard, LoginGuard, SensorParserConfigService, SensorParserConfigHistoryService, SensorEnrichmentConfigService, SensorIndexingConfigService, StormService, KafkaService, GrokValidationService, StellarService, HdfsService, - GlobalConfigService, MetronAlerts, MetronDialogBox, appRoutingProviders, { provide: APP_CONFIG, useValue: METRON_REST_CONFIG }], + GlobalConfigService, MetronAlerts, MetronDialogBox, BlocklyService, + appRoutingProviders, { provide: APP_CONFIG, useValue: METRON_REST_CONFIG }], bootstrap: [ AppComponent ] }) export class AppModule { } diff --git a/metron-interface/metron-config/src/app/general-settings/general-settings.module.ts b/metron-interface/metron-config/src/app/general-settings/general-settings.module.ts index 83537b7764..1e8d72b33f 100644 --- a/metron-interface/metron-config/src/app/general-settings/general-settings.module.ts +++ b/metron-interface/metron-config/src/app/general-settings/general-settings.module.ts @@ -21,10 +21,11 @@ import {FormsModule, ReactiveFormsModule} from '@angular/forms'; import {routing} from './general-settings.routing'; import {GeneralSettingsComponent} from './general-settings.component'; import {AceEditorModule} from '../shared/ace-editor/ace-editor.module'; +import {BlocklyEditorModule} from "../shared/blockly-editor/blockly-editor.module"; @NgModule ({ - imports: [ CommonModule, routing, FormsModule, ReactiveFormsModule, AceEditorModule ], + imports: [ CommonModule, routing, FormsModule, ReactiveFormsModule, AceEditorModule, BlocklyEditorModule ], declarations: [ GeneralSettingsComponent ] }) export class GeneralSettingsModule { } diff --git a/metron-interface/metron-config/src/app/model/stellar-function-description.ts b/metron-interface/metron-config/src/app/model/stellar-function-description.ts index 5dd7f9a5dc..a9b9d0c301 100644 --- a/metron-interface/metron-config/src/app/model/stellar-function-description.ts +++ b/metron-interface/metron-config/src/app/model/stellar-function-description.ts @@ -20,11 +20,13 @@ export class StellarFunctionDescription { description: string; params: string[]; returns: string; + category: string; - constructor(name: string, description: string, params: string[], returns?: string) { + constructor(name: string, description: string, params: string[], returns: string, category: string) { this.name = name; this.description = description; this.params = params; this.returns = returns; + this.category = category; } } diff --git a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html index 33e8fd5215..d00a10afaa 100644 --- a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html +++ b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html @@ -31,7 +31,7 @@ (hideFieldSchema)="hidePane(pane.FIELDSCHEMA)"> diff --git a/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/rule-blockly/sensor-blockly-editor.component.html b/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/rule-blockly/sensor-blockly-editor.component.html new file mode 100644 index 0000000000..786314664d --- /dev/null +++ b/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/rule-blockly/sensor-blockly-editor.component.html @@ -0,0 +1,52 @@ + +
+ +
Edit Rule
+ + +
+
+ +
+
+ + +
+
+ + +
+
+ + +
+ +
+
+
+ +
+
+
+ + +
+
+
+ +
diff --git a/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/rule-blockly/sensor-blockly-editor.component.scss b/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/rule-blockly/sensor-blockly-editor.component.scss new file mode 100644 index 0000000000..a818a2f0eb --- /dev/null +++ b/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/rule-blockly/sensor-blockly-editor.component.scss @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing software + * distributed under the License is distributed on an "AS IS" BASIS + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@import "../../../_variables.scss"; + +textarea +{ + height: auto; +} + +.form-title +{ + padding-left: 25px; +} + +.form-group +{ + padding-left: 25px; + padding-right: 20px; + padding-bottom: 10px; +} + +.close-button +{ + padding-right: 20px; +} + +$range-gradient-start: #AC9B5A; +$range-gradient-end: #D8747C; +input[type="range"] { + height: 4px; + -webkit-appearance: none; + -moz-apperance: none; + background: $range-gradient-start; + background: -webkit-linear-gradient(left, $range-gradient-start, $range-gradient-end); + background: -moz-linear-gradient(left, $range-gradient-start, $range-gradient-end); + background: -ms-linear-gradient(left, $range-gradient-start, $range-gradient-end); + background: -o-linear-gradient(left, $range-gradient-start, $range-gradient-end); + background: linear-gradient(to right, $range-gradient-start, $range-gradient-end); + border-radius: 4px; + + &:focus + { + outline: none; + } +} + +input[type="range"]::-webkit-slider-thumb{ + -webkit-appearance:none; + -moz-apperance:none; + width:15px; + height:15px; + -webkit-border-radius:20px; + -moz-border-radius:20px; + -ms-border-radius:20px; + -o-border-radius:20px; + border-radius:20px; + background-color: $range-gradient-start; +} + +input[type="range"]::-moz-range-track +{ + background:transparent; border:0px; +} + +input[type=range]::-moz-range-thumb +{ + background-color: $range-gradient-start; +} diff --git a/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/rule-blockly/sensor-blockly-editor.component.ts b/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/rule-blockly/sensor-blockly-editor.component.ts new file mode 100644 index 0000000000..94927c3f25 --- /dev/null +++ b/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/rule-blockly/sensor-blockly-editor.component.ts @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import {Component, OnInit, Input, EventEmitter, Output, AfterViewInit, ViewChild, ElementRef} from '@angular/core'; +import {BlocklyService} from "../../../service/blockly.service"; +import {RiskLevelRule} from "../../../model/risk-level-rule"; +import {SensorParserConfigService} from "../../../service/sensor-parser-config.service"; +import {SampleDataComponent} from "../../../shared/sample-data/sample-data.component"; +import {ParseMessageRequest} from "../../../model/parse-message-request"; +import {BlocklyEditorComponent} from "../../../shared/blockly-editor/blockly-editor.component"; + +declare var Blockly: any; + +@Component({ + selector: 'metron-config-sensor-rule-blockly', + templateUrl: './sensor-blockly-editor.component.html', + styleUrls: ['./sensor-blockly-editor.component.scss'] +}) + +export class SensorBlocklyEditorComponent implements OnInit, AfterViewInit { + + @Input() riskLevelRule: RiskLevelRule; + @Input() sensorParserConfig: string; + + @ViewChild(SampleDataComponent) sampleData: SampleDataComponent; + @ViewChild(BlocklyEditorComponent) blocklyEditor: BlocklyEditorComponent; + @Output() onCancelBlocklyEditor: EventEmitter = new EventEmitter(); + @Output() onSubmitBlocklyEditor: EventEmitter = new EventEmitter(); + private newRiskLevelRule = new RiskLevelRule(); + + constructor(private sensorParserConfigService: SensorParserConfigService) { } + + ngOnInit() { + Object.assign(this.newRiskLevelRule, this.riskLevelRule); + } + + ngAfterViewInit(): void { + this.sampleData.getNextSample(); + } + + onStatementChange(statement: string): void { + this.newRiskLevelRule.rule = statement; + } + + onSave(): void { + this.onSubmitBlocklyEditor.emit(this.newRiskLevelRule); + } + + onCancel(): void { + this.onCancelBlocklyEditor.emit(true); + } + + onSampleDataChanged(sampleData: string) { + let parseMessageRequest = new ParseMessageRequest(); + parseMessageRequest.sensorParserConfig = JSON.parse(JSON.stringify(this.sensorParserConfig)); + parseMessageRequest.sampleData = sampleData; + this.sensorParserConfigService.parseMessage(parseMessageRequest).subscribe( + parserResult => { + let availableFields = Object.keys(parserResult); + this.blocklyEditor.updateAvailableFieldsBlock(availableFields); + }, + error => { + this.blocklyEditor.updateAvailableFieldsBlock([]); + }); + } + +} diff --git a/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/rule-blockly/sensor-blockly-editor.module.ts b/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/rule-blockly/sensor-blockly-editor.module.ts new file mode 100644 index 0000000000..2b475b194d --- /dev/null +++ b/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/rule-blockly/sensor-blockly-editor.module.ts @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {NgModule, NO_ERRORS_SCHEMA} from '@angular/core'; +import {SensorBlocklyEditorComponent} from "./sensor-blockly-editor.component"; +import {NumberSpinnerModule} from "../../../shared/number-spinner/number-spinner.module"; +import {SharedModule} from "../../../shared/shared.module"; +import {SampleDataModule} from "../../../shared/sample-data/sample-data.module"; +import {BlocklyEditorModule} from "../../../shared/blockly-editor/blockly-editor.module"; + +@NgModule ({ + imports: [ SharedModule, NumberSpinnerModule, SampleDataModule, BlocklyEditorModule ], + exports: [ SensorBlocklyEditorComponent ], + declarations: [ SensorBlocklyEditorComponent ], + schemas: [ NO_ERRORS_SCHEMA ] +}) +export class SensorBlocklyEditorModule {} diff --git a/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/sensor-threat-triage.component.html b/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/sensor-threat-triage.component.html index 9d67df6d9a..8c45c3cc73 100644 --- a/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/sensor-threat-triage.component.html +++ b/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/sensor-threat-triage.component.html @@ -17,6 +17,10 @@ (onCancelTextEditor)="onCancelTextEditor()" (onSubmitTextEditor)="onSubmitTextEditor($event)"> + +
Threat Triage Rules
@@ -73,6 +77,8 @@
+
diff --git a/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/sensor-threat-triage.component.ts b/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/sensor-threat-triage.component.ts index db32b0420a..9a4644de41 100644 --- a/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/sensor-threat-triage.component.ts +++ b/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/sensor-threat-triage.component.ts @@ -20,6 +20,7 @@ import {Component, Input, EventEmitter, Output, OnChanges, SimpleChanges} from ' import {SensorEnrichmentConfig } from '../../model/sensor-enrichment-config'; import {RiskLevelRule} from '../../model/risk-level-rule'; import {SensorEnrichmentConfigService} from '../../service/sensor-enrichment-config.service'; +import {SensorParserConfig} from "../../model/sensor-parser-config"; export enum SortOrderOption { Lowest_Score, Highest_Score, Lowest_Name, Highest_Name @@ -38,6 +39,7 @@ export enum ThreatTriageFilter { export class SensorThreatTriageComponent implements OnChanges { @Input() showThreatTriage: boolean; + @Input() sensorParserConfig: SensorParserConfig; @Input() sensorEnrichmentConfig: SensorEnrichmentConfig; @Output() hideThreatTriage: EventEmitter = new EventEmitter(); @@ -45,6 +47,7 @@ export class SensorThreatTriageComponent implements OnChanges { visibleRules: RiskLevelRule[] = []; showTextEditor = false; + showBlocklyEditor = false; currentRiskLevelRule: RiskLevelRule; lowAlerts = 0; @@ -92,6 +95,7 @@ export class SensorThreatTriageComponent implements OnChanges { onEditRule(riskLevelRule: RiskLevelRule) { this.currentRiskLevelRule = riskLevelRule; this.showTextEditor = true; + this.showBlocklyEditor = false; } onDeleteRule(riskLevelRule: RiskLevelRule) { @@ -104,6 +108,23 @@ export class SensorThreatTriageComponent implements OnChanges { this.showTextEditor = true; } + onSubmitBlocklyEditor(riskLevelRule: RiskLevelRule): void { + this.deleteRule(this.currentRiskLevelRule); + this.sensorEnrichmentConfig.threatIntel.triageConfig.riskLevelRules.push(riskLevelRule); + this.showBlocklyEditor = false; + this.init(); + } + + onCancelBlocklyEditor(): void { + this.showBlocklyEditor = false; + } + + onEditBlockly(riskLevelRule: RiskLevelRule) { + this.currentRiskLevelRule = riskLevelRule; + this.showTextEditor = false; + this.showBlocklyEditor = true; + } + deleteRule(riskLevelRule: RiskLevelRule) { let index = this.sensorEnrichmentConfig.threatIntel.triageConfig.riskLevelRules.indexOf(riskLevelRule); if (index != -1) { diff --git a/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/sensor-threat-triage.module.ts b/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/sensor-threat-triage.module.ts index 66838d90a4..8538620c61 100644 --- a/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/sensor-threat-triage.module.ts +++ b/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/sensor-threat-triage.module.ts @@ -19,10 +19,10 @@ import { NgModule } from '@angular/core'; import {SharedModule} from '../../shared/shared.module'; import {SensorThreatTriageComponent} from './sensor-threat-triage.component'; import {SensorRuleEditorModule} from './rule-editor/sensor-rule-editor.module'; - +import {SensorBlocklyEditorModule} from './rule-blockly/sensor-blockly-editor.module'; @NgModule ({ - imports: [ SharedModule, SensorRuleEditorModule ], + imports: [ SharedModule, SensorRuleEditorModule, SensorBlocklyEditorModule ], declarations: [ SensorThreatTriageComponent ], exports: [ SensorThreatTriageComponent ] }) diff --git a/metron-interface/metron-config/src/app/service/blockly.service.ts b/metron-interface/metron-config/src/app/service/blockly.service.ts new file mode 100644 index 0000000000..09065e6bf6 --- /dev/null +++ b/metron-interface/metron-config/src/app/service/blockly.service.ts @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import {Injectable, Inject} from '@angular/core'; +import {Http, Headers, RequestOptions, Response, ResponseOptions} from '@angular/http'; +import {Observable} from 'rxjs/Observable'; +import {HttpUtil} from '../util/httpUtil'; +import {IAppConfig} from '../app.config.interface'; +import {APP_CONFIG} from '../app.config'; + +@Injectable() +export class BlocklyService { + url = this.config.apiEndpoint + '/blockly'; + defaultHeaders = {'Content-Type': 'application/json', 'X-Requested-With': 'XMLHttpRequest'}; + + constructor(private http: Http, @Inject(APP_CONFIG) private config: IAppConfig) { + } + + public statementToXml(statement: string): Observable<{}> { + return this.http.post(this.url + '/statement/to/xml', statement, new RequestOptions({headers: new Headers(this.defaultHeaders)})) + .map(HttpUtil.extractString) + .catch(HttpUtil.handleError); + } + +} diff --git a/metron-interface/metron-config/src/app/service/stellar.service.ts b/metron-interface/metron-config/src/app/service/stellar.service.ts index be04906528..8131be440c 100644 --- a/metron-interface/metron-config/src/app/service/stellar.service.ts +++ b/metron-interface/metron-config/src/app/service/stellar.service.ts @@ -65,4 +65,10 @@ export class StellarService { .catch(HttpUtil.handleError); } + public listFunctionsByCategory(): Observable<{string: StellarFunctionDescription[]}> { + return this.http.get(this.url + '/list/functions/by/category', new RequestOptions({headers: new Headers(this.defaultHeaders)})) + .map(HttpUtil.extractData) + .catch(HttpUtil.handleError); + } + } diff --git a/metron-interface/metron-config/src/app/shared/blockly-editor/blockly-editor.component.html b/metron-interface/metron-config/src/app/shared/blockly-editor/blockly-editor.component.html new file mode 100644 index 0000000000..bbf254fcb0 --- /dev/null +++ b/metron-interface/metron-config/src/app/shared/blockly-editor/blockly-editor.component.html @@ -0,0 +1,21 @@ + +
+
+ +
+ +
diff --git a/metron-interface/metron-config/src/app/shared/blockly-editor/blockly-editor.component.scss b/metron-interface/metron-config/src/app/shared/blockly-editor/blockly-editor.component.scss new file mode 100644 index 0000000000..19cd4670dd --- /dev/null +++ b/metron-interface/metron-config/src/app/shared/blockly-editor/blockly-editor.component.scss @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing software + * distributed under the License is distributed on an "AS IS" BASIS + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@import "../../_variables.scss"; + +textarea +{ + height: auto; +} + +.form-title +{ + padding-left: 25px; +} + +.form-group +{ + padding-left: 25px; + padding-right: 20px; + padding-bottom: 10px; +} + +.close-button +{ + padding-right: 20px; +} + diff --git a/metron-interface/metron-config/src/app/shared/blockly-editor/blockly-editor.component.spec.ts b/metron-interface/metron-config/src/app/shared/blockly-editor/blockly-editor.component.spec.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/metron-interface/metron-config/src/app/shared/blockly-editor/blockly-editor.component.ts b/metron-interface/metron-config/src/app/shared/blockly-editor/blockly-editor.component.ts new file mode 100644 index 0000000000..10a56dfda2 --- /dev/null +++ b/metron-interface/metron-config/src/app/shared/blockly-editor/blockly-editor.component.ts @@ -0,0 +1,203 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import {Component, OnInit, OnChanges, Input, EventEmitter, Output, AfterViewInit, ViewChild, ElementRef, SimpleChanges} from '@angular/core'; +import {BlocklyService} from "../../service/blockly.service"; +import {StellarService} from "../../service/stellar.service"; +import {StellarFunctionDescription} from "../../model/stellar-function-description"; + +declare var Blockly: any; + +@Component({ + selector: 'metron-config-blockly-editor', + templateUrl: './blockly-editor.component.html', + styleUrls: ['./blockly-editor.component.scss'] +}) + +export class BlocklyEditorComponent implements OnInit, AfterViewInit { + + @Input() value: string; + @Input() availableFields: string[] = []; + + @Output() onStatementChange: EventEmitter = new EventEmitter(); + + xml: string; + + @ViewChild('xmltemplate') xmltemplate: ElementRef; + statement: string; + workspace; + + constructor(private blocklyService: BlocklyService, private stellarService: StellarService) { } + + ngOnInit() { + // B'coz of https://github.com/google/blockly/issues/299 + Blockly.WorkspaceSvg.prototype.preloadAudio_ = function() {}; + } + + ngAfterViewInit() { + this.stellarService.listFunctionsByCategory().subscribe(stellarFunctionMap => { + this.generateBlocks(stellarFunctionMap); + this.generateToolbox(stellarFunctionMap); + this.injectBlockly(); + this.updateAvailableFieldsBlock(this.availableFields); + this.updateStatement(this.value); + }); + } + + generateBlocks(stellarFunctionMap: {string: StellarFunctionDescription[]}) { + for(let category of Object.keys(stellarFunctionMap)) { + for (let stellarFunctionDescription of stellarFunctionMap[category]) { + Blockly.Blocks['stellar_' + stellarFunctionDescription.name] = { + init: function() { + this.appendDummyInput() + .appendField(stellarFunctionDescription.name); + for(let param of stellarFunctionDescription.params) { + let formattedParam = param.split('-')[0].trim(); + this.appendValueInput(formattedParam.toUpperCase()) + .setCheck(null) + .appendField(formattedParam); + } + this.setOutput(true, null); + this.setColour(160); + this.setTooltip(stellarFunctionDescription.description); + this.setHelpUrl('http://www.example.com/'); + } + }; + Blockly.JavaScript['stellar_' + stellarFunctionDescription.name] = function(block) { + let values = []; + for(let param of stellarFunctionDescription.params) { + let formattedParam = param.split('-')[0].trim(); + values.push(Blockly.JavaScript.valueToCode(block, formattedParam.toUpperCase(), Blockly.JavaScript.ORDER_ADDITION)); + } + var code = stellarFunctionDescription.name + '(' + values.join(',') + ')'; + return [code, Blockly.JavaScript.ORDER_ADDITION]; + }; + } + } + } + + generateToolbox(stellarFunctionMap: {string: StellarFunctionDescription[]}) { + let xml = ''; + this.xml = xml; + } + + injectBlockly() { + this.xmltemplate.nativeElement.outerHTML = this.xml; + this.workspace = Blockly.inject('blocklyDiv', + {media: 'assets/blockly/media/', + css: false, + grid: + {spacing: 15, + length: 15, + colour: '#4d4d4d', + snap: true}, + toolbox: document.getElementById('toolbox')}); + + this.workspace.addChangeListener(event => { + if (event.type != 'ui') { + this.statement = Blockly.JavaScript.workspaceToCode(this.workspace).replace(/[;\n]/g, ''); + this.onStatementChange.emit(this.statement); + } + }); + } + + loadCurrentStatement() { + if (this.statement && this.workspace) { + this.blocklyService.statementToXml(this.statement).subscribe(xml => { + this.workspace.clear(); + let dom = Blockly.Xml.textToDom(xml); + Blockly.Xml.domToWorkspace(dom, this.workspace); + }); + } + } + + updateStatement(statement) { + this.statement = this.value; + this.loadCurrentStatement(); + } + + updateAvailableFieldsBlock(fields) { + if (!fields || fields.length == 0) { + fields = ['ip_src_addr', 'ip_src_port', 'ip_dst_addr', 'ip_dst_port', 'protocol', 'timestamp', 'includes_reverse_traffic']; + } + fields.sort(); + let fieldArray = []; + for(let field of fields) { + fieldArray.push([field, field]); + } + Blockly.Blocks['available_fields'] = { + init: function() { + this.appendDummyInput() + .appendField(new Blockly.FieldDropdown(fieldArray), "FIELD_NAME"); + this.setOutput(true, "String"); + this.setTooltip('These are the available fields'); + this.setHelpUrl('http://www.example.com/'); + this.setColour(270); + } + }; + this.loadCurrentStatement(); + } + +} diff --git a/metron-interface/metron-config/src/app/shared/blockly-editor/blockly-editor.module.ts b/metron-interface/metron-config/src/app/shared/blockly-editor/blockly-editor.module.ts new file mode 100644 index 0000000000..048ac83f11 --- /dev/null +++ b/metron-interface/metron-config/src/app/shared/blockly-editor/blockly-editor.module.ts @@ -0,0 +1,13 @@ +import {NgModule} from '@angular/core'; + +import {BlocklyEditorComponent} from './blockly-editor.component'; +import {SharedModule} from "../shared.module"; + +@NgModule({ + imports: [ SharedModule ], + exports: [ BlocklyEditorComponent ], + declarations: [ BlocklyEditorComponent ], + providers: [], +}) +export class BlocklyEditorModule { +} \ No newline at end of file diff --git a/metron-interface/metron-config/src/assets/blockly/stellar_blocks.js b/metron-interface/metron-config/src/assets/blockly/stellar_blocks.js new file mode 100644 index 0000000000..9bcb7c172e --- /dev/null +++ b/metron-interface/metron-config/src/assets/blockly/stellar_blocks.js @@ -0,0 +1,287 @@ +'use strict'; + +goog.provide('Blockly.Blocks.colour'); + +goog.require('Blockly.Blocks'); + +Blockly.Blocks['stellar_and'] = { + init: function() { + var OPERATORS = + [[Blockly.Msg.LOGIC_OPERATION_AND, 'AND'], + [Blockly.Msg.LOGIC_OPERATION_OR, 'OR']]; + this.appendDummyInput() + .appendField(new Blockly.FieldDropdown(OPERATORS), 'OP'); + this.appendValueInput("ARG1") + .setCheck('Boolean'); + this.setOutput(true, "Boolean"); + this.setTooltip(''); + this.setHelpUrl('http://www.example.com/'); + this.setColour(50); + }, + onchange: function(changeEvent) { + if (changeEvent.newParentId == this.id) { + var numFields = this.inputList.length - 1; + this.appendValueInput('ARG' + (numFields + 1)) + .setCheck('Boolean'); + } + if (changeEvent.oldParentId == this.id) { + this.removeInput(this.inputList[this.inputList.length - 1].name); + this.moveInputBefore(changeEvent.oldInputName, null); + for(var j = 1; j < this.inputList.length; j++) { + this.inputList[j].name = 'ARG' + j; + } + } + }, + domToMutation: function(xmlElement) { + console.log(xmlElement); + } +}; +Blockly.Blocks['stellar_arithmetic'] = { + /** + * Block for basic arithmetic operator. + * @this Blockly.Block + */ + init: function() { + this.jsonInit({ + "message0": "%1 %2 %3", + "args0": [ + { + "type": "input_value", + "name": "A" + // "check": "Number" + }, + { + "type": "field_dropdown", + "name": "OP", + "options": + [[Blockly.Msg.MATH_ADDITION_SYMBOL, 'ADD'], + [Blockly.Msg.MATH_SUBTRACTION_SYMBOL, 'MINUS'], + [Blockly.Msg.MATH_MULTIPLICATION_SYMBOL, 'MULTIPLY'], + [Blockly.Msg.MATH_DIVISION_SYMBOL, 'DIVIDE']] + }, + { + "type": "input_value", + "name": "B" + // "check": "Number" + } + ], + "inputsInline": true, + "output": "Number", + "colour": Blockly.Blocks.math.HUE, + "helpUrl": Blockly.Msg.MATH_ARITHMETIC_HELPURL + }); + // Assign 'this' to a variable for use in the tooltip closure below. + var thisBlock = this; + this.setTooltip(function() { + var mode = thisBlock.getFieldValue('OP'); + var TOOLTIPS = { + 'ADD': Blockly.Msg.MATH_ARITHMETIC_TOOLTIP_ADD, + 'MINUS': Blockly.Msg.MATH_ARITHMETIC_TOOLTIP_MINUS, + 'MULTIPLY': Blockly.Msg.MATH_ARITHMETIC_TOOLTIP_MULTIPLY, + 'DIVIDE': Blockly.Msg.MATH_ARITHMETIC_TOOLTIP_DIVIDE + }; + return TOOLTIPS[mode]; + }); + } +}; +Blockly.Blocks['stellar_in'] = { + /** + * Block for finding an item in the list. + * @this Blockly.Block + */ + init: function() { + this.setHelpUrl(Blockly.Msg.LISTS_INDEX_OF_HELPURL); + this.setColour(Blockly.Blocks.lists.HUE); + this.setOutput(true, 'Boolean'); + this.appendValueInput('INPUT'); + this.appendValueInput('LIST') + .appendField(new Blockly.FieldDropdown([["in", "in"],["not in", "not in"]]), 'OP') + .setCheck('Array'); + this.setInputsInline(true); + // Assign 'this' to a variable for use in the tooltip closure below. + var thisBlock = this; + this.setTooltip(function() { + return Blockly.Msg.LISTS_INDEX_OF_TOOLTIP.replace('%1', + this.workspace.options.oneBasedIndex ? '0' : '-1'); + }); + } +}; +Blockly.Blocks['stellar_map_create'] = { + /** + * Block for creating a list with any number of elements of any type. + * @this Blockly.Block + */ + init: function() { + this.setHelpUrl(Blockly.Msg.LISTS_CREATE_WITH_HELPURL); + this.setColour(Blockly.Blocks.lists.HUE); + this.itemCount_ = 3; + this.updateShape_(); + this.setOutput(true, 'Map'); + this.setMutator(new Blockly.Mutator(['lists_create_with_item'])); + this.setTooltip(Blockly.Msg.LISTS_CREATE_WITH_TOOLTIP); + }, + /** + * Create XML to represent list inputs. + * @return {!Element} XML storage element. + * @this Blockly.Block + */ + mutationToDom: function() { + var container = document.createElement('mutation'); + container.setAttribute('items', this.itemCount_); + return container; + }, + /** + * Parse XML to restore the list inputs. + * @param {!Element} xmlElement XML storage element. + * @this Blockly.Block + */ + domToMutation: function(xmlElement) { + this.itemCount_ = parseInt(xmlElement.getAttribute('items'), 10); + this.updateShape_(); + }, + /** + * Populate the mutator's dialog with this block's components. + * @param {!Blockly.Workspace} workspace Mutator's workspace. + * @return {!Blockly.Block} Root block in mutator. + * @this Blockly.Block + */ + decompose: function(workspace) { + var containerBlock = workspace.newBlock('lists_create_with_container'); + containerBlock.initSvg(); + var connection = containerBlock.getInput('STACK').connection; + for (var i = 0; i < this.itemCount_; i++) { + var itemBlock = workspace.newBlock('lists_create_with_item'); + itemBlock.initSvg(); + connection.connect(itemBlock.previousConnection); + connection = itemBlock.nextConnection; + } + return containerBlock; + }, + /** + * Reconfigure this block based on the mutator dialog's components. + * @param {!Blockly.Block} containerBlock Root block in mutator. + * @this Blockly.Block + */ + compose: function(containerBlock) { + var itemBlock = containerBlock.getInputTargetBlock('STACK'); + // Count number of inputs. + var connections = []; + while (itemBlock) { + connections.push(itemBlock.valueConnection_); + itemBlock = itemBlock.nextConnection && + itemBlock.nextConnection.targetBlock(); + } + // Disconnect any children that don't belong. + for (var i = 0; i < this.itemCount_; i++) { + var connection = this.getInput('ADD' + i).connection.targetConnection; + if (connection && connections.indexOf(connection) == -1) { + connection.disconnect(); + } + } + this.itemCount_ = connections.length; + this.updateShape_(); + // Reconnect any child blocks. + for (var i = 0; i < this.itemCount_; i++) { + Blockly.Mutator.reconnect(connections[i], this, 'ADD' + i); + } + }, + /** + * Store pointers to any connected child blocks. + * @param {!Blockly.Block} containerBlock Root block in mutator. + * @this Blockly.Block + */ + saveConnections: function(containerBlock) { + var itemBlock = containerBlock.getInputTargetBlock('STACK'); + var i = 0; + while (itemBlock) { + var input = this.getInput('ADD' + i); + itemBlock.valueConnection_ = input && input.connection.targetConnection; + i++; + itemBlock = itemBlock.nextConnection && + itemBlock.nextConnection.targetBlock(); + } + }, + /** + * Modify this block to have the correct number of inputs. + * @private + * @this Blockly.Block + */ + updateShape_: function() { + if (this.itemCount_ && this.getInput('EMPTY')) { + this.removeInput('EMPTY'); + } else if (!this.itemCount_ && !this.getInput('EMPTY')) { + this.appendDummyInput('EMPTY') + .appendField('create empty map'); + } + // Add new inputs. + for (var i = 0; i < this.itemCount_; i++) { + if (!this.getInput('ADD' + i)) { + var input = this.appendValueInput('ADD' + i).setCheck('KeyValue'); + if (i == 0) { + input.appendField("create map with"); + } + } + } + // Remove deleted inputs. + while (this.getInput('ADD' + i)) { + this.removeInput('ADD' + i); + i++; + } + } +}; +Blockly.Blocks['stellar_key_value'] = { + /** + * Block for finding an item in the list. + * @this Blockly.Block + */ + init: function() { + this.setHelpUrl(Blockly.Msg.LISTS_INDEX_OF_HELPURL); + this.setColour(Blockly.Blocks.lists.HUE); + this.setOutput(true, 'KeyValue'); + this.appendValueInput('KEY'); + this.appendValueInput('VALUE') + .appendField(" : "); + this.setInputsInline(true); + // Assign 'this' to a variable for use in the tooltip closure below. + var thisBlock = this; + this.setTooltip(function() { + return Blockly.Msg.LISTS_INDEX_OF_TOOLTIP.replace('%1', + this.workspace.options.oneBasedIndex ? '0' : '-1'); + }); + } +}; +Blockly.Blocks['stellar_negate'] = { + /** + * Block for negation. + * @this Blockly.Block + */ + init: function() { + this.jsonInit({ + "message0": Blockly.Msg.LOGIC_NEGATE_TITLE, + "args0": [ + { + "type": "input_value", + "name": "BOOL", + "check": "Boolean" + } + ], + "output": "Boolean", + "colour": Blockly.Blocks.logic.HUE, + "tooltip": Blockly.Msg.LOGIC_NEGATE_TOOLTIP, + "helpUrl": Blockly.Msg.LOGIC_NEGATE_HELPURL + }); + } +}; +Blockly.Blocks['stellar_EXISTS'] = { + init: function() { + this.appendDummyInput() + .appendField("EXISTS"); + this.appendValueInput("INPUT") + .setCheck(null) + .appendField("input"); + this.setOutput(true, null); + this.setColour(160); + this.setTooltip(''); + this.setHelpUrl('http://www.example.com/'); + } +}; \ No newline at end of file diff --git a/metron-interface/metron-config/src/assets/blockly/stellar_generator.js b/metron-interface/metron-config/src/assets/blockly/stellar_generator.js new file mode 100644 index 0000000000..4a05c2a22a --- /dev/null +++ b/metron-interface/metron-config/src/assets/blockly/stellar_generator.js @@ -0,0 +1,67 @@ +Blockly.JavaScript['available_fields'] = function(block) { + var field_name = block.getFieldValue('FIELD_NAME'); + return [field_name, Blockly.JavaScript.ORDER_ADDITION]; +}; +Blockly.JavaScript['stellar_and'] = function(block) { + var arguments = []; + for (var i = 1; i < block.inputList.length - 1; i++) { + arguments[i - 1] = Blockly.JavaScript.valueToCode(block, block.inputList[i].name, Blockly.JavaScript.ORDER_ADDITION); + } + if (block.getFieldValue('OP') == 'OR') { + return [arguments.join(' || '), Blockly.JavaScript.ORDER_LOGICAL_OR]; + } else { + return [arguments.join(' && '), Blockly.JavaScript.ORDER_LOGICAL_AND]; + } +}; +Blockly.JavaScript['stellar_arithmetic'] = function(block) { + // Basic arithmetic operators, and power. + var OPERATORS = { + 'ADD': [' + ', Blockly.JavaScript.ORDER_ADDITION], + 'MINUS': [' - ', Blockly.JavaScript.ORDER_SUBTRACTION], + 'MULTIPLY': [' * ', Blockly.JavaScript.ORDER_MULTIPLICATION], + 'DIVIDE': [' / ', Blockly.JavaScript.ORDER_DIVISION] + }; + var tuple = OPERATORS[block.getFieldValue('OP')]; + var operator = tuple[0]; + var order = tuple[1]; + var argument0 = Blockly.JavaScript.valueToCode(block, 'A', order) || '0'; + var argument1 = Blockly.JavaScript.valueToCode(block, 'B', order) || '0'; + var code = argument0 + operator + argument1; + return [code, order]; +}; +Blockly.JavaScript['stellar_in'] = function(block) { + var value_input = Blockly.JavaScript.valueToCode(block, 'INPUT', Blockly.JavaScript.ORDER_ADDITION); + var value_list = Blockly.JavaScript.valueToCode(block, 'LIST', Blockly.JavaScript.ORDER_ADDITION); + var field_op = block.getFieldValue('OP'); + var code = value_input + ' ' + field_op + ' ' + value_list; + return [code, Blockly.JavaScript.ORDER_ADDITION]; +}; +Blockly.JavaScript['stellar_map_create'] = function(block) { + // Create a list with any number of elements of any type. + var elements = new Array(block.itemCount_); + for (var i = 0; i < block.itemCount_; i++) { + elements[i] = Blockly.JavaScript.valueToCode(block, 'ADD' + i, + Blockly.JavaScript.ORDER_COMMA) || 'null'; + } + var code = '{' + elements.join(', ') + '}'; + return [code, Blockly.JavaScript.ORDER_ATOMIC]; +}; +Blockly.JavaScript['stellar_key_value'] = function(block) { + var value_key = Blockly.JavaScript.valueToCode(block, 'KEY', Blockly.JavaScript.ORDER_ADDITION); + var value_value = Blockly.JavaScript.valueToCode(block, 'VALUE', Blockly.JavaScript.ORDER_ADDITION); + var code = value_key + " : " + value_value; + return [code, Blockly.JavaScript.ORDER_ADDITION]; +}; +Blockly.JavaScript['stellar_negate'] = function(block) { + // Negation. + var order = Blockly.JavaScript.ORDER_LOGICAL_NOT; + var argument0 = Blockly.JavaScript.valueToCode(block, 'BOOL', order) || + 'true'; + var code = 'not' + argument0; + return [code, order]; +}; +Blockly.JavaScript['stellar_EXISTS'] = function(block) { + var value_input = Blockly.JavaScript.valueToCode(block, 'INPUT', Blockly.JavaScript.ORDER_ADDITION); + var code = 'EXISTS(' + value_input + ')'; + return [code, Blockly.JavaScript.ORDER_ADDITION]; +}; \ No newline at end of file diff --git a/metron-interface/metron-config/src/index.html b/metron-interface/metron-config/src/index.html index 2c3f8c5b94..785f09ea44 100644 --- a/metron-interface/metron-config/src/index.html +++ b/metron-interface/metron-config/src/index.html @@ -22,6 +22,12 @@ + + + + + + diff --git a/metron-interface/metron-config/src/styles.scss b/metron-interface/metron-config/src/styles.scss index db57e91bf4..e1a5f7d340 100644 --- a/metron-interface/metron-config/src/styles.scss +++ b/metron-interface/metron-config/src/styles.scss @@ -18,6 +18,7 @@ /* You can add global styles to this file, and also import other style files */ @import "app/_variables.scss"; @import "app/_main.scss"; +@import "app/_blockly.scss"; $height: 60px; diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/StellarFunctionDescription.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/StellarFunctionDescription.java index a1b9f03a21..322add07d8 100644 --- a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/StellarFunctionDescription.java +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/StellarFunctionDescription.java @@ -19,34 +19,40 @@ public class StellarFunctionDescription { - private String name; - private String description; - private String[] params; - private String returns; - - public StellarFunctionDescription(String name, String description, String[] params, String returns) { - this.name = name; - this.description = description; - this.params = params; - this.returns = returns; - } + private String name; + private String description; + private String[] params; + private String returns; + private String category; - public String getName() { - return name; - } + public StellarFunctionDescription(String name, String description, String[] params, String returns, String category) { + this.name = name; + this.description = description; + this.params = params; + this.returns = returns; + this.category = category; + } - public String getDescription() { - return description; - } + public String getName() { + return name; + } - public String[] getParams() { - if (params == null) { - return new String[0]; - } - return params; - } + public String getDescription() { + return description; + } - public String getReturns() { - return returns; + public String[] getParams() { + if (params == null) { + return new String[0]; } + return params; + } + + public String getReturns() { + return returns; + } + + public String getCategory() { + return category; + } } diff --git a/metron-interface/metron-rest/pom.xml b/metron-interface/metron-rest/pom.xml index 2f59433260..2e6ec8cabb 100644 --- a/metron-interface/metron-rest/pom.xml +++ b/metron-interface/metron-rest/pom.xml @@ -172,6 +172,11 @@ + + org.apache.metron + stellar-common + ${project.parent.version} + io.springfox springfox-swagger2 diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/BlocklyController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/BlocklyController.java new file mode 100644 index 0000000000..b8066ecbe1 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/BlocklyController.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import io.swagger.annotations.ApiResponse; +import org.apache.metron.rest.service.GlobalConfigService; +import org.apache.metron.rest.service.BlocklyService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; + +@RestController +@RequestMapping("/api/v1/blockly") +public class BlocklyController { + + @Autowired + private BlocklyService blocklyService; + + @ApiOperation(value = "Converts a Stellar statement to Blockly xml") + @ApiResponse(message = "Blockly xml", code = 200) + @RequestMapping(value = "/statement/to/xml", method = RequestMethod.POST) + ResponseEntity statementToXml(@ApiParam(name="statement", value="Stellar statement to be converted to Blockly", required=true)@RequestBody String stellarStatement) throws Exception { + return new ResponseEntity<>(blocklyService.statementToXml(stellarStatement), HttpStatus.OK); + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java index 13327f9279..b05186dac7 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java @@ -77,4 +77,11 @@ ResponseEntity> listFunctions() throws RestExce ResponseEntity> listSimpleFunctions() throws RestException { return new ResponseEntity<>(stellarService.getSimpleStellarFunctions(), HttpStatus.OK); } + + @ApiOperation(value = "Lists the Stellar functions that can be found on the classpath, grouped by category") + @ApiResponse(message = "Returns a list of Stellar functions grouped by category", code = 200) + @RequestMapping(value = "/list/functions/by/category", method = RequestMethod.GET) + ResponseEntity>> listFunctionsByCategory() throws Exception { + return new ResponseEntity<>(stellarService.getStellarFunctionsByCategory(), HttpStatus.OK); + } } diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/BlocklyService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/BlocklyService.java new file mode 100644 index 0000000000..264f15dd9a --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/BlocklyService.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service; + +import org.apache.metron.rest.RestException; + +public interface BlocklyService { + + String statementToXml(String statement) throws RestException; +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java index 14cd31fc9a..8e94742b76 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java @@ -36,4 +36,6 @@ public interface StellarService { List getSimpleStellarFunctions(); + Map> getStellarFunctionsByCategory(); + } diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java index 715bd370da..d5ab2ee4ab 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java @@ -33,9 +33,11 @@ import org.springframework.stereotype.Service; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; @Service @@ -44,6 +46,15 @@ public class StellarServiceImpl implements StellarService { private CuratorFramework client; @Autowired + public StellarServiceImpl(Optional client) { + this.client = client.isPresent() ? client.get() : null; + try { + ConfigurationsUtils.setupStellarStatically(this.client); + } catch (Exception e) { + throw new IllegalStateException("Unable to setup stellar statically: " + e.getMessage(), e); + } + } + public StellarServiceImpl(CuratorFramework client) { this.client = client; try { @@ -88,15 +99,35 @@ public List getStellarFunctions() { List stellarFunctionDescriptions = new ArrayList<>(); Iterable stellarFunctionsInfo = StellarFunctions.FUNCTION_RESOLVER().getFunctionInfo(); stellarFunctionsInfo.forEach(stellarFunctionInfo -> { - stellarFunctionDescriptions.add(new StellarFunctionDescription( - stellarFunctionInfo.getName(), - stellarFunctionInfo.getDescription(), - stellarFunctionInfo.getParams(), - stellarFunctionInfo.getReturns())); + String category = getStellarCategory(stellarFunctionInfo); + if (category != null) { + stellarFunctionDescriptions.add(new StellarFunctionDescription( + stellarFunctionInfo.getName(), + stellarFunctionInfo.getDescription(), + stellarFunctionInfo.getParams(), + stellarFunctionInfo.getReturns(), + getStellarCategory(stellarFunctionInfo))); + } }); return stellarFunctionDescriptions; } + private String getStellarCategory(StellarFunctionInfo stellarFunctionInfo) { + String categoryName; + if (stellarFunctionInfo.getFunction().getClass().getEnclosingClass() != null) { + categoryName = stellarFunctionInfo.getFunction().getClass().getEnclosingClass().getSimpleName(); + } else { + Class superClass = (Class) stellarFunctionInfo.getFunction().getClass().getGenericSuperclass(); + categoryName = superClass.getSimpleName(); + } + if (categoryName.contains("Validation")) { + return categoryName.replaceFirst("Validation", ""); + } else if (categoryName.contains("Functions")){ + return categoryName.replaceFirst("Functions", ""); + } + return null; + } + @Override public List getSimpleStellarFunctions() { List stellarFunctionDescriptions = getStellarFunctions(); @@ -104,4 +135,19 @@ public List getSimpleStellarFunctions() { stellarFunctionDescription.getParams().length == 1).sorted((o1, o2) -> o1.getName().compareTo(o2.getName())).collect(Collectors.toList()); } + @Override + public Map> getStellarFunctionsByCategory() { + Map> stellarFunctionsByCategory = new HashMap<>(); + List stellarFunctions = getStellarFunctions(); + stellarFunctions.forEach(stellarFunctionDescription -> { + String category = stellarFunctionDescription.getCategory(); + stellarFunctionsByCategory.putIfAbsent(category, new ArrayList<>()); + stellarFunctionsByCategory.get(category).add(stellarFunctionDescription); + }); + for(String key: stellarFunctionsByCategory.keySet()) { + Collections.sort(stellarFunctionsByCategory.get(key), (o1, o2) -> o1.getName().compareTo(o2.getName())); + } + return stellarFunctionsByCategory; + } + } diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/Block.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/Block.java new file mode 100644 index 0000000000..73f25a63b4 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/Block.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl.blockly; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import java.util.ArrayList; +import java.util.List; + +@XmlAccessorType(XmlAccessType.FIELD) +public class Block { + + @XmlAttribute + private String type; + + @XmlElement(name="mutation") + private Mutation mutation; + + @XmlElement(name="field") + private List fields; + + @XmlElement(name="value") + private List values; + + @XmlAttribute + private String x; + + @XmlAttribute + private String y; + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Block withType(String type) { + this.type = type; + return this; + } + + public Mutation getMutation() { + return this.mutation; + } + + public void setMutation(Mutation mutation) { + this.mutation = mutation; + } + + public Block withMutation(Mutation mutation) { + this.mutation = mutation; + return this; + } + + public List getFields() { + return fields; + } + + public void setFields(List fields) { + this.fields = fields; + } + + public Block addField(Field field) { + if (fields == null) { + fields = new ArrayList<>(); + } + fields.add(field); + return this; + } + + public List getValues() { + return values; + } + + public void setValues(List values) { + this.values = values; + } + + public Block addValue(Value value) { + if (values == null) { + values = new ArrayList<>(); + } + values.add(value); + return this; + } + + public String getX() { + return x; + } + + public void setX(String x) { + this.x = x; + } + + public Block withX(String x) { + this.x = x; + return this; + } + + public String getY() { + return y; + } + + public void setY(String y) { + this.y = y; + } + + public Block withY(String y) { + this.y = y; + return this; + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/BlocklyCompiler.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/BlocklyCompiler.java new file mode 100644 index 0000000000..7af63aa83c --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/BlocklyCompiler.java @@ -0,0 +1,366 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl.blockly; + +import org.apache.metron.stellar.common.evaluators.ArithmeticEvaluator; +import org.apache.metron.stellar.common.evaluators.ComparisonExpressionWithOperatorEvaluator; +import org.apache.metron.stellar.common.evaluators.NumberLiteralEvaluator; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.FunctionMarker; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Token; +import org.apache.metron.stellar.dsl.VariableResolver; +import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; +import org.apache.metron.stellar.common.StellarCompiler; +import org.apache.metron.stellar.common.generated.StellarParser; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Stack; + +public class BlocklyCompiler extends StellarCompiler { + + private Xml xml = new Xml(); + private Map functionParamMap = new HashMap<>(); + private Stack> tokenStack; + + public BlocklyCompiler(VariableResolver variableResolver, FunctionResolver functionResolver, Context context, Stack> tokenStack) { + super(ArithmeticEvaluator.INSTANCE, NumberLiteralEvaluator.INSTANCE, ComparisonExpressionWithOperatorEvaluator.INSTANCE); + this.tokenStack = tokenStack; + functionResolver.getFunctionInfo().forEach(stellarFunctionInfo -> functionParamMap.put(stellarFunctionInfo.getName(), stellarFunctionInfo.getParams())); + } + + @Override + public void exitNullConst(StellarParser.NullConstContext ctx) { + Block nullBlock = new Block().withType("logic_null"); + tokenStack.push(new Token<>(nullBlock, Block.class)); + } + + @Override + public void exitArithExpr_plus(StellarParser.ArithExpr_plusContext ctx) { + Token right = popStack(); + Token left = popStack(); + Block plusBlock = new Block().withType("stellar_arithmetic") + .addField(new Field().withName("OP").withValue("ADD")) + .addValue(new Value().withName("A").addBlock((Block) left.getValue())) + .addValue(new Value().withName("B").addBlock((Block) right.getValue())); + tokenStack.push(new Token<>(plusBlock, Block.class)); + } + + @Override + public void exitArithExpr_minus(StellarParser.ArithExpr_minusContext ctx) { + Token right = popStack(); + Token left = popStack(); + Block minusBlock = new Block().withType("stellar_arithmetic") + .addField(new Field().withName("OP").withValue("MINUS")) + .addValue(new Value().withName("A").addBlock((Block) left.getValue())) + .addValue(new Value().withName("B").addBlock((Block) right.getValue())); + tokenStack.push(new Token<>(minusBlock, Block.class)); + } + + @Override + public void exitArithExpr_div(StellarParser.ArithExpr_divContext ctx) { + Token right = popStack(); + Token left = popStack(); + Block divBlock = new Block().withType("stellar_arithmetic") + .addField(new Field().withName("OP").withValue("DIVIDE")) + .addValue(new Value().withName("A").addBlock((Block) left.getValue())) + .addValue(new Value().withName("B").addBlock((Block) right.getValue())); + tokenStack.push(new Token<>(divBlock, Block.class)); + } + + @Override + public void exitArithExpr_mul(StellarParser.ArithExpr_mulContext ctx) { + Token right = popStack(); + Token left = popStack(); + Block mulBlock = new Block().withType("stellar_arithmetic") + .addField(new Field().withName("OP").withValue("MULTIPLY")) + .addValue(new Value().withName("A").addBlock((Block) left.getValue())) + .addValue(new Value().withName("B").addBlock((Block) right.getValue())); + tokenStack.push(new Token<>(mulBlock, Block.class)); + } + + private void handleConditional() { + Token elseExpr = popStack(); + Token thenExpr = popStack(); + Token ifExpr = popStack(); + Block ternaryBlock = new Block().withType("logic_ternary") + .addValue(new Value().withName("IF").addBlock((Block) ifExpr.getValue())) + .addValue(new Value().withName("THEN").addBlock((Block) thenExpr.getValue())) + .addValue(new Value().withName("ELSE").addBlock((Block) elseExpr.getValue())); + tokenStack.push(new Token<>(ternaryBlock, Block.class)); + } + + @Override + public void exitTernaryFuncWithoutIf(StellarParser.TernaryFuncWithoutIfContext ctx) { + handleConditional(); + } + + @Override + public void exitTernaryFuncWithIf(StellarParser.TernaryFuncWithIfContext ctx) { + handleConditional(); + } + + private void handleIn() { + handleInNotIn(true); + } + + private void handleNotIn() { + handleInNotIn(false); + } + + private void handleInNotIn(boolean in) { + Token right = popStack(); + Token left = popStack(); + Block inNotInBlock = new Block().withType("stellar_in") + .addValue(new Value().withName("INPUT").addBlock((Block) left.getValue())) + .addValue(new Value().withName("LIST").addBlock((Block) right.getValue())); + if (in) { + inNotInBlock.addField(new Field().withName("OP").withValue("in")); + } else { + inNotInBlock.addField(new Field().withName("OP").withValue("not in")); + } + tokenStack.push(new Token<>(inNotInBlock, Block.class)); + } + + @Override + public void exitInExpression(StellarParser.InExpressionContext ctx) { + handleIn(); + } + +// @Override +// public void exitNInExpression(StellarParser.NInExpressionContext ctx) { +// handleNotIn(); +// } + + @Override + public void exitNotFunc(StellarParser.NotFuncContext ctx) { + Token arg = popStack(); + Block notBlock = new Block().withType("stellar_negate") + .addValue(new Value().withName("BOOL").addBlock((Block) arg.getValue())); + tokenStack.push(new Token<>(notBlock, Block.class)); + } + + private Block getAvailableFieldsBlock(String fieldName) { + return new Block().withType("available_fields").addField(new Field().withName("FIELD_NAME").withValue(fieldName)); + } + + @Override + public void exitVariable(StellarParser.VariableContext ctx) { + Block availableFieldsBlock = getAvailableFieldsBlock(ctx.getText()); + tokenStack.push(new Token<>(availableFieldsBlock, Block.class)); + } + + @Override + public void exitStringLiteral(StellarParser.StringLiteralContext ctx) { + Block textBlock = new Block().withType("text") + .addField(new Field().withName("TEXT").withValue(ctx.getText().substring(1, ctx.getText().length() - 1))); + tokenStack.push(new Token<>(textBlock, Block.class)); + } + + private void handleNumber(String numberText) { + Block numBlock = new Block().withType("math_number") + .addField(new Field().withName("NUM").withValue(numberText)); + tokenStack.push(new Token<>(numBlock, Block.class)); + } + + @Override + public void exitIntLiteral(StellarParser.IntLiteralContext ctx) { + handleNumber(ctx.getText()); + } + + @Override + public void exitDoubleLiteral(StellarParser.DoubleLiteralContext ctx) { + handleNumber(ctx.getText()); + } + + @Override + public void exitFloatLiteral(StellarParser.FloatLiteralContext ctx) { + handleNumber(ctx.getText()); + } + + @Override + public void exitLongLiteral(StellarParser.LongLiteralContext ctx) { + handleNumber(ctx.getText()); + } + + @Override + public void exitLogicalExpressionAnd(StellarParser.LogicalExpressionAndContext ctx) { + Token right = popStack(); + Token left = popStack(); + Block andBlock = new Block().withType("logic_operation") + .addField(new Field().withName("OP").withValue("AND")) + .addValue(new Value().withName("A").addBlock((Block) left.getValue())) + .addValue(new Value().withName("B").addBlock((Block) right.getValue())); + tokenStack.push(new Token<>(andBlock, Block.class)); + } + + @Override + public void exitLogicalExpressionOr(StellarParser.LogicalExpressionOrContext ctx) { + Token right = popStack(); + Token left = popStack(); + Block orBlock = new Block().withType("logic_operation") + .addField(new Field().withName("OP").withValue("OR")) + .addValue(new Value().withName("A").addBlock((Block) left.getValue())) + .addValue(new Value().withName("B").addBlock((Block) right.getValue())); + tokenStack.push(new Token<>(orBlock, Block.class)); + } + + @Override + public void exitLogicalConst(StellarParser.LogicalConstContext ctx) { + Block booleanBlock = new Block().withType("logic_boolean") + .addField(new Field().withName("BOOL").withValue(ctx.getText().toUpperCase())); + tokenStack.push(new Token<>(booleanBlock, Block.class)); + } + + @Override + public void exitTransformationFunc(StellarParser.TransformationFuncContext ctx) { + String functionName = ctx.getChild(0).getText(); + String[] paramNames = functionParamMap.get(functionName); + Block functionBlock = new Block().withType("stellar_" + functionName); + List args = getFunctionArguments(popStack()); + for(int i = 0; i < args.size(); i++) { + String paramName = paramNames[i].replaceAll("(.*?) .*", "$1").trim().toUpperCase(); + Value value = new Value().addBlock((Block) args.get(i)).withName(paramName); + functionBlock.addValue(value); + } + tokenStack.push(new Token<>(functionBlock, Object.class)); + } + + private List getFunctionArguments(Token token) { + if (token.getUnderlyingType().equals(List.class)) { + return (List) token.getValue(); + + } else { + throw new ParseException("Unable to process in clause because " + token.getValue() + " is not a set"); + } + } + + @Override + public void exitExistsFunc(StellarParser.ExistsFuncContext ctx) { + Block availableFieldsBlock = getAvailableFieldsBlock(ctx.getChild(2).getText()); + Block existsBlock = new Block().withType("stellar_EXISTS") + .addValue(new Value().withName("INPUT").addBlock(availableFieldsBlock)); + tokenStack.push(new Token<>(existsBlock, Block.class)); + } + + @Override + public void enterFunc_args(StellarParser.Func_argsContext ctx) { + tokenStack.push(new Token<>(new FunctionMarker(), FunctionMarker.class)); + } + + @Override + public void exitFunc_args(StellarParser.Func_argsContext ctx) { + LinkedList args = new LinkedList<>(); + while (true) { + Token token = popStack(); + if (token.getUnderlyingType().equals(FunctionMarker.class)) { + break; + } else { + args.addFirst(token.getValue()); + } + } + tokenStack.push(new Token<>(args, List.class)); + } + + @Override + public void exitMap_entity(StellarParser.Map_entityContext ctx) { + List items = new ArrayList<>(); + Block mapBlock = new Block().withType("stellar_map_create"); + Block keyValueBlock = new Block().withType("stellar_key_value"); + for (int i = 0; true; i++) { + Token token = popStack(); + if (token.getUnderlyingType().equals(FunctionMarker.class)) { + break; + } else { + if (i % 2 == 0) { + keyValueBlock.addValue(new Value().withName("VALUE").addBlock((Block) token.getValue())); + } else { + keyValueBlock.addValue(new Value().withName("KEY").addBlock((Block) token.getValue())); + items.add(0, keyValueBlock); + keyValueBlock = new Block().withType("stellar_key_value"); + } + } + } + mapBlock.withMutation(new Mutation().withItems(items.size())); + for(int i = 0; i < items.size(); i++) { + mapBlock.addValue(new Value().withName("ADD" + i).addBlock(items.get(i))); + } + tokenStack.push(new Token<>(mapBlock, Block.class)); + } + + @Override + public void exitList_entity(StellarParser.List_entityContext ctx) { + List items = new ArrayList<>(); + Block listBlock = new Block().withType("lists_create_with"); + while (true) { + Token token = popStack(); + if (token.getUnderlyingType().equals(FunctionMarker.class)) { + break; + } else { + items.add(0, (Block) token.getValue()); + } + } + listBlock.withMutation(new Mutation().withItems(items.size())); + for(int i = 0; i < items.size(); i++) { + listBlock.addValue(new Value().withName("ADD" + i).addBlock(items.get(i))); + } + tokenStack.push(new Token<>(listBlock, Block.class)); + } + + @Override + public void exitComparisonExpressionWithOperator(StellarParser.ComparisonExpressionWithOperatorContext ctx) { + String op = ctx.getChild(1).getText(); + Token right = popStack(); + Token left = popStack(); + Block compareBlock = new Block().withType("logic_compare") + .addValue(new Value().withName("A").addBlock((Block) left.getValue())) + .addValue(new Value().withName("B").addBlock((Block) right.getValue())); + Field operatorField = new Field().withName("OP"); + if (op.equals("==")) { + operatorField.withValue("EQ"); + } else if (op.equals("!=")) { + operatorField.withValue("NEQ"); + } else if (op.equals("<")) { + operatorField.withValue("LT"); + } else if (op.equals(">")) { + operatorField.withValue("GT"); + } else if (op.equals(">=")) { + operatorField.withValue("GTE"); + } else { + operatorField.withValue("LTE"); + } + compareBlock.addField(operatorField); + tokenStack.push(new Token<>(compareBlock, Block.class)); + } + + public Xml getXml() { + xml.addBlock((Block) popStack().getValue()); + return this.xml; + } + + private Token popStack() { + if (tokenStack.empty()) { + throw new ParseException("Unable to pop an empty stack"); + } + return tokenStack.pop(); + } +} \ No newline at end of file diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/BlocklyServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/BlocklyServiceImpl.java new file mode 100644 index 0000000000..ee7b99cef2 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/BlocklyServiceImpl.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl.blockly; + +import org.antlr.v4.runtime.ANTLRInputStream; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.TokenStream; +import org.apache.commons.io.output.ByteArrayOutputStream; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.service.BlocklyService; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ErrorListener; +import org.apache.metron.stellar.dsl.StellarFunctions; +import org.apache.metron.stellar.dsl.functions.resolver.ClasspathFunctionResolver; +import org.apache.metron.stellar.common.generated.StellarLexer; +import org.apache.metron.stellar.common.generated.StellarParser; +import org.springframework.stereotype.Service; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import java.util.Properties; +import java.util.Stack; + +import static org.apache.metron.stellar.dsl.Context.Capabilities.STELLAR_CONFIG; + +@Service +public class BlocklyServiceImpl implements BlocklyService { + + public static void main(String[] args) throws RestException { + String statement = "IS_EMAIL(sensor_type) && sensor_type == 'yaf'"; + //String statement = "foo in [ TO_LOWER('CASEY'), 'david' ]"; + //String statement = "STATS_PERCENTILE( STATS_MERGE( PROFILE_GET('host-in-degree', ip_src_addr, 1, 'HOURS')), 95)"; + //String statement = "not(ENDS_WITH(domain_without_subdomains, '.com') or ENDS_WITH(domain_without_subdomains, '.net'))"; + //String statement = "(1.1 < 2.2 ? 'one' : 'two') == 'two'"; + //String statement = "sensor_type == null or true"; + //String statement = "EXISTS(sensor_type)"; + //String statement = "application not in ['test1', 'test2', 'test3']"; + //String statement = "MAP_EXISTS('test',{'test' : application, 'field' : 'value'})"; + BlocklyService blocklyService = new BlocklyServiceImpl(); + System.out.println(blocklyService.statementToXml(statement)); + } + + @Override + public String statementToXml(String statement) throws RestException { + ANTLRInputStream input = new ANTLRInputStream(statement); + StellarLexer lexer = new StellarLexer(input); + lexer.removeErrorListeners(); + lexer.addErrorListener(new ErrorListener()); + TokenStream tokens = new CommonTokenStream(lexer); + StellarParser parser = new StellarParser(tokens); + + Properties properties = new Properties(); + properties.put(ClasspathFunctionResolver.Config.STELLAR_SEARCH_INCLUDES_KEY, "org.apache.metron.*"); + Context.Builder contextBuilder = new Context.Builder() + .with(STELLAR_CONFIG, () -> properties); + StellarFunctions.initialize(contextBuilder.build()); + BlocklyCompiler blocklyCompiler = new BlocklyCompiler(null, StellarFunctions.FUNCTION_RESOLVER(), Context.EMPTY_CONTEXT(), new Stack<>()); + parser.addParseListener(blocklyCompiler); + parser.removeErrorListeners(); + parser.addErrorListener(new ErrorListener()); + parser.transformation(); + JAXBContext context = null; + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + try { + context = JAXBContext.newInstance(Xml.class); + Marshaller marshaller = context.createMarshaller(); + Xml xml = blocklyCompiler.getXml(); + xml.getBlocks().get(0).withX("10").withY("10"); + marshaller.marshal(xml, byteArrayOutputStream); + } catch (JAXBException e) { + throw new RestException(e); + } + return byteArrayOutputStream.toString(); + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/Field.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/Field.java new file mode 100644 index 0000000000..556be5b47a --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/Field.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl.blockly; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlValue; + +@XmlAccessorType(XmlAccessType.FIELD) +public class Field { + + @XmlAttribute + private String name; + + @XmlValue + private String value; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Field withName(String name) { + this.name = name; + return this; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public Field withValue(String value) { + this.value = value; + return this; + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/Mutation.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/Mutation.java new file mode 100644 index 0000000000..34c2ed2aaa --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/Mutation.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl.blockly; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; + +@XmlAccessorType(XmlAccessType.FIELD) +public class Mutation { + + @XmlAttribute + private int items; + + public int getItems() { + return items; + } + + public void setItems(int items) { + this.items = items; + } + + public Mutation withItems(int items) { + this.items = items; + return this; + } +} \ No newline at end of file diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/Value.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/Value.java new file mode 100644 index 0000000000..1ba870de3f --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/Value.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl.blockly; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import java.util.ArrayList; +import java.util.List; + +@XmlAccessorType(XmlAccessType.FIELD) +public class Value { + + @XmlAttribute + private String name; + + @XmlElement(name="block") + private List blocks; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Value withName(String name) { + this.name = name; + return this; + } + + public List getBlocks() { + return blocks; + } + + public void setBlocks(List blocks) { + this.blocks = blocks; + } + + public Value addBlock(Block block) { + if (blocks == null) { + blocks = new ArrayList<>(); + } + blocks.add(block); + return this; + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/Xml.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/Xml.java new file mode 100644 index 0000000000..2504544772 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/blockly/Xml.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl.blockly; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import javax.xml.bind.Unmarshaller; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlValue; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class Xml { + + @XmlElement(name="block") + private List blocks; + + public List getBlocks() { + return blocks; + } + + public void setBlocks(List blocks) { + this.blocks = blocks; + } + + public Xml addBlock(Block block) { + if (blocks == null) { + blocks = new ArrayList<>(); + } + blocks.add(block); + return this; + } + + private static String xml = "test"; + + public static void main(String[] args) throws JAXBException { + JAXBContext context = JAXBContext.newInstance(Xml.class); +// Unmarshaller unmarshaller = context.createUnmarshaller(); +// Xml xmlObject = (Xml) unmarshaller.unmarshal(new File("/Users/rmerriman/Projects/Metron/code/forks/merrimanr/incubator-metron/test.xml")); +// //Xml xmlObject = (Xml) unmarshaller.unmarshal(new ByteArrayInputStream(xml.getBytes())); +// System.out.print(xmlObject); + + + Xml xmlObject = new Xml().addBlock(new Block().withType("stellar_and").addField(new Field().withName("OP").withValue("AND"))); + Marshaller marshaller = context.createMarshaller(); + //marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); + marshaller.marshal(xmlObject, System.out); + } +} diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/blockly/BlocklyServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/blockly/BlocklyServiceImplTest.java new file mode 100644 index 0000000000..82cb5aeded --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/blockly/BlocklyServiceImplTest.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl.blockly; + +import org.apache.commons.io.FileUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.config.HadoopConfig; +import org.apache.metron.rest.service.impl.blockly.BlocklyServiceImpl; +import org.apache.metron.rest.service.impl.HdfsServiceImpl; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import javax.xml.bind.JAXBException; +import java.io.File; +import java.io.IOException; + +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes={BlocklyServiceImplTest.BlocklyTestContextConfiguration.class}) +@ActiveProfiles(TEST_PROFILE) +public class BlocklyServiceImplTest { + + @Configuration + @Profile(TEST_PROFILE) + static class BlocklyTestContextConfiguration { + + @Bean + public CuratorFramework curatorFramework() { + return mock(CuratorFramework.class); + } + } + + @Autowired + private CuratorFramework client; + + @Test + public void test() throws IOException { + String statement = "IS_EMAIL(sensor_type) && sensor_type == 'yaf'"; + //String statement = "foo in [ TO_LOWER('CASEY'), 'david' ]"; + //String statement = "STATS_PERCENTILE( STATS_MERGE( PROFILE_GET('host-in-degree', ip_src_addr, 1, 'HOURS')), 95)"; + //String statement = "not(ENDS_WITH(domain_without_subdomains, '.com') or ENDS_WITH(domain_without_subdomains, '.net'))"; + //String statement = "(1.1 < 2.2 ? 'one' : 'two') == 'two'"; + //String statement = "sensor_type == null or true"; + //String statement = "EXISTS(sensor_type)"; + //String statement = "application not in ['test1', 'test2', 'test3']"; + //String statement = "MAP_EXISTS('test',{'test' : application, 'field' : 'value'})"; + BlocklyServiceImpl blocklyService = new BlocklyServiceImpl(); + try { + System.out.println(blocklyService.statementToXml(statement)); + } catch (RestException e) { + e.printStackTrace(); + } + } +} diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java index fb3da9f625..2edbb932b8 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java @@ -43,7 +43,7 @@ public class StellarCompiler extends StellarBaseListener { private static Token EXPRESSION_REFERENCE = new Token<>(null, Object.class); private static Token LAMBDA_VARIABLES = new Token<>(null, Object.class); - private Expression expression; + protected Expression expression; private final ArithmeticEvaluator arithmeticEvaluator; private final NumberLiteralEvaluator numberLiteralEvaluator; private final ComparisonExpressionWithOperatorEvaluator comparisonExpressionWithOperatorEvaluator; From 397a74178815ca0ffdc5ea3b5c8afe2da481f3d2 Mon Sep 17 00:00:00 2001 From: merrimanr Date: Fri, 4 Aug 2017 17:08:16 -0500 Subject: [PATCH 2/2] fixed deployment bug --- metron-interface/metron-config/package.json | 2 +- metron-interface/metron-config/proxy.conf.json | 4 ++-- .../rule-blockly/sensor-blockly-editor.component.html | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/metron-interface/metron-config/package.json b/metron-interface/metron-config/package.json index d662075db4..23a01d4dab 100644 --- a/metron-interface/metron-config/package.json +++ b/metron-interface/metron-config/package.json @@ -20,7 +20,7 @@ "copy-blockly": "cp -f $npm_package_config_node_blockly/blockly_compressed.js $npm_package_config_node_blockly/blocks_compressed.js $npm_package_config_node_blockly/javascript_compressed.js src/assets/blockly", "copy-blockly-msg": "mkdir -p src/assets/blockly/msg/js && cp -f $npm_package_config_node_blockly/msg/js/en.js src/assets/blockly/msg/js", "copy-blockly-media": "mkdir -p src/assets/blockly/media && cp -f $npm_package_config_node_blockly/media/* src/assets/blockly/media", - "postinstall": "npm run copy-ace & npm run copy-ace-snippets" + "postinstall": "npm run copy-ace & npm run copy-ace-snippets & npm run copy-blockly & npm run copy-blockly-msg & npm run copy-blockly-media" }, "private": true, "dependencies": { diff --git a/metron-interface/metron-config/proxy.conf.json b/metron-interface/metron-config/proxy.conf.json index 612bd67420..29466ccc00 100644 --- a/metron-interface/metron-config/proxy.conf.json +++ b/metron-interface/metron-config/proxy.conf.json @@ -1,10 +1,10 @@ { "/api/v1": { - "target": "http://node1:8082", + "target": "http://localhost:8080", "secure": false }, "/logout": { - "target": "http://node1:8082", + "target": "http://localhost:8080", "secure": false } } diff --git a/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/rule-blockly/sensor-blockly-editor.component.html b/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/rule-blockly/sensor-blockly-editor.component.html index 786314664d..9250251f85 100644 --- a/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/rule-blockly/sensor-blockly-editor.component.html +++ b/metron-interface/metron-config/src/app/sensors/sensor-threat-triage/rule-blockly/sensor-blockly-editor.component.html @@ -35,7 +35,7 @@
- +