diff --git a/CHANGELOG.md b/CHANGELOG.md index ff1a62006..ae8de8ccd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,26 @@ +## [0.8.2] +###### 2018-06-14 + +### Added +* CSV to Array action in the Utilities app + + +### Changed +* The action results SSE stream truncates the result using the + `MAX_STREAM_RESULTS_SIZE_KB` config option + + +### Fixed +* Bytes conversion bug in the RedisCacheAdapter +* Bug in playbook editor using users and roles as arguments +* Bug where some callbacks weren't getting registered +* Column width bug in playbook editor, execution, and metrics pages +* OpenAPI validation bug with newest version of the swagger validator + + ## [0.8.2] ###### 2018-05-03 diff --git a/apps/Utilities/actions.py b/apps/Utilities/actions.py index 0920d6f28..d768c5aa5 100644 --- a/apps/Utilities/actions.py +++ b/apps/Utilities/actions.py @@ -34,6 +34,11 @@ def echo_array(data): return data +@action +def csv_as_array(data): + return data.split(",") + + @action def json_select(json_reference, element): return json.loads(json_reference)[element] diff --git a/apps/Utilities/api.yaml b/apps/Utilities/api.yaml index f0e29f746..845a77da7 100644 --- a/apps/Utilities/api.yaml +++ b/apps/Utilities/api.yaml @@ -71,6 +71,19 @@ actions: description: echoed list schema: type: array + csv as array: + run: actions.csv_as_array + description: returns a csv string as an array + parameters: + - name: data + description: csv to return + required: true + type: string + returns: + Success: + description: echoed list + schema: + type: array 'json select': run: actions.json_select description: Gets a selected sub element of a json diff --git a/docs/index.rst b/docs/index.rst index 6ae1c07ca..bbddb27ba 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,7 +7,7 @@ Welcome to Walkoff's documentation! *********************************** Welcome to Walkoff's Python documentation. If you are looking for documentation and tutorials on getting started with Walkoff, please first look at -`our Github Pages site `_. Here you'll find tutorials and documentation on both UI +`our Github Pages site `_. Here you'll find tutorials and documentation on both UI usage and app and interface development. This documentation is intended to help app and interface developers as well as provide a reference for project contributors. diff --git a/setup_walkoff.py b/setup_walkoff.py index 8445e3172..6a0ee9ee1 100644 --- a/setup_walkoff.py +++ b/setup_walkoff.py @@ -1,7 +1,5 @@ import os -from scripts.compose_api import compose_api - def main(): print('\nInstalling Python Dependencies...') diff --git a/tests/test_workflow_results_stream.py b/tests/test_workflow_results_stream.py index a20735767..09f7cc1f9 100644 --- a/tests/test_workflow_results_stream.py +++ b/tests/test_workflow_results_stream.py @@ -81,6 +81,23 @@ def test_format_action_data_with_results(self): self.assert_and_strip_timestamp(result) self.assertDictEqual(result, expected) + def test_format_action_data_with_long_results(self): + size_limit = 128 + self.app.config['MAX_STREAM_RESULTS_SIZE_KB'] = size_limit + workflow_id = str(uuid4()) + kwargs = {'data': {'workflow': {'execution_id': workflow_id}, + 'data': {'result': 'x'*1024*2*size_limit}}} # should exceed limit + sender = self.get_sample_action_sender() + status = ActionStatusEnum.executing + result = format_action_data_with_results(sender, kwargs, status) + expected = sender + expected['action_id'] = expected.pop('id') + expected['workflow_execution_id'] = workflow_id + expected['status'] = status.name + expected['result'] = {'truncated': 'x'*1024*size_limit} + self.assert_and_strip_timestamp(result) + self.assertDictEqual(result, expected) + def check_action_callback(self, callback, status, event, mock_publish, with_result=False): sender = self.get_sample_action_sender() kwargs = self.get_action_kwargs(with_result=with_result) diff --git a/walkoff.py b/walkoff.py index ccc61174e..edb791ed1 100644 --- a/walkoff.py +++ b/walkoff.py @@ -23,7 +23,6 @@ def run(app, host, port): pids = spawn_worker_processes() monkey.patch_all() - app.running_context.executor.initialize_threading(app, pids) # The order of these imports matter for initialization (should probably be fixed) diff --git a/walkoff/api/objects/appapi.yaml b/walkoff/api/objects/appapi.yaml index 77a2d724a..0c0606bc7 100644 --- a/walkoff/api/objects/appapi.yaml +++ b/walkoff/api/objects/appapi.yaml @@ -372,4 +372,5 @@ ParameterSchema: type: boolean enum: type: array + items: {} minItems: 1 diff --git a/walkoff/client/src/execution/execution.component.ts b/walkoff/client/src/execution/execution.component.ts index a3ebbc4b3..849edadf7 100644 --- a/walkoff/client/src/execution/execution.component.ts +++ b/walkoff/client/src/execution/execution.component.ts @@ -53,6 +53,7 @@ export class ExecutionComponent implements OnInit, AfterViewChecked, OnDestroy { workflowStatusEventSource: any; actionStatusEventSource: any; + recalculateTableCallback: any; constructor( private executionService: ExecutionService, private authService: AuthService, private cdr: ChangeDetectorRef, @@ -90,6 +91,14 @@ export class ExecutionComponent implements OnInit, AfterViewChecked, OnDestroy { Observable.interval(30000).subscribe(() => { this.recalculateRelativeTimes(); }); + + this.recalculateTableCallback = (e: JQuery.Event) => { + if (this.actionStatusTable && this.actionStatusTable.recalculate) { + this.actionStatusTable.recalculate(); + } + } + + $(document).on('shown.bs.modal', '.actionStatusModal', this.recalculateTableCallback) } /** @@ -115,6 +124,9 @@ export class ExecutionComponent implements OnInit, AfterViewChecked, OnDestroy { if (this.actionStatusEventSource && this.actionStatusEventSource.close) { this.actionStatusEventSource.close(); } + if (this.recalculateTableCallback) { + $(document).off('shown.bs.modal', '.actionStatusModal', this.recalculateTableCallback) + } } /** diff --git a/walkoff/client/src/metrics/metrics.component.ts b/walkoff/client/src/metrics/metrics.component.ts index 124205d68..0de8acb6d 100644 --- a/walkoff/client/src/metrics/metrics.component.ts +++ b/walkoff/client/src/metrics/metrics.component.ts @@ -1,5 +1,4 @@ -import { Component, ViewEncapsulation, OnInit, ViewChild } from '@angular/core'; -import * as _ from 'lodash'; +import { Component, ViewEncapsulation, OnInit, ViewChild, ChangeDetectorRef } from '@angular/core'; import { ToastyService, ToastyConfig } from 'ng2-toasty'; import { Select2OptionData } from 'ng2-select2'; import 'rxjs/add/operator/debounceTime'; @@ -8,10 +7,8 @@ import { DatatableComponent } from '@swimlane/ngx-datatable'; import { MetricsService } from './metrics.service'; import { UtilitiesService } from '../utilities.service'; -import { Playbook } from '../models/playbook/playbook'; import { AppMetric } from '../models/metric/appMetric'; import { WorkflowMetric } from '../models/metric/workflowMetric'; -import { ActionMetric } from '../models/metric/actionMetric'; @Component({ selector: 'metrics-component', @@ -28,13 +25,16 @@ export class MetricsComponent implements OnInit { appFilter: string = ''; workflowMetrics: WorkflowMetric[] = []; availableApps: Select2OptionData[] = []; - appSelectConfig: Select2Options; + appSelectConfig: Select2Options; + recalculateTableCallback: any; - @ViewChild('appMetricsTable') appMetricsTable: DatatableComponent; + @ViewChild('appMetricsTable') appMetricsTable: DatatableComponent; + @ViewChild('workflowMetricsTable') workflowMetricsTable: DatatableComponent; constructor( private metricsService: MetricsService, private toastyService: ToastyService, - private toastyConfig: ToastyConfig, private utils: UtilitiesService, + private toastyConfig: ToastyConfig, private utils: UtilitiesService, + private cdr: ChangeDetectorRef ) {} ngOnInit(): void { @@ -45,10 +45,39 @@ export class MetricsComponent implements OnInit { placeholder: 'Select an App to view its Metrics', }; + this.recalculateTableCallback = (e: JQuery.Event) => { + this.recalculateTable(e); + } + $(document).on('shown.bs.tab', 'a[data-toggle="tab"]', this.recalculateTableCallback) + this.getAppMetrics(); this.getWorkflowMetrics(); } + /** + * Closes our SSEs on component destroy. + */ + ngOnDestroy(): void { + if (this.recalculateTableCallback) { $(document).off('shown.bs.tab', 'a[data-toggle="tab"]', this.recalculateTableCallback); } + } + + /** + * This angular function is used primarily to recalculate column widths for execution results table. + */ + recalculateTable(event: JQuery.Event) : void { + let table: DatatableComponent; + switch(event.target.getAttribute('href')) { + case '#apps': + table = this.appMetricsTable; + break; + case '#workflows': + table = this.workflowMetricsTable; + } + if (table && table.recalculate) { + this.cdr.detectChanges(); + table.recalculate(); + } + } /** * Grabs case events from the server for the selected case (from the JS event supplied). * Will update the case events data table with the new case events. @@ -62,22 +91,6 @@ export class MetricsComponent implements OnInit { this.appFilter = event.value; } } - - test(event: any): void { - //this.appMetricsTable.recalculate(); - //setTimeout(() => this.appMetricsTable.recalculate(), 100) - //console.log(this.appMetricsTable); - } - - /** - * This angular function is used primarily to recalculate column widths for execution results table. - */ - ngAfterViewChecked(): void { - // Check if the table size has changed, - if (this.appMetricsTable && this.appMetricsTable.recalculate) { - this.appMetricsTable.recalculate(); - } - } getFilteredAppMetrics(): AppMetric[] { if (!this.appFilter || this.appFilter == 'all') return this.appMetrics; diff --git a/walkoff/client/src/playbook/playbook.argument.component.ts b/walkoff/client/src/playbook/playbook.argument.component.ts index ae88331de..d20fbb0e2 100644 --- a/walkoff/client/src/playbook/playbook.argument.component.ts +++ b/walkoff/client/src/playbook/playbook.argument.component.ts @@ -42,7 +42,7 @@ export class PlaybookArgumentComponent implements OnInit { parameterSchema: ParameterSchema; selectData: Select2OptionData[]; selectConfig: Select2Options; - selectInitialValue: number[]; + selectInitialValue: string[]; // tslint:disable-next-line:no-empty constructor() { } @@ -56,11 +56,11 @@ export class PlaybookArgumentComponent implements OnInit { */ ngOnInit(): void { this.initParameterSchema(); - this.initUserSelect(); - this.initRoleSelect(); this.initDeviceSelect() this.initBranchCounterSelect(); this.initTypeSelector(); + this.initUserSelect(); + this.initRoleSelect(); } initDeviceSelect(): void { @@ -130,14 +130,16 @@ export class PlaybookArgumentComponent implements OnInit { placeholder: 'Select user', }; + this.selectInitialValue = JSON.parse(JSON.stringify(this.argument.value)); + if (this.parameterSchema.type === 'array') { this.selectConfig.placeholder += '(s)'; this.selectConfig.multiple = true; this.selectConfig.allowClear = true; this.selectConfig.closeOnSelect = false; + if (Array.isArray(this.argument.value)) + this.selectInitialValue = this.argument.value.map((val: number) => val.toString()); } - - this.selectInitialValue = JSON.parse(JSON.stringify(this.argument.value)); } /** @@ -155,14 +157,16 @@ export class PlaybookArgumentComponent implements OnInit { placeholder: 'Select role', }; + this.selectInitialValue = JSON.parse(JSON.stringify(this.argument.value)); + if (this.parameterSchema.type === 'array') { this.selectConfig.placeholder += '(s)'; this.selectConfig.multiple = true; this.selectConfig.allowClear = true; this.selectConfig.closeOnSelect = false; + if (Array.isArray(this.argument.value)) + this.selectInitialValue = this.argument.value.map((val: number) => val.toString()); } - - this.selectInitialValue = JSON.parse(JSON.stringify(this.argument.value)); } initBranchCounterSelect(): void { diff --git a/walkoff/client/src/playbook/playbook.argument.html b/walkoff/client/src/playbook/playbook.argument.html index 03bc258ea..2060079a7 100644 --- a/walkoff/client/src/playbook/playbook.argument.html +++ b/walkoff/client/src/playbook/playbook.argument.html @@ -35,11 +35,11 @@ - - diff --git a/walkoff/client/src/playbook/playbook.component.ts b/walkoff/client/src/playbook/playbook.component.ts index ce832e372..a723c6b83 100644 --- a/walkoff/client/src/playbook/playbook.component.ts +++ b/walkoff/client/src/playbook/playbook.component.ts @@ -56,7 +56,9 @@ export class PlaybookComponent implements OnInit, AfterViewChecked, OnDestroy { @ViewChild('workflowResultsContainer') workflowResultsContainer: ElementRef; @ViewChild('workflowResultsTable') workflowResultsTable: DatatableComponent; @ViewChild('consoleContainer') consoleContainer: ElementRef; - @ViewChild('consoleTable') consoleTable: DatatableComponent; + @ViewChild('consoleTable') consoleTable: DatatableComponent; + @ViewChild('errorLogContainer') errorLogContainer: ElementRef; + @ViewChild('errorLogTable') errorLogTable: DatatableComponent; devices: Device[] = []; relevantDevices: Device[] = []; @@ -88,6 +90,7 @@ export class PlaybookComponent implements OnInit, AfterViewChecked, OnDestroy { eventSource: any; consoleEventSource: any; playbookToImport: File; + recalculateConsoleTableCallback: any; // Simple bootstrap modal params modalParams: { @@ -143,6 +146,9 @@ export class PlaybookComponent implements OnInit, AfterViewChecked, OnDestroy { Observable.interval(30000).subscribe(() => { this.recalculateRelativeTimes(); }); + + this.recalculateConsoleTableCallback = (e: JQuery.Event) => this.recalculateConsoleTable(e); + $(document).on('shown.bs.tab', 'a[data-toggle="tab"]', this.recalculateConsoleTableCallback) } /** @@ -164,6 +170,7 @@ export class PlaybookComponent implements OnInit, AfterViewChecked, OnDestroy { ngOnDestroy(): void { if (this.eventSource && this.eventSource.close) { this.eventSource.close(); } if (this.consoleEventSource && this.consoleEventSource.close) { this.consoleEventSource.close(); } + if (this.recalculateConsoleTableCallback) { $(document).off('shown.bs.tab', 'a[data-toggle="tab"]', this.recalculateConsoleTableCallback); } } ///------------------------------------------------------------------------------------------------------ @@ -1693,4 +1700,25 @@ export class PlaybookComponent implements OnInit, AfterViewChecked, OnDestroy { if (!this.loadedWorkflow) return []; return this.loadedWorkflow.all_errors.map(error => ({ error })); } + + /** + * This function is used primarily to recalculate column widths for execution results table. + */ + recalculateConsoleTable(e: JQuery.Event) { + let table: DatatableComponent; + switch(e.target.getAttribute('href')) { + case '#console': + table = this.consoleTable; + break; + case '#executionLog': + table = this.workflowResultsTable; + break; + case '#errorLog': + table = this.errorLogTable; + } + if (table && table.recalculate) { + this.cdr.detectChanges(); + table.recalculate(); + } + } } diff --git a/walkoff/config.py b/walkoff/config.py index 5e14fdad6..f2c82f5a9 100644 --- a/walkoff/config.py +++ b/walkoff/config.py @@ -136,6 +136,7 @@ class Config(object): JWT_TOKEN_LOCATION = 'headers' JWT_BLACKLIST_PRUNE_FREQUENCY = 1000 + MAX_STREAM_RESULTS_SIZE_KB = 156 @classmethod def load_config(cls, config_path=None): diff --git a/walkoff/server/app.py b/walkoff/server/app.py index 609d3d331..0fba5ae3f 100644 --- a/walkoff/server/app.py +++ b/walkoff/server/app.py @@ -10,6 +10,7 @@ from walkoff.helpers import import_submodules from walkoff.server import context from walkoff.server.blueprints import custominterface, workflowresults, notifications, console, root +import walkoff.server.workflowresults logger = logging.getLogger(__name__) diff --git a/walkoff/server/blueprints/workflowresults.py b/walkoff/server/blueprints/workflowresults.py index 4829ecc52..d161ac201 100644 --- a/walkoff/server/blueprints/workflowresults.py +++ b/walkoff/server/blueprints/workflowresults.py @@ -1,4 +1,6 @@ from datetime import datetime +import sys +import json from flask import current_app @@ -30,7 +32,14 @@ def format_action_data(sender, kwargs, status): def format_action_data_with_results(sender, kwargs, status): result = format_action_data(sender, kwargs, status) - result['result'] = kwargs['data']['data']['result'] + action_result = kwargs['data']['data']['result'] + with current_app.app_context(): + max_len = current_app.config['MAX_STREAM_RESULTS_SIZE_KB'] * 1024 + result_str = str(action_result) + if len(result_str) > max_len: + result['result'] = {'truncated': result_str[:max_len]} + else: + result['result'] = action_result return result diff --git a/walkoff/sse.py b/walkoff/sse.py index 675eecd32..02f2f8f38 100644 --- a/walkoff/sse.py +++ b/walkoff/sse.py @@ -214,6 +214,8 @@ def send(self, retry=None, **kwargs): event_id = 0 for response in channel_queue.listen(): + if response == 1: + continue if isinstance(response, binary_type): response = response.decode('utf-8') response = json.loads(response)