From 91eb7bb9543020df30d4279572ac5b9acf65085a Mon Sep 17 00:00:00 2001 From: yshepilov Date: Sun, 28 Feb 2021 18:35:44 +0100 Subject: [PATCH] #245 #341: added html and html_iframe output types + fixed disabling bash formatting --- samples/configs/destroy_world.json | 3 +- samples/configs/ploty_html_output.json | 7 + samples/configs/simple_html_output.json | 7 + samples/scripts/html_output_test.py | 214 ++++++++++++++++++ samples/scripts/ploty_html_output.py | 9 + src/config/config_service.py | 7 +- src/config/exceptions.py | 3 + src/migrations/migrate.py | 19 ++ src/model/external_model.py | 3 +- src/model/script_config.py | 29 ++- src/tests/config_service_test.py | 4 +- src/tests/script_config_test.py | 30 ++- src/tests/web/script_config_socket_test.py | 2 +- src/web/server.py | 3 +- .../scripts-config/ScriptConfigForm.vue | 17 +- .../scripts-config/script-fields.js | 10 +- web-src/src/common/components/log_panel.vue | 101 ++++++--- .../terminal/ansi/TerminalOutput.js | 30 +++ .../terminal/{ => ansi}/terminal_model.js | 0 .../terminal/{ => ansi}/terminal_view.js | 0 .../terminal/html/HtmlIFrameOutput.js | 29 +++ .../components/terminal/html/HtmlOutput.js | 26 +++ .../components/terminal/text/TextOutput.js | 23 ++ .../components/scripts/script-view.vue | 5 +- web-src/tests/unit/admin/ScriptConfig_test.js | 15 +- web-src/tests/unit/terminal_model_test.js | 2 +- web-src/tests/unit/terminal_view_test.js | 4 +- 27 files changed, 535 insertions(+), 67 deletions(-) create mode 100644 samples/configs/ploty_html_output.json create mode 100644 samples/configs/simple_html_output.json create mode 100755 samples/scripts/html_output_test.py create mode 100644 samples/scripts/ploty_html_output.py create mode 100644 src/config/exceptions.py create mode 100644 web-src/src/common/components/terminal/ansi/TerminalOutput.js rename web-src/src/common/components/terminal/{ => ansi}/terminal_model.js (100%) rename web-src/src/common/components/terminal/{ => ansi}/terminal_view.js (100%) create mode 100644 web-src/src/common/components/terminal/html/HtmlIFrameOutput.js create mode 100644 web-src/src/common/components/terminal/html/HtmlOutput.js create mode 100644 web-src/src/common/components/terminal/text/TextOutput.js diff --git a/samples/configs/destroy_world.json b/samples/configs/destroy_world.json index 1e7f66ea..9c3a524d 100644 --- a/samples/configs/destroy_world.json +++ b/samples/configs/destroy_world.json @@ -2,5 +2,6 @@ "name": "destroy_world", "script_path": "./samples/scripts/destroy_world.py", "description": "This is a very dangerous script, please be careful when running. Don't forget your protective helmet.", - "requires_terminal": false + "requires_terminal": false, + "output_format": "text" } \ No newline at end of file diff --git a/samples/configs/ploty_html_output.json b/samples/configs/ploty_html_output.json new file mode 100644 index 00000000..66de673e --- /dev/null +++ b/samples/configs/ploty_html_output.json @@ -0,0 +1,7 @@ +{ + "name": "Ploty HTML output", + "script_path": "./samples/scripts/ploty_html_output.py", + "description": "A sample script which outputs html data from ploty", + "output_format": "html_iframe", + "group": "HTML" +} \ No newline at end of file diff --git a/samples/configs/simple_html_output.json b/samples/configs/simple_html_output.json new file mode 100644 index 00000000..34f36435 --- /dev/null +++ b/samples/configs/simple_html_output.json @@ -0,0 +1,7 @@ +{ + "name": "Simple HTML output", + "script_path": "./samples/scripts/html_output_test.py", + "description": "A sample script which outputs simple html data\nCredits: https://www.smashingmagazine.com/2009/08/designing-a-html-5-layout-from-scratch/", + "group": "HTML", + "output_format": "html" +} \ No newline at end of file diff --git a/samples/scripts/html_output_test.py b/samples/scripts/html_output_test.py new file mode 100755 index 00000000..c0cc198b --- /dev/null +++ b/samples/scripts/html_output_test.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 + +text = ''' +
+ + + + +
+
    +
  1. +
    +

    + This be the title +

    + +
    + + + +
    + +

    Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque venenatis nunc vitae libero iaculis elementum. Nullam et justo non sapien dapibus blandit nec et leo. Ut ut malesuada tellus.

    +
    +
    2. + +
  2. + ... +
    3. + +
  3. + ... +
    4. + +
+
+ +
+
+

blogroll

+ + +
+ + +
+ +''' + +print(text) diff --git a/samples/scripts/ploty_html_output.py b/samples/scripts/ploty_html_output.py new file mode 100644 index 00000000..d5f995f1 --- /dev/null +++ b/samples/scripts/ploty_html_output.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 + +import plotly.express as px + +df = px.data.iris() +fig = px.scatter(df, x="sepal_width", y="sepal_length", color="species", size='petal_length', + hover_data=['petal_width']) +html_output = fig.to_html() +print(html_output) # warning long output (3298755 characters) diff --git a/src/config/config_service.py b/src/config/config_service.py index 038e0f44..15a94a2c 100644 --- a/src/config/config_service.py +++ b/src/config/config_service.py @@ -3,6 +3,7 @@ import os import re +from config.exceptions import InvalidConfigException from model import script_config from model.model_helper import InvalidFileException from model.script_config import get_sorted_config @@ -196,8 +197,7 @@ def _load_script_config(self, path, content_or_json_dict, user, parameter_values path, user.get_username(), user.get_audit_name(), - pty_enabled_default=os_utils.is_pty_supported(), - ansi_enabled_default=os_utils.is_linux() or os_utils.is_mac()) + pty_enabled_default=os_utils.is_pty_supported()) if parameter_values is not None: config.set_all_param_values(parameter_values, skip_invalid_parameters) @@ -225,6 +225,3 @@ def __init__(self, message): super().__init__(message) -class InvalidConfigException(Exception): - def __init__(self, message): - super().__init__(message) diff --git a/src/config/exceptions.py b/src/config/exceptions.py new file mode 100644 index 00000000..81f4545a --- /dev/null +++ b/src/config/exceptions.py @@ -0,0 +1,3 @@ +class InvalidConfigException(Exception): + def __init__(self, message): + super().__init__(message) diff --git a/src/migrations/migrate.py b/src/migrations/migrate.py index 4badedb0..2514111d 100644 --- a/src/migrations/migrate.py +++ b/src/migrations/migrate.py @@ -8,6 +8,7 @@ import execution.logging from execution.logging import ExecutionLoggingService +from model import model_helper from utils import file_utils from utils.date_utils import sec_to_datetime, to_millis from utils.string_utils import is_blank @@ -261,6 +262,24 @@ def __migrate_output_files_parameters_substitution(context): _write_json(conf_file, json_object, content) +# 1.16 -> 1.17 migration +@_migration('migrate_bash_formatting_to_output_format') +def __migrate_bash_formatting_to_output_format(context): + for (conf_file, json_object, content) in _load_runner_files(context.conf_folder): + if 'bash_formatting' not in json_object: + continue + + if model_helper.read_bool_from_config('bash_formatting', json_object, default=True) is False: + output_format = 'text' + else: + output_format = 'terminal' + + del json_object['bash_formatting'] + json_object['output_format'] = output_format + + _write_json(conf_file, json_object, content) + + def _write_json(file_path, json_object, old_content): space_matches = re.findall('^\s+', old_content, flags=re.MULTILINE) if space_matches: diff --git a/src/model/external_model.py b/src/model/external_model.py index 0db35639..02e9390a 100644 --- a/src/model/external_model.py +++ b/src/model/external_model.py @@ -25,7 +25,8 @@ def config_to_external(config, id, external_id=None): 'name': config.name, 'description': config.description, 'schedulable': config.schedulable, - 'parameters': parameters + 'parameters': parameters, + 'outputFormat': config.output_format } diff --git a/src/model/script_config.py b/src/model/script_config.py index 42bdc08c..6566cbb2 100644 --- a/src/model/script_config.py +++ b/src/model/script_config.py @@ -5,6 +5,7 @@ from collections import OrderedDict from auth.authorization import ANY_USER +from config.exceptions import InvalidConfigException from model import parameter_config from model.model_helper import is_empty, fill_parameter_values, read_bool_from_config, InvalidValueException, \ read_str_from_config, replace_auth_vars @@ -13,6 +14,8 @@ from utils import file_utils from utils.object_utils import merge_dicts +OUTPUT_FORMATS = ['terminal', 'html', 'html_iframe', 'text'] + LOGGER = logging.getLogger('script_server.script_config') @@ -29,7 +32,7 @@ def __init__(self): 'description', 'requires_terminal', 'working_directory', - 'ansi_enabled', + 'output_format', 'output_files', 'schedulable', '_included_config') @@ -40,13 +43,11 @@ def __init__(self, path, username, audit_name, - pty_enabled_default=True, - ansi_enabled_default=True): + pty_enabled_default=True): super().__init__() short_config = read_short(path, config_object) self.name = short_config.name - self._ansi_enabled_default = ansi_enabled_default self._pty_enabled_default = pty_enabled_default self._config_folder = os.path.dirname(path) @@ -172,8 +173,7 @@ def _reload_config(self): required_terminal = read_bool_from_config('requires_terminal', config, default=self._pty_enabled_default) self.requires_terminal = required_terminal - ansi_enabled = read_bool_from_config('bash_formatting', config, default=self._ansi_enabled_default) - self.ansi_enabled = ansi_enabled + self.output_format = read_output_format(config) self.output_files = config.get('output_files', []) @@ -377,7 +377,10 @@ def get_sorted_config(config): 'admin_users', 'schedulable', 'include', - 'output_files', 'requires_terminal', 'bash_formatting', 'parameters'] + 'output_files', + 'requires_terminal', + 'output_format', + 'parameters'] def get_order(key): if key == 'parameters': @@ -393,3 +396,15 @@ def get_order(key): config['parameters'][i] = parameter_config.get_sorted_config(param) return sorted_config + + +def read_output_format(config): + output_format = config.get('output_format') + if not output_format: + output_format = 'terminal' + + output_format = output_format.strip().lower() + if output_format not in OUTPUT_FORMATS: + raise InvalidConfigException('Invalid output format, should be one of: ' + str(OUTPUT_FORMATS)) + + return output_format diff --git a/src/tests/config_service_test.py b/src/tests/config_service_test.py index dfae4d14..ac3cb8bd 100644 --- a/src/tests/config_service_test.py +++ b/src/tests/config_service_test.py @@ -5,8 +5,8 @@ from auth.authorization import Authorizer, EmptyGroupProvider from auth.user import User -from config.config_service import ConfigService, ConfigNotAllowedException, AdminAccessRequiredException, \ - InvalidConfigException +from config.config_service import ConfigService, ConfigNotAllowedException, AdminAccessRequiredException +from config.exceptions import InvalidConfigException from model.model_helper import InvalidFileException from tests import test_utils from tests.test_utils import AnyUserAuthorizer diff --git a/src/tests/script_config_test.py b/src/tests/script_config_test.py index 71f0d1ad..f4088203 100644 --- a/src/tests/script_config_test.py +++ b/src/tests/script_config_test.py @@ -2,7 +2,10 @@ import unittest from collections import OrderedDict +from parameterized import parameterized + from config.constants import PARAM_TYPE_SERVER_FILE, PARAM_TYPE_MULTISELECT +from config.exceptions import InvalidConfigException from model.script_config import ConfigModel, InvalidValueException, _TemplateProperty, ParameterNotFoundException, \ get_sorted_config from react.properties import ObservableDict, ObservableList @@ -25,7 +28,7 @@ def test_create_full_config(self): description = 'A script for test_create_full_config' working_directory = '/root' requires_terminal = False - bash_formatting = True + output_format = 'terminal' output_files = ['file1', 'file2'] config_model = _create_config_model(name, config={ @@ -33,7 +36,7 @@ def test_create_full_config(self): 'description': description, 'working_directory': working_directory, 'requires_terminal': requires_terminal, - 'bash_formatting': bash_formatting, + 'output_format': output_format, 'output_files': output_files, 'scheduling': {'enabled': True}}) @@ -42,7 +45,7 @@ def test_create_full_config(self): self.assertEqual(description, config_model.description) self.assertEqual(working_directory, config_model.working_directory) self.assertEqual(requires_terminal, config_model.requires_terminal) - self.assertEqual(bash_formatting, config_model.ansi_enabled) + self.assertEqual(output_format, config_model.output_format) self.assertEqual(output_files, config_model.output_files) self.assertTrue(config_model.schedulable) @@ -93,6 +96,27 @@ def test_create_with_no_value_dependant_parameter(self): self.assertRaisesRegex(Exception, 'Unsupported parameter "p2"', _create_config_model, 'conf', parameters=parameters) + @parameterized.expand([ + ('html', 'html'), + ('terminal', 'terminal'), + ('', 'terminal'), + ('HTML_iframe', 'html_iframe'), + (' Text ', 'text'), + ]) + def test_create_with_output_format(self, output_format, expected_output_format): + name = 'conf_y' + + config_model = _create_config_model(name, config={ + 'output_format': output_format}) + + self.assertEqual(expected_output_format, config_model.output_format) + + def test_create_with_wrong_output_format(self): + name = 'conf_y' + + self.assertRaisesRegex(InvalidConfigException, 'Invalid output format', _create_config_model, name, config={ + 'output_format': 'abc'}) + class ConfigModelValuesTest(unittest.TestCase): diff --git a/src/tests/web/script_config_socket_test.py b/src/tests/web/script_config_socket_test.py index 0712873e..c206a6f0 100644 --- a/src/tests/web/script_config_socket_test.py +++ b/src/tests/web/script_config_socket_test.py @@ -67,7 +67,7 @@ def assert_model(self, response, event_type, external_model_id=None, list2_value self.assertEqual( {'event': event_type, - 'data': {'clientModelId': external_model_id, 'name': 'Test script 1', + 'data': {'clientModelId': external_model_id, 'name': 'Test script 1', 'outputFormat': 'terminal', 'description': None, 'schedulable': False, 'parameters': [ {'name': 'text 1', 'description': None, 'withoutValue': False, 'required': True, 'default': None, 'type': 'text', 'min': None, 'max': None, 'max_length': None, 'values': None, 'secure': False, diff --git a/src/web/server.py b/src/web/server.py index e9a700f7..cf401f0c 100755 --- a/src/web/server.py +++ b/src/web/server.py @@ -19,7 +19,8 @@ from auth.identification import AuthBasedIdentification, IpBasedIdentification from auth.tornado_auth import TornadoAuth from communications.alerts_service import AlertsService -from config.config_service import ConfigService, ConfigNotAllowedException, InvalidConfigException +from config.config_service import ConfigService, ConfigNotAllowedException +from config.exceptions import InvalidConfigException from execution.execution_service import ExecutionService from execution.logging import ExecutionLoggingService from features.file_download_feature import FileDownloadFeature diff --git a/web-src/src/admin/components/scripts-config/ScriptConfigForm.vue b/web-src/src/admin/components/scripts-config/ScriptConfigForm.vue index d1dd828f..cdcbdf36 100644 --- a/web-src/src/admin/components/scripts-config/ScriptConfigForm.vue +++ b/web-src/src/admin/components/scripts-config/ScriptConfigForm.vue @@ -32,7 +32,7 @@
- + @@ -49,20 +49,21 @@ import {forEachKeyValue, isEmptyArray, isEmptyString, isNull} from '@/common/uti import get from 'lodash/get'; import { allowAllField, - bashFormattingField, descriptionField, groupField, includeScriptField, nameField, + outputFormatField, requiresTerminalField, scriptPathField, workDirField } from './script-fields'; -import {allowAllAdminsField} from "@/admin/components/scripts-config/script-fields"; +import {allowAllAdminsField} from '@/admin/components/scripts-config/script-fields'; +import Combobox from '@/common/components/combobox' export default { name: 'ScriptConfigForm', - components: {TextArea, ChipsList, TextField, CheckBox}, + components: {Combobox, TextArea, ChipsList, TextField, CheckBox}, props: { value: { @@ -81,7 +82,7 @@ export default { workingDirectory: null, requiresTerminal: null, includeScript: null, - bashFormatting: null, + outputFormat: null, allowedUsers: [], allowAllUsers: true, adminUsers: [], @@ -93,7 +94,7 @@ export default { descriptionField, allowAllField, allowAllAdminsField, - bashFormattingField, + outputFormatField, requiresTerminalField, includeScriptField } @@ -108,7 +109,7 @@ export default { 'workingDirectory': 'working_directory', 'requiresTerminal': 'requires_terminal', 'includeScript': 'include', - 'bashFormatting': 'bash_formatting' + 'outputFormat': 'output_format' }; forEachKeyValue(simpleFields, (vmField, configField) => { @@ -133,7 +134,7 @@ export default { this.workingDirectory = config['working_directory']; this.requiresTerminal = get(config, 'requires_terminal', true); this.includeScript = config['include']; - this.bashFormatting = get(config, 'bash_formatting', true); + this.outputFormat = config['output_format']; this.updateAccessFieldInVm(config, 'allowedUsers', 'allowAllUsers', diff --git a/web-src/src/admin/components/scripts-config/script-fields.js b/web-src/src/admin/components/scripts-config/script-fields.js index 3aa9b1da..31ab9fda 100644 --- a/web-src/src/admin/components/scripts-config/script-fields.js +++ b/web-src/src/admin/components/scripts-config/script-fields.js @@ -22,12 +22,14 @@ export const allowAllField = { export const allowAllAdminsField = { name: 'Any admin' }; -export const bashFormattingField = { - name: 'Bash formatting', - description: 'Enable ANSI escape sequences for text formatting and cursor moves' +export const outputFormatField = { + name: 'Output format', + description: 'Specifies in which format the script outputs data ' + + '(terminal, plain text, html tags, or an extended html page (iframe)', + values: ['terminal', 'text', 'html', 'html_iframe'] }; export const requiresTerminalField = { - name: 'Requires terminal', + name: 'Enable pseudo-terminal', description: 'Enables pseudo-terminal. ' + 'This is need for some utilities which behave differently, when executed from terminal' }; diff --git a/web-src/src/common/components/log_panel.vue b/web-src/src/common/components/log_panel.vue index 1ae89987..ba24165f 100644 --- a/web-src/src/common/components/log_panel.vue +++ b/web-src/src/common/components/log_panel.vue @@ -14,14 +14,20 @@