From 57ebff01fec8a5f72ecdf7808fed759e9aa77abd Mon Sep 17 00:00:00 2001 From: Matt Shin Date: Wed, 17 Jan 2018 14:53:38 +0000 Subject: [PATCH 01/21] Fix reload with stop point Reload will no longer reset existing stop point. --- lib/cylc/scheduler.py | 13 +++---- tests/reload/20-stop-point.t | 43 ++++++++++++++++++++++++ tests/reload/20-stop-point/reference.log | 7 ++++ tests/reload/20-stop-point/suite.rc | 33 ++++++++++++++++++ 4 files changed, 88 insertions(+), 8 deletions(-) create mode 100755 tests/reload/20-stop-point.t create mode 100644 tests/reload/20-stop-point/reference.log create mode 100644 tests/reload/20-stop-point/suite.rc diff --git a/lib/cylc/scheduler.py b/lib/cylc/scheduler.py index ec9b531011..d998faec6d 100644 --- a/lib/cylc/scheduler.py +++ b/lib/cylc/scheduler.py @@ -864,7 +864,11 @@ def command_reload_suite(self): self.task_events_mgr.broadcast_mgr.linearized_ancestors = ( self.config.get_linearized_ancestors()) self.suite_db_mgr.put_runtime_inheritance(self.config) - self.pool.set_do_reload(self.config, self.final_point) + if self.stop_point is None: + stop_point = self.final_point + else: + stop_point = self.stop_point + self.pool.set_do_reload(self.config, stop_point) self.task_events_mgr.mail_interval = self._get_cylc_conf( "task event mail interval") self.task_events_mgr.mail_footer = self._get_events_conf("mail footer") @@ -1598,13 +1602,6 @@ def release_suite(self): sdm = self.suite_db_mgr sdm.db_deletes_map[sdm.TABLE_SUITE_PARAMS].append({"key": "is_held"}) - def clear_stop_times(self): - """Clear attributes associated with stop time.""" - self.stop_point = None - self.stop_clock_time = None - self.stop_clock_time_string = None - self.stop_task = None - def paused(self): """Is the suite paused?""" return self.pool.is_held diff --git a/tests/reload/20-stop-point.t b/tests/reload/20-stop-point.t new file mode 100755 index 0000000000..f0822b607e --- /dev/null +++ b/tests/reload/20-stop-point.t @@ -0,0 +1,43 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2018 NIWA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#------------------------------------------------------------------------------- +# Test set stop point then reload. Reload should not reset stop point. +. "$(dirname "$0")/test_header" + +set_test_number 3 +install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" + +run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}" +suite_run_ok "${TEST_NAME_BASE}-run" \ + cylc run --debug --no-detach --reference-test "${SUITE_NAME}" +if ! which sqlite3 > /dev/null; then + skip 1 "sqlite3 not installed?" + purge_suite "${SUITE_NAME}" + exit 0 +fi +sqlite3 "${SUITE_RUN_DIR}/.service/db" \ + 'SELECT cycle,name,run_status FROM task_jobs' | sort >'db.out' +cmp_ok 'db.out' <<'__OUT__' +1|reload|0 +1|set-stop-point|0 +1|t1|0 +2|t1|0 +3|t1|0 +__OUT__ + +purge_suite "${SUITE_NAME}" +exit diff --git a/tests/reload/20-stop-point/reference.log b/tests/reload/20-stop-point/reference.log new file mode 100644 index 0000000000..c8c405c807 --- /dev/null +++ b/tests/reload/20-stop-point/reference.log @@ -0,0 +1,7 @@ +2016-07-08T14:53:12+01 INFO - Initial point: 1 +2016-07-08T14:53:12+01 INFO - Final point: 5 +2016-07-08T14:53:12+01 INFO - [set-stop-point.1] -triggered off [] +2016-07-08T14:53:19+01 INFO - [reload.1] -triggered off ['set-stop-point.1'] +2016-07-08T14:53:19+01 INFO - [t1.1] -triggered off ['reload.1'] +2016-07-08T14:53:19+01 INFO - [t1.2] -triggered off ['t1.1'] +2016-07-08T14:53:19+01 INFO - [t1.3] -triggered off ['t1.2'] diff --git a/tests/reload/20-stop-point/suite.rc b/tests/reload/20-stop-point/suite.rc new file mode 100644 index 0000000000..3d30241ebe --- /dev/null +++ b/tests/reload/20-stop-point/suite.rc @@ -0,0 +1,33 @@ +#!Jinja2 +[cylc] + [[events]] + abort on stalled = True + [[reference test]] + live mode suite timeout = PT1M + required run mode = live +[scheduling] + cycling mode = integer + initial cycle point = 1 + final cycle point = 5 + [[dependencies]] + [[[R1]]] + graph="set-stop-point => reload => t1" + [[[P1]]] + graph="t1[-P1] => t1" + +[runtime] + [[set-stop-point]] + script=cylc stop "${CYLC_SUITE_NAME}" '3' + [[reload]] + script=""" +wait "${CYLC_TASK_MESSAGE_STARTED_PID}" 2>/dev/null || true +cylc reload "${CYLC_SUITE_NAME}" +LOG="${CYLC_SUITE_LOG_DIR}/log" +while ! grep -q 'Reload completed' "${LOG}"; do + sleep 1 # make sure reload completes +done +""" + [[[job]]] + execution time limit = PT1M + [[t1]] + script=true From 1b9f247f4c0dc8dbc8e3f1d336bde29612d0df7e Mon Sep 17 00:00:00 2001 From: Matt Shin Date: Thu, 18 Jan 2018 09:03:33 +0000 Subject: [PATCH 02/21] CUG: doc param logical quirk --- doc/src/cylc-user-guide/cug.tex | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/src/cylc-user-guide/cug.tex b/doc/src/cylc-user-guide/cug.tex index e338a79f9b..426a12f8a0 100644 --- a/doc/src/cylc-user-guide/cug.tex +++ b/doc/src/cylc-user-guide/cug.tex @@ -4779,6 +4779,20 @@ \subsubsection{Parameter Offsets In The Graph} graph = "proc_small => proc_big => proc_huge" \end{lstlisting} +However, a mixture of conditional logic in these statements may create +surprises. For example: + +\begin{lstlisting} + graph = foo & baz => foo # for m = cat, dog +\end{lstlisting} +currently expands to: +\begin{lstlisting} + graph = foo_cat & baz => foo_dog +# when users may expect it to be: +# graph = foo_cat => foo_dog +# graph = baz => foo_cat & foo_dog +\end{lstlisting} + \subsubsection{Task Families And Parameterization} Task family members can be generated by parameter expansion: From fbd1da6d87d7010d99326e2ab43d3adfd58b82ef Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Sat, 20 Jan 2018 13:36:50 +1300 Subject: [PATCH 03/21] Fix recent pflag bug. --- lib/cylc/task_events_mgr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cylc/task_events_mgr.py b/lib/cylc/task_events_mgr.py index bb5859795a..f57948045f 100644 --- a/lib/cylc/task_events_mgr.py +++ b/lib/cylc/task_events_mgr.py @@ -361,12 +361,12 @@ def process_message(self, itask, severity, message, poll_func, except (TypeError, ValueError): itask.timeout_timers[TASK_STATUS_SUBMITTED] = None # Believe this and change state without polling (could poll?). - cylc.flags.pflag = True + self.pflag = True itask.state.reset_state(TASK_STATUS_SUBMITTED) elif an_output_was_satisfied: # Message of an as-yet unreported custom task output. # No state change. - cylc.flags.pflag = True + self.pflag = True self.suite_db_mgr.put_update_task_outputs(itask) else: # Unhandled messages. These include: From 296346482a6508addcf363478ddac1e31ae1fda5 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Sun, 21 Jan 2018 11:44:37 +1300 Subject: [PATCH 04/21] New test for message triggers. --- tests/message-triggers/02-action.t | 48 +++++++++++++++++++++++ tests/message-triggers/02-action/suite.rc | 45 +++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 tests/message-triggers/02-action.t create mode 100644 tests/message-triggers/02-action/suite.rc diff --git a/tests/message-triggers/02-action.t b/tests/message-triggers/02-action.t new file mode 100644 index 0000000000..f4de5bf768 --- /dev/null +++ b/tests/message-triggers/02-action.t @@ -0,0 +1,48 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2018 NIWA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#------------------------------------------------------------------------------- +# Test that message triggers (custom outputs) are actioned immediately even if +# nothing else is happening at the time - GitHub #2548. + +. $(dirname $0)/test_header + +set_test_number 4 + +install_suite $TEST_NAME_BASE $TEST_NAME_BASE + +TEST_NAME=$TEST_NAME_BASE-validate +run_ok $TEST_NAME cylc validate $SUITE_NAME + +# The suite tests that two tasks suicide immediately on message triggers. +TEST_NAME=$TEST_NAME_BASE-run +suite_run_ok $TEST_NAME cylc run --no-detach $SUITE_NAME + +# Check that bar does nothing but suicide. +TEST_NAME=$TEST_NAME_BASE-cmp-bar +cylc cat-log $SUITE_NAME | grep bar.1 | awk '{$1=""; print $0}' > bar.log +cmp_ok bar.log - << __END__ + INFO - [bar.1] -suiciding +__END__ + +# Check that baz does nothing but suicide. +TEST_NAME=$TEST_NAME_BASE-cmp-baz +cylc cat-log $SUITE_NAME | grep baz.1 | awk '{$1=""; print $0}' > baz.log +cmp_ok baz.log - << __END__ + INFO - [baz.1] -suiciding +__END__ + +purge_suite $SUITE_NAME diff --git a/tests/message-triggers/02-action/suite.rc b/tests/message-triggers/02-action/suite.rc new file mode 100644 index 0000000000..04c3cfca42 --- /dev/null +++ b/tests/message-triggers/02-action/suite.rc @@ -0,0 +1,45 @@ +[cylc] + abort if any task fails = True + [[events]] + timeout = PT30S + abort on timeout = True +[scheduling] + [[dependencies]] + graph = """ + foo:fail => bar & baz + foo:a => !bar + foo:b => !baz""" +[runtime] + [[foo]] + # Abort if messages don't result in the expected suite state change. + script = """ +sleep 5 +# There should be 3 task proxies in the suite state. +cylc dump -t $CYLC_SUITE_NAME | sort > log.1 +diff log.1 - << __EOF__ +bar, 1, waiting, unspawned +baz, 1, waiting, unspawned +foo, 1, running, spawned +__EOF__ + +cylc message "the quick brown fox" +sleep 5 +# There should now be 2 task proxies in the suite state. +cylc dump -t $CYLC_SUITE_NAME | sort > log.2 +diff log.2 - << __EOF__ +baz, 1, waiting, unspawned +foo, 1, running, spawned +__EOF__ + +cylc message "jumped over the lazy dog" +sleep 5 +# There should now be only 1 task proxy in the suite state. +cylc dump -t $CYLC_SUITE_NAME | sort > log.3 +diff log.3 - << __EOF__ +foo, 1, running, spawned +__EOF__ +""" + [[[outputs]]] + a = "the quick brown fox" + b = "jumped over the lazy dog" + [[bar, baz]] From ee2d5f239fff7b4256f52c0422255b1d9617b93a Mon Sep 17 00:00:00 2001 From: Matt Shin Date: Mon, 22 Jan 2018 15:13:52 +0000 Subject: [PATCH 05/21] Doc: param logical quirk with heavier statement --- doc/src/cylc-user-guide/cug.tex | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/src/cylc-user-guide/cug.tex b/doc/src/cylc-user-guide/cug.tex index 426a12f8a0..4a191653eb 100644 --- a/doc/src/cylc-user-guide/cug.tex +++ b/doc/src/cylc-user-guide/cug.tex @@ -4779,8 +4779,9 @@ \subsubsection{Parameter Offsets In The Graph} graph = "proc_small => proc_big => proc_huge" \end{lstlisting} -However, a mixture of conditional logic in these statements may create -surprises. For example: +However, a quirk in the current system means that you should avoid mixing +conditional logic in these statements. For example, the following will do the +unexpected: \begin{lstlisting} graph = foo & baz => foo # for m = cat, dog @@ -4793,6 +4794,14 @@ \subsubsection{Parameter Offsets In The Graph} # graph = baz => foo_cat & foo_dog \end{lstlisting} +For the time being, writing out the logic explicitly will give you the correct +graph. + +\begin{lstlisting} + graph = """foo => foo # for m = cat, dog + baz => foo""" +\end{lstlisting} + \subsubsection{Task Families And Parameterization} Task family members can be generated by parameter expansion: From 402d37035f7f26d309fb836a55166edbdf940fc5 Mon Sep 17 00:00:00 2001 From: Matt Shin Date: Mon, 22 Jan 2018 15:53:55 +0000 Subject: [PATCH 06/21] gscan: fix display of group stopped suites --- lib/cylc/gui/scanutil.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/cylc/gui/scanutil.py b/lib/cylc/gui/scanutil.py index b2df184dff..a045705356 100644 --- a/lib/cylc/gui/scanutil.py +++ b/lib/cylc/gui/scanutil.py @@ -518,6 +518,7 @@ def update_suites_info(updater, full_mode=False): dict: {(host, owner, name): suite_info, ...} where each "suite_info" is a dict with keys: KEY_GROUP: group name of suite + KEY_META: suite metadata (new in 7.6) KEY_OWNER: suite owner name KEY_PORT: suite port, for running suites only KEY_STATES: suite state @@ -585,10 +586,12 @@ def update_suites_info(updater, full_mode=False): continue if prev_result.get(KEY_PORT): # A previously running suite is no longer running. - # Get suite info with "cat-state", if possible, and include in the - # results set. + # Get suite info with "ls-checkpoints", if possible, and include in + # the results set. try: - prev_result = _update_stopped_suite_info((host, owner, name)) + prev_result.update( + _update_stopped_suite_info((host, owner, name))) + del prev_result[KEY_PORT] except (IndexError, TypeError, ValueError): continue if prev_result.get(KEY_UPDATE_TIME, 0) > expire_threshold: From b7bf1b859359e1e7be7d4b44bcc303638eec373b Mon Sep 17 00:00:00 2001 From: Sadie Bartholomew Date: Thu, 7 Dec 2017 11:06:42 +0000 Subject: [PATCH 07/21] Add cylc show option for parsing task requisites --- bin/cylc-show | 69 +++++++++++-------- tests/cylc-show/05-complex.t | 7 +- tests/cylc-show/05-complex/expected_output.py | 46 +++++++++++++ 3 files changed, 91 insertions(+), 31 deletions(-) create mode 100644 tests/cylc-show/05-complex/expected_output.py diff --git a/bin/cylc-show b/bin/cylc-show index 0718c26222..aae906c73c 100755 --- a/bin/cylc-show +++ b/bin/cylc-show @@ -29,6 +29,8 @@ if '--use-ssh' in sys.argv[1:]: if remrun(): sys.exit(0) +import json + import cylc.flags from cylc.option_parsers import CylcOptionParser as COP from cylc.network.httpclient import SuiteRuntimeServiceClient @@ -46,6 +48,10 @@ def main(): parser.add_option('--list-prereqs', action="store_true", default=False, help="Print a task's pre-requisites as a list.") + parser.add_option('--json', action="store_true", default=False, + help="Print all tasks and corresponding requisites in " + "JSON format.") + options, args = parser.parse_args() suite = args[0] task_args = args[1:] @@ -77,36 +83,39 @@ def main(): results, bad_items = pclient.get_info( 'get_task_requisites', items=task_ids, list_prereqs=options.list_prereqs) - for task_id, result in sorted(results.items()): - if len(results) > 1: - print "----\nTASK ID: %s" % task_id - if options.list_prereqs: - for prereq in result["prerequisites"]: - print prereq - else: - for key, value in sorted( - result["descriptions"].items(), reverse=True): - print "%s: %s" % (key, value or "(not given)") - - for name, done in [("prerequisites", "satisfied"), - ("outputs", "completed")]: - print '\n%s (- => not %s):' % (name, done) - if not result[name]: - print ' (None)' - for msg, state in result[name]: - if state: - print ' + ' + msg - else: - print ' - ' + msg - - if result["extras"]: - print '\nother:' - for key, value in result["extras"].items(): - print ' o %s ... %s' % (key, value) - for bad_item in bad_items: - sys.stderr.write("No matching tasks found: %s\n" % bad_item) - if bad_items: - sys.exit(1) + if options.json: + print json.dumps(results) + else: + for task_id, result in sorted(results.items()): + if len(results) > 1: + print "----\nTASK ID: %s" % task_id + if options.list_prereqs: + for prereq in result["prerequisites"]: + print prereq + else: + for key, value in sorted( + result["descriptions"].items(), reverse=True): + print "%s: %s" % (key, value or "(not given)") + + for name, done in [("prerequisites", "satisfied"), + ("outputs", "completed")]: + print '\n%s (- => not %s):' % (name, done) + if not result[name]: + print ' (None)' + for msg, state in result[name]: + if state: + print ' + ' + msg + else: + print ' - ' + msg + + if result["extras"]: + print '\nother:' + for key, value in result["extras"].items(): + print ' o %s ... %s' % (key, value) + for bad_item in bad_items: + sys.stderr.write("No matching tasks found: %s\n" % bad_item) + if bad_items: + sys.exit(1) if __name__ == "__main__": diff --git a/tests/cylc-show/05-complex.t b/tests/cylc-show/05-complex.t index b4a7a11037..f7d76e22b4 100644 --- a/tests/cylc-show/05-complex.t +++ b/tests/cylc-show/05-complex.t @@ -18,7 +18,7 @@ # Test cylc show for a basic task. . $(dirname $0)/test_header #------------------------------------------------------------------------------- -set_test_number 6 +set_test_number 8 #------------------------------------------------------------------------------- install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- @@ -60,5 +60,10 @@ e.20000102T0000Z succeeded f.20000101T0000Z succeeded __OUT__ #------------------------------------------------------------------------------- +TEST_NAME="${TEST_NAME_BASE}-short" +run_ok "${TEST_NAME}" cylc show --json "${SUITE_NAME}" 'f.20000102T00Z' +run_ok "${TEST_NAME}" &(cylc show --json "${SUITE_NAME}" 'f.20000102T00Z') \ +| python $TEST_SOURCE_DIR/$TEST_NAME_BASE/expected_output.py +#------------------------------------------------------------------------------- cylc stop "${SUITE_NAME}" --now purge_suite "${SUITE_NAME}" diff --git a/tests/cylc-show/05-complex/expected_output.py b/tests/cylc-show/05-complex/expected_output.py new file mode 100644 index 0000000000..88656595e0 --- /dev/null +++ b/tests/cylc-show/05-complex/expected_output.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +import sys +import json + +output = sys.stdin.read() +exp_output = json.dumps( +{ + "f.20000102T0000Z": { + "prerequisites": [ + ["0 & 1 & (2 | (3 & 4)) & 5", False], + ["\t0 = a.20000102T0000Z succeeded", False], + ["\t1 = b.20000102T0000Z succeeded", False], + ["\t2 = c.20000102T0000Z succeeded", False], + ["\t3 = d.20000102T0000Z succeeded", False], + ["\t4 = e.20000102T0000Z succeeded", False], + ["\t5 = f.20000101T0000Z succeeded", False] + ], + "outputs": [ + ["f.20000102T0000Z submitted", False], + ["f.20000102T0000Z started", False], + ["f.20000102T0000Z succeeded", False] + ], + "extras": {}, + "descriptions": { + "description": "", + "title": "" + } + } +}) + +def is_valid_json(possiblejson): + """Tests if input is a valid JSON data structure""" + try: + valid_json_expression = json.loads(possiblejson) + except ValueError, e: + return False + return True + +if is_valid_json(output): + if cmp(json.loads(output),json.loads(exp_output)) == 0: + sys.exit() + else: + sys.exit("Output in JSON format but not as expected ... not ok") +else: + sys.exit("Output not in JSON format ... not ok") From 92d1143ebc6f5be3b06672e4b212399b5be1c35b Mon Sep 17 00:00:00 2001 From: Sadie Bartholomew Date: Fri, 8 Dec 2017 11:29:02 +0000 Subject: [PATCH 08/21] Responding to feedback RE cylc show parsing option --- tests/cylc-show/05-complex/expected_output.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/cylc-show/05-complex/expected_output.py b/tests/cylc-show/05-complex/expected_output.py index 88656595e0..70b52afd43 100644 --- a/tests/cylc-show/05-complex/expected_output.py +++ b/tests/cylc-show/05-complex/expected_output.py @@ -32,15 +32,15 @@ def is_valid_json(possiblejson): """Tests if input is a valid JSON data structure""" try: - valid_json_expression = json.loads(possiblejson) - except ValueError, e: + json.loads(possiblejson) + except ValueError: return False return True if is_valid_json(output): - if cmp(json.loads(output),json.loads(exp_output)) == 0: + if json.loads(output) == json.loads(exp_output): sys.exit() else: - sys.exit("Output in JSON format but not as expected ... not ok") + sys.exit("Output in JSON format but not as expected") else: - sys.exit("Output not in JSON format ... not ok") + sys.exit("Output not in JSON format") From 141a4c1630a9974e4407c72a4b56f65ed10640e0 Mon Sep 17 00:00:00 2001 From: Sadie Bartholomew Date: Mon, 11 Dec 2017 18:27:14 +0000 Subject: [PATCH 09/21] Make parsing option universal and inclusive of meta items --- bin/cylc-show | 23 +++-- lib/cylc/scheduler.py | 3 +- lib/cylc/taskdef.py | 5 +- tests/cylc-show/00-simple.t | 87 +++++++++++++++++-- tests/cylc-show/05-complex.t | 8 +- tests/cylc-show/05-complex/expected_output.py | 46 ---------- tests/cylc-show/simple/reference.log | 17 ++-- tests/cylc-show/simple/suite.rc | 32 ++++--- tests/lib/bash/test_header | 20 +++++ 9 files changed, 152 insertions(+), 89 deletions(-) delete mode 100644 tests/cylc-show/05-complex/expected_output.py diff --git a/bin/cylc-show b/bin/cylc-show index aae906c73c..253a0db41a 100755 --- a/bin/cylc-show +++ b/bin/cylc-show @@ -49,8 +49,7 @@ def main(): help="Print a task's pre-requisites as a list.") parser.add_option('--json', action="store_true", default=False, - help="Print all tasks and corresponding requisites in " - "JSON format.") + help="Print output in JSON format.") options, args = parser.parse_args() suite = args[0] @@ -64,8 +63,11 @@ def main(): if not task_args: # Print suite info. suite_info = pclient.get_info('get_suite_info') - for key, value in sorted(suite_info.items(), reverse=True): - print '%s: %s' % (key, value or "(not given)") + if options.json: + print json.dumps(suite_info) + else: + for key, value in sorted(suite_info.items(), reverse=True): + print '%s: %s' % (key, value or "(not given)") return task_names = [arg for arg in task_args if TaskID.is_valid_name(arg)] @@ -73,11 +75,14 @@ def main(): if task_names: results = pclient.get_info('get_task_info', names=task_names) - for task_name, result in sorted(results.items()): - if len(results) > 1: - print "----\nTASK NAME: %s" % task_name - for key, value in sorted(result.items(), reverse=True): - print "%s: %s" % (key, value or "(not given)") + if options.json: + print json.dumps(results) + else: + for task_name, result in sorted(results.items()): + if len(results) > 1: + print "----\nTASK NAME: %s" % task_name + for key, value in sorted(result.items(), reverse=True): + print "%s: %s" % (key, value or "(not given)") if task_ids: results, bad_items = pclient.get_info( diff --git a/lib/cylc/scheduler.py b/lib/cylc/scheduler.py index d998faec6d..3f69c8827a 100644 --- a/lib/cylc/scheduler.py +++ b/lib/cylc/scheduler.py @@ -624,8 +624,7 @@ def info_get_task_jobfile_path(self, task_id): def info_get_suite_info(self): """Return a dict containing the suite title and description.""" - return {'title': self.config.cfg['meta']['title'], - 'description': self.config.cfg['meta']['description']} + return self.config.cfg['meta'] def info_get_suite_state_summary(self): """Return the global, task, and family summary data structures.""" diff --git a/lib/cylc/taskdef.py b/lib/cylc/taskdef.py index 11253b1502..0fc174c297 100644 --- a/lib/cylc/taskdef.py +++ b/lib/cylc/taskdef.py @@ -101,10 +101,7 @@ def add_sequence(self, sequence): def describe(self): """Return title and description of the current task.""" - info = {} - for item in 'title', 'description': - info[item] = self.rtconfig['meta'][item] - return info + return self.rtconfig['meta'] def check_for_explicit_cycling(self): """Check for explicitly somewhere. diff --git a/tests/cylc-show/00-simple.t b/tests/cylc-show/00-simple.t index 18e1d60ac1..fa41ade40b 100644 --- a/tests/cylc-show/00-simple.t +++ b/tests/cylc-show/00-simple.t @@ -18,11 +18,11 @@ # Test cylc show for a basic task. . $(dirname $0)/test_header #------------------------------------------------------------------------------- -set_test_number 3 +set_test_number 8 #------------------------------------------------------------------------------- install_suite $TEST_NAME_BASE simple #------------------------------------------------------------------------------- -TEST_SHOW_OUTPUT_PATH="$PWD/$TEST_NAME_BASE-show.stdout" +TEST_SHOW_OUTPUT_PATH="$PWD/$TEST_NAME_BASE-show" #------------------------------------------------------------------------------- TEST_NAME=$TEST_NAME_BASE-validate run_ok $TEST_NAME cylc validate \ @@ -33,21 +33,98 @@ suite_run_ok $TEST_NAME cylc run --reference-test --debug --no-detach \ --set=TEST_OUTPUT_PATH="$TEST_SHOW_OUTPUT_PATH" "$SUITE_NAME" #------------------------------------------------------------------------------- TEST_NAME=$TEST_NAME_BASE-show -contains_ok $TEST_NAME.stdout <<__SHOW_OUTPUT__ +cmp_ok "$TEST_NAME-suite" <<__SHOW_OUTPUT__ title: a test suite +group: (not given) description: the quick brown fox +custom: custard +URL: (not given) +__SHOW_OUTPUT__ + +cmp_ok "$TEST_NAME-task" <<__SHOW_OUTPUT__ title: a task description: jumped over the lazy dog +baz: pub +URL: (not given) +__SHOW_OUTPUT__ + +cmp_ok "$TEST_NAME-taskinstance" <<__SHOW_OUTPUT__ title: a task description: jumped over the lazy dog +baz: pub +URL: (not given) prerequisites (- => not satisfied): - - show.20141106T0900Z succeeded + - show-taskinstance-json.20141106T0900Z succeeded + - show-taskinstance.20141106T0900Z succeeded outputs (- => not completed): - - foo.20141106T0900Z started - foo.20141106T0900Z submitted + - foo.20141106T0900Z started - foo.20141106T0900Z succeeded __SHOW_OUTPUT__ #------------------------------------------------------------------------------- +TEST_NAME=$TEST_NAME_BASE-show-json +cmp_json_ok "$TEST_NAME-suite" "$TEST_NAME-suite" <<__SHOW_OUTPUT__ +{ + "URL":"", + "title":"a test suite", + "group":"", + "description":"the quick brown fox", + "custom":"custard" +} +__SHOW_OUTPUT__ + +cmp_json_ok "$TEST_NAME-task" "$TEST_NAME-task" <<__SHOW_OUTPUT__ +{ + "foo":{ + "URL":"", + "baz":"pub", + "description":"jumped over the lazy dog", + "title":"a task" + } +} +__SHOW_OUTPUT__ + +cmp_json_ok "$TEST_NAME-taskinstance" "$TEST_NAME-taskinstance" \ + <<__SHOW_OUTPUT__ +{ + "foo.20141106T0900Z":{ + "prerequisites":[ + [ + "show-taskinstance-json.20141106T0900Z succeeded", + false + ], + [ + "show-taskinstance.20141106T0900Z succeeded", + false + ] + ], + "outputs":[ + [ + "foo.20141106T0900Z submitted", + false + ], + [ + "foo.20141106T0900Z started", + false + ], + [ + "foo.20141106T0900Z succeeded", + false + ] + ], + "extras":{ + + }, + "descriptions":{ + "URL":"", + "baz":"pub", + "description":"jumped over the lazy dog", + "title":"a task" + } + } +} +__SHOW_OUTPUT__ +#------------------------------------------------------------------------------- purge_suite $SUITE_NAME diff --git a/tests/cylc-show/05-complex.t b/tests/cylc-show/05-complex.t index f7d76e22b4..b29e1ff537 100644 --- a/tests/cylc-show/05-complex.t +++ b/tests/cylc-show/05-complex.t @@ -18,7 +18,7 @@ # Test cylc show for a basic task. . $(dirname $0)/test_header #------------------------------------------------------------------------------- -set_test_number 8 +set_test_number 6 #------------------------------------------------------------------------------- install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" #------------------------------------------------------------------------------- @@ -33,6 +33,7 @@ run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" 'f.20000102T00Z' cmp_ok "${TEST_NAME}.stdout" << '__OUT__' title: (not given) description: (not given) +URL: (not given) prerequisites (- => not satisfied): - 0 & 1 & (2 | (3 & 4)) & 5 @@ -60,10 +61,5 @@ e.20000102T0000Z succeeded f.20000101T0000Z succeeded __OUT__ #------------------------------------------------------------------------------- -TEST_NAME="${TEST_NAME_BASE}-short" -run_ok "${TEST_NAME}" cylc show --json "${SUITE_NAME}" 'f.20000102T00Z' -run_ok "${TEST_NAME}" &(cylc show --json "${SUITE_NAME}" 'f.20000102T00Z') \ -| python $TEST_SOURCE_DIR/$TEST_NAME_BASE/expected_output.py -#------------------------------------------------------------------------------- cylc stop "${SUITE_NAME}" --now purge_suite "${SUITE_NAME}" diff --git a/tests/cylc-show/05-complex/expected_output.py b/tests/cylc-show/05-complex/expected_output.py deleted file mode 100644 index 70b52afd43..0000000000 --- a/tests/cylc-show/05-complex/expected_output.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python - -import sys -import json - -output = sys.stdin.read() -exp_output = json.dumps( -{ - "f.20000102T0000Z": { - "prerequisites": [ - ["0 & 1 & (2 | (3 & 4)) & 5", False], - ["\t0 = a.20000102T0000Z succeeded", False], - ["\t1 = b.20000102T0000Z succeeded", False], - ["\t2 = c.20000102T0000Z succeeded", False], - ["\t3 = d.20000102T0000Z succeeded", False], - ["\t4 = e.20000102T0000Z succeeded", False], - ["\t5 = f.20000101T0000Z succeeded", False] - ], - "outputs": [ - ["f.20000102T0000Z submitted", False], - ["f.20000102T0000Z started", False], - ["f.20000102T0000Z succeeded", False] - ], - "extras": {}, - "descriptions": { - "description": "", - "title": "" - } - } -}) - -def is_valid_json(possiblejson): - """Tests if input is a valid JSON data structure""" - try: - json.loads(possiblejson) - except ValueError: - return False - return True - -if is_valid_json(output): - if json.loads(output) == json.loads(exp_output): - sys.exit() - else: - sys.exit("Output in JSON format but not as expected") -else: - sys.exit("Output not in JSON format") diff --git a/tests/cylc-show/simple/reference.log b/tests/cylc-show/simple/reference.log index f23b621ee3..3a2111c1cd 100644 --- a/tests/cylc-show/simple/reference.log +++ b/tests/cylc-show/simple/reference.log @@ -1,6 +1,11 @@ -2014-11-17T09:01:57Z INFO - Initial point: 20141106T0900Z -2014-11-17T09:01:57Z INFO - Final point: 20141106T0900Z -2014-11-17T09:01:58Z INFO - [show.20141106T0900Z] -triggered off [] -2014-11-17T09:02:03Z INFO - [foo.20141106T0900Z] -triggered off ['show.20141106T0900Z'] -2014-11-17T09:02:06Z INFO - [bar.20141106T0900Z] -triggered off ['foo.20141106T0900Z'] -2014-11-17T09:02:06Z INFO - [baz.20141106T0900Z] -triggered off ['foo.20141106T0900Z'] +2017-12-11T17:23:54Z INFO - Initial point: 20141106T0900Z +2017-12-11T17:23:54Z INFO - Final point: 20141106T0900Z +2017-12-11T17:23:54Z INFO - [show-suite-json.20141106T0900Z] -triggered off [] +2017-12-11T17:23:54Z INFO - [show-suite.20141106T0900Z] -triggered off [] +2017-12-11T17:23:57Z INFO - [show-task-json.20141106T0900Z] -triggered off ['show-suite-json.20141106T0900Z'] +2017-12-11T17:23:57Z INFO - [show-task.20141106T0900Z] -triggered off ['show-suite.20141106T0900Z'] +2017-12-11T17:24:00Z INFO - [show-taskinstance-json.20141106T0900Z] -triggered off ['show-task-json.20141106T0900Z'] +2017-12-11T17:24:00Z INFO - [show-taskinstance.20141106T0900Z] -triggered off ['show-task.20141106T0900Z'] +2017-12-11T17:24:03Z INFO - [foo.20141106T0900Z] -triggered off ['show-taskinstance-json.20141106T0900Z', 'show-taskinstance.20141106T0900Z'] +2017-12-11T17:24:05Z INFO - [bar.20141106T0900Z] -triggered off ['foo.20141106T0900Z'] +2017-12-11T17:24:05Z INFO - [baz.20141106T0900Z] -triggered off ['foo.20141106T0900Z'] diff --git a/tests/cylc-show/simple/suite.rc b/tests/cylc-show/simple/suite.rc index 29a1b46cca..cab8217eb3 100644 --- a/tests/cylc-show/simple/suite.rc +++ b/tests/cylc-show/simple/suite.rc @@ -2,6 +2,7 @@ [meta] title = a test suite description = the quick brown fox + custom = custard [cylc] UTC mode = True [scheduling] @@ -10,7 +11,9 @@ [[dependencies]] [[[PT1H]]] graph = """ - show => foo + show-suite => show-task => show-taskinstance + show-suite-json => show-task-json => show-taskinstance-json + show-taskinstance & show-taskinstance-json => foo foo[-PT1H] => foo foo:start => baz & bar """ @@ -20,15 +23,22 @@ [[[meta]]] title = a task description = jumped over the lazy dog - + baz = pub [[bar,baz]] script = true - [[show]] - script = """ -# suite info -cylc show "$CYLC_SUITE_NAME" >{{ TEST_OUTPUT_PATH }} -# task info -cylc show "$CYLC_SUITE_NAME" foo >>{{ TEST_OUTPUT_PATH }} -# task instance info -cylc show "$CYLC_SUITE_NAME" foo.20141106T0900Z >>{{ TEST_OUTPUT_PATH }} -""" + [[show-suite]] + script = cylc show "$CYLC_SUITE_NAME" >>{{ TEST_OUTPUT_PATH }}-suite + [[show-task]] + script = cylc show "$CYLC_SUITE_NAME" foo >>{{ TEST_OUTPUT_PATH }}-task + [[show-taskinstance]] + script = cylc show "$CYLC_SUITE_NAME" foo.20141106T0900Z \ + >>{{ TEST_OUTPUT_PATH }}-taskinstance + [[show-suite-json]] + script = cylc show --json "$CYLC_SUITE_NAME" \ + >>{{ TEST_OUTPUT_PATH }}-json-suite + [[show-task-json]] + script = cylc show --json "$CYLC_SUITE_NAME" foo \ + >>{{ TEST_OUTPUT_PATH }}-json-task + [[show-taskinstance-json]] + script = cylc show --json "$CYLC_SUITE_NAME" foo.20141106T0900Z \ + >>{{ TEST_OUTPUT_PATH }}-json-taskinstance diff --git a/tests/lib/bash/test_header b/tests/lib/bash/test_header index ff31c65b89..f6a8f35c25 100644 --- a/tests/lib/bash/test_header +++ b/tests/lib/bash/test_header @@ -47,6 +47,10 @@ # compare files. However, if an alternate command such as "xxdiff -D" # is desirable (e.g. for debugging), # "export CYLC_TEST_DIFF_CMD=xxdiff -D". +# cmp_json_ok FILE_TEST [FILE_CONTROL] +# Compare FILE_TEST with a file or stdin given by FILE_CONTROL (stdin +# if FILE_CONTROL is "-" or missing), where FILE_TEST must also be in +# valid JSON format. # contains_ok FILE_TEST [FILE_CONTROL] # Make sure that each line in FILE_TEST is present in FILE_CONTROL # (stdin if FILE_CONTROL is "-" or missing). @@ -212,6 +216,22 @@ cmp_ok() { fail "${TEST_NAME}" } +cmp_json_ok() { + run_ok "$1" python -c "import sys +import json + +output = json.load(open(sys.argv[1], 'r')) +if sys.argv[2]: + expected_output = json.load(open(sys.argv[2], 'r')) +else: + expected_output = json.load(sys.stdin) + +if output == expected_output: + sys.exit(0) +else: + sys.exit('%s not equal to %s' % (output, expected_output))" "$2" "${3:-}" +} + contains_ok() { local FILE_TEST="$1" local FILE_CONTROL="${2:--}" From 9ff7b50b38d851ff4981de73d3071ff275cf9cc1 Mon Sep 17 00:00:00 2001 From: Sadie Bartholomew Date: Tue, 12 Dec 2017 14:23:20 +0000 Subject: [PATCH 10/21] Fix tests failing with metadata print-out and rename descripions as meta --- bin/cylc-show | 8 ++++---- lib/cylc/task_pool.py | 4 ++-- tests/cylc-reset/03-output-2.t | 1 + tests/cylc-show/00-simple.t | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/bin/cylc-show b/bin/cylc-show index 253a0db41a..3d3485ad53 100755 --- a/bin/cylc-show +++ b/bin/cylc-show @@ -18,9 +18,9 @@ """cylc [info] show [OPTIONS] ARGS -Interrogate a suite server program for the suite title and description; or for -the title and description of one of its tasks; or for the current state of the -prerequisites, outputs, and clock-triggering of a specific task instance.""" +Interrogate a suite server program for the suite metadata; or for the metadata +of one of its tasks; or for the current state of the prerequisites, outputs, +and clock-triggering of a specific task instance.""" import sys if '--use-ssh' in sys.argv[1:]: @@ -99,7 +99,7 @@ def main(): print prereq else: for key, value in sorted( - result["descriptions"].items(), reverse=True): + result["meta"].items(), reverse=True): print "%s: %s" % (key, value or "(not given)") for name, done in [("prerequisites", "satisfied"), diff --git a/lib/cylc/task_pool.py b/lib/cylc/task_pool.py index 6692cc8f06..9747c60e32 100644 --- a/lib/cylc/task_pool.py +++ b/lib/cylc/task_pool.py @@ -1228,7 +1228,7 @@ def get_task_requisites(self, items, list_prereqs=False): Result in a dict of a dict: { "task_id": { - "descriptions": {key: value, ...}, + "meta": {key: value, ...}, "prerequisites": {key: value, ...}, "outputs": {key: value, ...}, "extras": {key: value, ...}, @@ -1263,7 +1263,7 @@ def get_task_requisites(self, items, list_prereqs=False): for _, msg, is_completed in itask.state.outputs.get_all(): outputs.append(["%s %s" % (itask.identity, msg), is_completed]) results[itask.identity] = { - "descriptions": itask.tdef.describe(), + "meta": itask.tdef.describe(), "prerequisites": itask.state.prerequisites_dump(), "outputs": outputs, "extras": extras} diff --git a/tests/cylc-reset/03-output-2.t b/tests/cylc-reset/03-output-2.t index 6ca74459c6..04eac2fdbd 100755 --- a/tests/cylc-reset/03-output-2.t +++ b/tests/cylc-reset/03-output-2.t @@ -26,6 +26,7 @@ run_ok "${TEST_NAME_BASE}" cylc run --reference-test --debug --no-detach "${SUIT cmp_ok "${SUITE_RUN_DIR}/cylc-show.out" <<'__OUT__' title: (not given) description: (not given) +URL: (not given) prerequisites (- => not satisfied): (None) diff --git a/tests/cylc-show/00-simple.t b/tests/cylc-show/00-simple.t index fa41ade40b..13bdd0775a 100644 --- a/tests/cylc-show/00-simple.t +++ b/tests/cylc-show/00-simple.t @@ -117,7 +117,7 @@ cmp_json_ok "$TEST_NAME-taskinstance" "$TEST_NAME-taskinstance" \ "extras":{ }, - "descriptions":{ + "meta":{ "URL":"", "baz":"pub", "description":"jumped over the lazy dog", From cffc852bc18d1d15154b75f5a3fdad2ebc5fb384 Mon Sep 17 00:00:00 2001 From: Sadie Bartholomew Date: Tue, 12 Dec 2017 17:48:17 +0000 Subject: [PATCH 11/21] Simplify cylc show simple test graph to force consistent output ordering --- tests/cylc-show/00-simple.t | 19 +++++++------------ tests/cylc-show/simple/reference.log | 21 ++++++++++----------- tests/cylc-show/simple/suite.rc | 8 +++----- 3 files changed, 20 insertions(+), 28 deletions(-) diff --git a/tests/cylc-show/00-simple.t b/tests/cylc-show/00-simple.t index 13bdd0775a..bbb741e427 100644 --- a/tests/cylc-show/00-simple.t +++ b/tests/cylc-show/00-simple.t @@ -55,12 +55,11 @@ baz: pub URL: (not given) prerequisites (- => not satisfied): - - show-taskinstance-json.20141106T0900Z succeeded - - show-taskinstance.20141106T0900Z succeeded + + bar.20141106T0900Z succeeded outputs (- => not completed): - - foo.20141106T0900Z submitted - - foo.20141106T0900Z started + + foo.20141106T0900Z submitted + + foo.20141106T0900Z started - foo.20141106T0900Z succeeded __SHOW_OUTPUT__ #------------------------------------------------------------------------------- @@ -92,22 +91,18 @@ cmp_json_ok "$TEST_NAME-taskinstance" "$TEST_NAME-taskinstance" \ "foo.20141106T0900Z":{ "prerequisites":[ [ - "show-taskinstance-json.20141106T0900Z succeeded", - false - ], - [ - "show-taskinstance.20141106T0900Z succeeded", - false + "bar.20141106T0900Z succeeded", + "satisfied naturally" ] ], "outputs":[ [ "foo.20141106T0900Z submitted", - false + true ], [ "foo.20141106T0900Z started", - false + true ], [ "foo.20141106T0900Z succeeded", diff --git a/tests/cylc-show/simple/reference.log b/tests/cylc-show/simple/reference.log index 3a2111c1cd..faed584a19 100644 --- a/tests/cylc-show/simple/reference.log +++ b/tests/cylc-show/simple/reference.log @@ -1,11 +1,10 @@ -2017-12-11T17:23:54Z INFO - Initial point: 20141106T0900Z -2017-12-11T17:23:54Z INFO - Final point: 20141106T0900Z -2017-12-11T17:23:54Z INFO - [show-suite-json.20141106T0900Z] -triggered off [] -2017-12-11T17:23:54Z INFO - [show-suite.20141106T0900Z] -triggered off [] -2017-12-11T17:23:57Z INFO - [show-task-json.20141106T0900Z] -triggered off ['show-suite-json.20141106T0900Z'] -2017-12-11T17:23:57Z INFO - [show-task.20141106T0900Z] -triggered off ['show-suite.20141106T0900Z'] -2017-12-11T17:24:00Z INFO - [show-taskinstance-json.20141106T0900Z] -triggered off ['show-task-json.20141106T0900Z'] -2017-12-11T17:24:00Z INFO - [show-taskinstance.20141106T0900Z] -triggered off ['show-task.20141106T0900Z'] -2017-12-11T17:24:03Z INFO - [foo.20141106T0900Z] -triggered off ['show-taskinstance-json.20141106T0900Z', 'show-taskinstance.20141106T0900Z'] -2017-12-11T17:24:05Z INFO - [bar.20141106T0900Z] -triggered off ['foo.20141106T0900Z'] -2017-12-11T17:24:05Z INFO - [baz.20141106T0900Z] -triggered off ['foo.20141106T0900Z'] +2017-12-12T17:43:33Z INFO - Initial point: 20141106T0900Z +2017-12-12T17:43:33Z INFO - Final point: 20141106T0900Z +2017-12-12T17:43:33Z INFO - [bar.20141106T0900Z] -triggered off [] +2017-12-12T17:43:36Z INFO - [foo.20141106T0900Z] -triggered off ['bar.20141106T0900Z'] +2017-12-12T17:43:38Z INFO - [show-task-json.20141106T0900Z] -triggered off ['foo.20141106T0900Z'] +2017-12-12T17:43:38Z INFO - [show-suite-json.20141106T0900Z] -triggered off ['foo.20141106T0900Z'] +2017-12-12T17:43:38Z INFO - [show-taskinstance-json.20141106T0900Z] -triggered off ['foo.20141106T0900Z'] +2017-12-12T17:43:38Z INFO - [show-taskinstance.20141106T0900Z] -triggered off ['foo.20141106T0900Z'] +2017-12-12T17:43:38Z INFO - [show-task.20141106T0900Z] -triggered off ['foo.20141106T0900Z'] +2017-12-12T17:43:38Z INFO - [show-suite.20141106T0900Z] -triggered off ['foo.20141106T0900Z'] diff --git a/tests/cylc-show/simple/suite.rc b/tests/cylc-show/simple/suite.rc index cab8217eb3..14e3e083d1 100644 --- a/tests/cylc-show/simple/suite.rc +++ b/tests/cylc-show/simple/suite.rc @@ -11,11 +11,9 @@ [[dependencies]] [[[PT1H]]] graph = """ - show-suite => show-task => show-taskinstance - show-suite-json => show-task-json => show-taskinstance-json - show-taskinstance & show-taskinstance-json => foo - foo[-PT1H] => foo - foo:start => baz & bar + bar => foo + foo:start => show-suite & show-task & show-taskinstance & \ + show-suite-json & show-task-json & show-taskinstance-json """ [runtime] [[foo]] From d56f3877c7b3a9c9c3391dd9d6036472704b8dbf Mon Sep 17 00:00:00 2001 From: Sadie Bartholomew Date: Thu, 14 Dec 2017 19:33:46 +0000 Subject: [PATCH 12/21] Add cylc scan --describe meta output and test --- bin/cylc-scan | 61 ++++++++++--------- tests/authentication/01-description.t | 4 ++ tests/authentication/02-state-totals.t | 4 ++ tests/authentication/03-full-read.t | 4 ++ tests/authentication/04-shutdown.t | 4 ++ tests/authentication/05-full-control.t | 4 ++ tests/authentication/06-suite-override.t | 4 ++ tests/authentication/07-sha-hash.t | 4 ++ tests/cylc-scan/00-simple.t | 41 ++++++++++++- .../reference.log | 0 tests/cylc-scan/ctb-cylc-scan-simple/suite.rc | 18 ++++++ tests/cylc-scan/simple/suite.rc | 10 --- 12 files changed, 116 insertions(+), 42 deletions(-) rename tests/cylc-scan/{simple => ctb-cylc-scan-simple}/reference.log (100%) create mode 100644 tests/cylc-scan/ctb-cylc-scan-simple/suite.rc delete mode 100644 tests/cylc-scan/simple/suite.rc diff --git a/bin/cylc-scan b/bin/cylc-scan index 47782e518f..a82e52d54c 100755 --- a/bin/cylc-scan +++ b/bin/cylc-scan @@ -104,7 +104,7 @@ def main(): parser.add_option( "-d", "--describe", - help="Print suite titles and descriptions if available.", + help="Print suite metadata if available.", action="store_true", default=False, dest="describe") parser.add_option( @@ -267,40 +267,43 @@ def main(): if options.describe: try: - title = suite_identity[KEY_META].get(KEY_TITLE) + meta_items = suite_identity[KEY_META] except KeyError: # Compat:<=7.5.0 - title = suite_identity.get(KEY_TITLE) - if title is None: - print indent + bold("(description and state totals withheld)") + meta_items = { + "title": suite_identity.get(KEY_TITLE), + "description": suite_identity.get(KEY_DESCRIPTION) + } + + # Deal with the title first for case of just withheld statement + if not meta_items["title"]: + print indent + bold("(description and state totals " + "withheld)") continue - print indent + bold("Title:") - if title == "": - line = "(no title)" else: - line = '"%s"' % title - print indent * 2 + line - - try: - description = suite_identity[KEY_META].get(KEY_DESCRIPTION) - except KeyError: - # Compat:<=7.5.0 - description = suite_identity.get(KEY_DESCRIPTION) - print indent + bold("Description:") - if description == "": - lines = "(no description)" - else: - lines = '"%s"' % description - line1 = True - for line in lines.split('\n'): - line = line.lstrip() - if not line1: - # Indent under the double quote. - line = " " + line - line1 = False - print indent * 2 + line + print (indent + bold("Title:\n") + indent * 2 + + '"%s"' % meta_items["title"]) + for metaitem in meta_items.keys(): + if metaitem != "title": + if metaitem == "description" or metaitem == "group": + print indent + bold(metaitem.capitalize() + ":") + else: + print indent + bold(metaitem + ":") + if not meta_items[metaitem]: + lines = "(no %s)" % metaitem + else: + lines = '"%s"' % meta_items[metaitem] + line1 = True + for line in lines.split('\n'): + line = line.lstrip() + if not line1: + # Indent under the double quote. + line = " " + line + line1 = False + print indent * 2 + line totals = suite_identity.get(KEY_STATES) + if options.print_totals: if totals is None: print indent + bold("(state totals withheld)") diff --git a/tests/authentication/01-description.t b/tests/authentication/01-description.t index e7baa36b46..5eb5945edb 100644 --- a/tests/authentication/01-description.t +++ b/tests/authentication/01-description.t @@ -48,6 +48,10 @@ cmp_ok 'scan.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." + URL: + (no URL) + Group: + (no group) Description: "Stalls when the first task fails." (state totals withheld) diff --git a/tests/authentication/02-state-totals.t b/tests/authentication/02-state-totals.t index e0d9cbeee2..b612667ac7 100644 --- a/tests/authentication/02-state-totals.t +++ b/tests/authentication/02-state-totals.t @@ -48,6 +48,10 @@ cmp_ok 'scan.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." + URL: + (no URL) + Group: + (no group) Description: "Stalls when the first task fails." Task state totals: diff --git a/tests/authentication/03-full-read.t b/tests/authentication/03-full-read.t index 9077958bb2..f4df8a5eea 100644 --- a/tests/authentication/03-full-read.t +++ b/tests/authentication/03-full-read.t @@ -48,6 +48,10 @@ cmp_ok 'scan.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." + URL: + (no URL) + Group: + (no group) Description: "Stalls when the first task fails." Task state totals: diff --git a/tests/authentication/04-shutdown.t b/tests/authentication/04-shutdown.t index c5a515623a..c148d9eef4 100644 --- a/tests/authentication/04-shutdown.t +++ b/tests/authentication/04-shutdown.t @@ -47,6 +47,10 @@ cmp_ok 'scan.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." + URL: + (no URL) + Group: + (no group) Description: "Stalls when the first task fails." Task state totals: diff --git a/tests/authentication/05-full-control.t b/tests/authentication/05-full-control.t index 6513babe57..cf76b3d9ac 100644 --- a/tests/authentication/05-full-control.t +++ b/tests/authentication/05-full-control.t @@ -46,6 +46,10 @@ cmp_ok 'scan.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." + URL: + (no URL) + Group: + (no group) Description: "Stalls when the first task fails." Task state totals: diff --git a/tests/authentication/06-suite-override.t b/tests/authentication/06-suite-override.t index fbe796a8d5..43099c45d0 100644 --- a/tests/authentication/06-suite-override.t +++ b/tests/authentication/06-suite-override.t @@ -48,6 +48,10 @@ cmp_ok 'scan.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." + URL: + (no URL) + Group: + (no group) Description: "Stalls when the first task fails. Suite overrides global authentication settings." diff --git a/tests/authentication/07-sha-hash.t b/tests/authentication/07-sha-hash.t index cb71d15b7f..d8e65e1dcb 100644 --- a/tests/authentication/07-sha-hash.t +++ b/tests/authentication/07-sha-hash.t @@ -49,6 +49,10 @@ cmp_ok 'scan.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." + URL: + (no URL) + Group: + (no group) Description: "Stalls when the first task fails." Task state totals: diff --git a/tests/cylc-scan/00-simple.t b/tests/cylc-scan/00-simple.t index afb988627a..6c6515bbb1 100644 --- a/tests/cylc-scan/00-simple.t +++ b/tests/cylc-scan/00-simple.t @@ -18,15 +18,50 @@ # Test cylc scan is picking up running suite . $(dirname $0)/test_header #------------------------------------------------------------------------------- -set_test_number 2 +set_test_number 6 create_test_globalrc +host_port_ref () { + SUITE=$1 + FQDN=$(python -c "import sys +sys.path.insert(0, '$CYLC_HOME') +import cylc.hostuserutil +print cylc.hostuserutil.get_fqdn_by_host('`hostname`')") + PORT=$(sed -n 's/CYLC_SUITE_PORT=//p' \ + "${HOME}/cylc-run/${SUITE}/.service/contact") + echo "$SUITE `whoami`@$FQDN:$PORT" +} #------------------------------------------------------------------------------- -install_suite $TEST_NAME_BASE simple +install_suite $TEST_NAME_BASE ctb-cylc-scan-simple #------------------------------------------------------------------------------- TEST_NAME=$TEST_NAME_BASE-validate run_ok $TEST_NAME cylc validate $SUITE_NAME #------------------------------------------------------------------------------- TEST_NAME=$TEST_NAME_BASE-run -suite_run_ok $TEST_NAME cylc run --reference-test --debug --no-detach $SUITE_NAME +run_ok $TEST_NAME cylc run $SUITE_NAME --hold +SCAN_LINE=$(host_port_ref "${SUITE_NAME}") #------------------------------------------------------------------------------- +TEST_NAME=$TEST_NAME_BASE-basic +run_ok $TEST_NAME cylc scan --no-bold -n "$SUITE_NAME" +cmp_ok "$TEST_NAME.stdout" <<<${SCAN_LINE} +#------------------------------------------------------------------------------- +TEST_NAME=$TEST_NAME_BASE-describe +run_ok $TEST_NAME cylc scan --no-bold --describe -n "$SUITE_NAME" +cmp_ok "$TEST_NAME.stdout" <<__SHOW_OUTPUT__ +${SCAN_LINE} + Title: + "A simple test" + Group: + (no group) + Description: + "A simple test to simply test whether cylc scan is + doing the right thing - let's see what happens." + URL: + (no URL) + datum: + "metadatum" + another_datum: + "another_metadatum" +__SHOW_OUTPUT__ +#------------------------------------------------------------------------------- +cylc stop $SUITE_NAME --now purge_suite $SUITE_NAME diff --git a/tests/cylc-scan/simple/reference.log b/tests/cylc-scan/ctb-cylc-scan-simple/reference.log similarity index 100% rename from tests/cylc-scan/simple/reference.log rename to tests/cylc-scan/ctb-cylc-scan-simple/reference.log diff --git a/tests/cylc-scan/ctb-cylc-scan-simple/suite.rc b/tests/cylc-scan/ctb-cylc-scan-simple/suite.rc new file mode 100644 index 0000000000..6a345070a9 --- /dev/null +++ b/tests/cylc-scan/ctb-cylc-scan-simple/suite.rc @@ -0,0 +1,18 @@ +[meta] + title = A simple test + description = """A simple test to simply test whether cylc scan is + doing the right thing - let's see what happens.""" + datum = metadatum + another_datum = another_metadatum + +[cylc] + [[reference test]] + required run mode = live + live mode suite timeout = PT1M +[scheduling] + [[dependencies]] + graph = "foo" +[runtime] + [[foo]] + [[bar]] + [[baz]] diff --git a/tests/cylc-scan/simple/suite.rc b/tests/cylc-scan/simple/suite.rc deleted file mode 100644 index 4c294d7776..0000000000 --- a/tests/cylc-scan/simple/suite.rc +++ /dev/null @@ -1,10 +0,0 @@ -[cylc] - [[reference test]] - required run mode = live - live mode suite timeout = PT1M -[scheduling] - [[dependencies]] - graph = "foo" -[runtime] - [[foo]] - script = cylc scan --comms-timeout=5 | grep "^${CYLC_SUITE_NAME} " From f79bbbdc09b66860a41e4e66cd424c3f6321fb06 Mon Sep 17 00:00:00 2001 From: Sadie Bartholomew Date: Fri, 15 Dec 2017 12:05:26 +0000 Subject: [PATCH 13/21] Move scan --describe tests into authentication testing --- bin/cylc-scan | 42 ++++++++++--------- tests/authentication/00-identity.t | 12 +++++- tests/authentication/01-description.t | 32 ++++++++++++-- tests/authentication/02-state-totals.t | 34 ++++++++++++--- tests/authentication/03-full-read.t | 34 ++++++++++++--- tests/authentication/04-shutdown.t | 32 ++++++++++++-- tests/authentication/05-full-control.t | 34 ++++++++++++--- tests/authentication/06-suite-override.t | 17 +++++++- tests/authentication/07-sha-hash.t | 34 ++++++++++++--- tests/authentication/basic/suite.rc | 5 ++- tests/cylc-scan/00-simple.t | 41 ++---------------- .../reference.log | 0 .../{ctb-cylc-scan-simple => simple}/suite.rc | 0 13 files changed, 227 insertions(+), 90 deletions(-) rename tests/cylc-scan/{ctb-cylc-scan-simple => simple}/reference.log (100%) rename tests/cylc-scan/{ctb-cylc-scan-simple => simple}/suite.rc (100%) diff --git a/bin/cylc-scan b/bin/cylc-scan index a82e52d54c..207037b8b8 100755 --- a/bin/cylc-scan +++ b/bin/cylc-scan @@ -275,32 +275,34 @@ def main(): "description": suite_identity.get(KEY_DESCRIPTION) } - # Deal with the title first for case of just withheld statement - if not meta_items["title"]: + # Deal with title first for case of just withheld statement + if meta_items["title"] is None: print indent + bold("(description and state totals " "withheld)") continue + elif not meta_items["title"]: + print indent + bold("Title:\n") + indent * 2 + "(no title)" else: print (indent + bold("Title:\n") + indent * 2 + '"%s"' % meta_items["title"]) - for metaitem in meta_items.keys(): - if metaitem != "title": - if metaitem == "description" or metaitem == "group": - print indent + bold(metaitem.capitalize() + ":") - else: - print indent + bold(metaitem + ":") - if not meta_items[metaitem]: - lines = "(no %s)" % metaitem - else: - lines = '"%s"' % meta_items[metaitem] - line1 = True - for line in lines.split('\n'): - line = line.lstrip() - if not line1: - # Indent under the double quote. - line = " " + line - line1 = False - print indent * 2 + line + for metaitem in meta_items.keys(): + if metaitem != "title": + if metaitem == "description" or metaitem == "group": + print indent + bold(metaitem.capitalize() + ":") + else: + print indent + bold(metaitem + ":") + if not meta_items[metaitem]: + lines = "(no %s)" % metaitem + else: + lines = '"%s"' % meta_items[metaitem] + line1 = True + for line in lines.split('\n'): + line = line.lstrip() + if not line1: + # Indent under the double quote. + line = " " + line + line1 = False + print indent * 2 + line totals = suite_identity.get(KEY_STATES) diff --git a/tests/authentication/00-identity.t b/tests/authentication/00-identity.t index e1e51dd8b9..fe5853c5a2 100644 --- a/tests/authentication/00-identity.t +++ b/tests/authentication/00-identity.t @@ -18,7 +18,7 @@ # Test authentication - privilege 'identity'. . $(dirname $0)/test_header -set_test_number 14 +set_test_number 15 install_suite "${TEST_NAME_BASE}" basic @@ -68,7 +68,7 @@ cylc suite-state "${SUITE_NAME}" --task=foo --status=failed --point=1 \ # Disable the suite passphrase (to leave us with public access privilege). mv "${SRV_D}/passphrase" "${SRV_D}/passphrase.DIS" -# Check scan output. +# Check scan --full output. cylc scan --comms-timeout=5 -fb -n "${SUITE_NAME}" 'localhost' \ >'scan.out' 2>'/dev/null' cmp_ok scan.out << __END__ @@ -76,6 +76,14 @@ ${SUITE_NAME} ${USER}@localhost:${PORT} (description and state totals withheld) __END__ +# Check scan --describe output. +cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}" 'localhost' \ + >'scan.out' 2>'/dev/null' +cmp_ok scan.out << __END__ +${SUITE_NAME} ${USER}@localhost:${PORT} + (description and state totals withheld) +__END__ + # "cylc show" should be denied. TEST_NAME="${TEST_NAME_BASE}-show" run_fail "${TEST_NAME}" cylc show "${SUITE_NAME}" diff --git a/tests/authentication/01-description.t b/tests/authentication/01-description.t index 5eb5945edb..3b3028437f 100644 --- a/tests/authentication/01-description.t +++ b/tests/authentication/01-description.t @@ -18,7 +18,7 @@ # Test authentication - privilege 'description'. . $(dirname $0)/test_header -set_test_number 7 +set_test_number 8 install_suite "${TEST_NAME_BASE}" basic @@ -40,7 +40,7 @@ cylc suite-state "${SUITE_NAME}" --task=foo --status=failed --point=1 \ SRV_D="$(cylc get-global-config --print-run-dir)/${SUITE_NAME}/.service" mv "${SRV_D}/passphrase" "${SRV_D}/passphrase.DIS" -# Check scan output. +# Check scan --full output. HOST="$(sed -n 's/^CYLC_SUITE_HOST=//p' "${SRV_D}/contact")" PORT="$(sed -n 's/^CYLC_SUITE_PORT=//p' "${SRV_D}/contact")" cylc scan --comms-timeout=5 -fb -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' @@ -48,13 +48,37 @@ cmp_ok 'scan.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." + Group: + (no group) + Description: + "Stalls when the first task fails. + Here we test out a multi-line description!" URL: (no URL) + another_metadata: + "1" + custom_metadata: + "something_custom" + (state totals withheld) +__END__ + +# Check scan --describe output. +cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' +cmp_ok 'scan.out' <<__END__ +${SUITE_NAME} ${USER}@${HOST}:${PORT} + Title: + "Authentication test suite." Group: (no group) Description: - "Stalls when the first task fails." - (state totals withheld) + "Stalls when the first task fails. + Here we test out a multi-line description!" + URL: + (no URL) + another_metadata: + "1" + custom_metadata: + "something_custom" __END__ # "cylc show" (suite info) OK. diff --git a/tests/authentication/02-state-totals.t b/tests/authentication/02-state-totals.t index b612667ac7..315827a67e 100644 --- a/tests/authentication/02-state-totals.t +++ b/tests/authentication/02-state-totals.t @@ -18,7 +18,7 @@ # Test authentication - privilege 'state-totals'. . $(dirname $0)/test_header -set_test_number 7 +set_test_number 8 install_suite "${TEST_NAME_BASE}" basic @@ -40,7 +40,7 @@ cylc suite-state "${SUITE_NAME}" --task=foo --status=failed --point=1 \ SRV_D="$(cylc get-global-config --print-run-dir)/${SUITE_NAME}/.service" mv "${SRV_D}/passphrase" "${SRV_D}/passphrase.DIS" -# Check scan output. +# Check scan --full output. HOST="$(sed -n 's/^CYLC_SUITE_HOST=//p' "${SRV_D}/contact")" PORT="$(sed -n 's/^CYLC_SUITE_PORT=//p' "${SRV_D}/contact")" cylc scan --comms-timeout=5 -fb -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' @@ -48,17 +48,41 @@ cmp_ok 'scan.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." - URL: - (no URL) Group: (no group) Description: - "Stalls when the first task fails." + "Stalls when the first task fails. + Here we test out a multi-line description!" + URL: + (no URL) + another_metadata: + "1" + custom_metadata: + "something_custom" Task state totals: failed:1 waiting:1 1 failed:1 waiting:1 __END__ +# Check scan --describe output. +cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' +cmp_ok 'scan.out' <<__END__ +${SUITE_NAME} ${USER}@${HOST}:${PORT} + Title: + "Authentication test suite." + Group: + (no group) + Description: + "Stalls when the first task fails. + Here we test out a multi-line description!" + URL: + (no URL) + another_metadata: + "1" + custom_metadata: + "something_custom" +__END__ + # "cylc show" (suite info) OK. TEST_NAME="${TEST_NAME_BASE}-show1" run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" diff --git a/tests/authentication/03-full-read.t b/tests/authentication/03-full-read.t index f4df8a5eea..367e3de96c 100644 --- a/tests/authentication/03-full-read.t +++ b/tests/authentication/03-full-read.t @@ -18,7 +18,7 @@ # Test authentication - privilege 'full-read'. . $(dirname $0)/test_header -set_test_number 8 +set_test_number 9 install_suite "${TEST_NAME_BASE}" basic @@ -40,7 +40,7 @@ cylc suite-state "${SUITE_NAME}" --task=foo --status=failed --point=1 \ SRV_D="$(cylc get-global-config --print-run-dir)/${SUITE_NAME}/.service" mv "${SRV_D}/passphrase" "${SRV_D}/passphrase.DIS" -# Check scan output. +# Check scan --full output. HOST="$(sed -n 's/^CYLC_SUITE_HOST=//p' "${SRV_D}/contact")" PORT="$(sed -n 's/^CYLC_SUITE_PORT=//p' "${SRV_D}/contact")" cylc scan --comms-timeout=5 -fb -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' @@ -48,17 +48,41 @@ cmp_ok 'scan.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." - URL: - (no URL) Group: (no group) Description: - "Stalls when the first task fails." + "Stalls when the first task fails. + Here we test out a multi-line description!" + URL: + (no URL) + another_metadata: + "1" + custom_metadata: + "something_custom" Task state totals: failed:1 waiting:1 1 failed:1 waiting:1 __END__ +# Check scan --describe output. +cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' +cmp_ok 'scan.out' <<__END__ +${SUITE_NAME} ${USER}@${HOST}:${PORT} + Title: + "Authentication test suite." + Group: + (no group) + Description: + "Stalls when the first task fails. + Here we test out a multi-line description!" + URL: + (no URL) + another_metadata: + "1" + custom_metadata: + "something_custom" +__END__ + # "cylc show" (suite info) OK. TEST_NAME="${TEST_NAME_BASE}-show1" run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" diff --git a/tests/authentication/04-shutdown.t b/tests/authentication/04-shutdown.t index c148d9eef4..c9d79bbbc5 100644 --- a/tests/authentication/04-shutdown.t +++ b/tests/authentication/04-shutdown.t @@ -18,7 +18,7 @@ # Test authentication - privilege 'shutdown'. . $(dirname $0)/test_header -set_test_number 9 +set_test_number 10 install_suite "${TEST_NAME_BASE}" basic @@ -42,22 +42,46 @@ mv "${SRV_D}/passphrase" "${SRV_D}/passphrase.DIS" HOST="$(sed -n 's/^CYLC_SUITE_HOST=//p' "${SRV_D}/contact")" PORT="$(sed -n 's/^CYLC_SUITE_PORT=//p' "${SRV_D}/contact")" + cylc scan --comms-timeout=5 -fb -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' cmp_ok 'scan.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." - URL: - (no URL) Group: (no group) Description: - "Stalls when the first task fails." + "Stalls when the first task fails. + Here we test out a multi-line description!" + URL: + (no URL) + another_metadata: + "1" + custom_metadata: + "something_custom" Task state totals: failed:1 waiting:1 1 failed:1 waiting:1 __END__ +cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' +cmp_ok 'scan.out' <<__END__ +${SUITE_NAME} ${USER}@${HOST}:${PORT} + Title: + "Authentication test suite." + Group: + (no group) + Description: + "Stalls when the first task fails. + Here we test out a multi-line description!" + URL: + (no URL) + another_metadata: + "1" + custom_metadata: + "something_custom" +__END__ + # "cylc show" (suite info) OK. TEST_NAME="${TEST_NAME_BASE}-show1" run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" diff --git a/tests/authentication/05-full-control.t b/tests/authentication/05-full-control.t index cf76b3d9ac..c234e43fa1 100644 --- a/tests/authentication/05-full-control.t +++ b/tests/authentication/05-full-control.t @@ -18,7 +18,7 @@ # Test authentication - privilege 'full-control' (with passphrase). . $(dirname $0)/test_header -set_test_number 9 +set_test_number 10 install_suite "${TEST_NAME_BASE}" basic @@ -37,7 +37,7 @@ unset CYLC_CONF_PATH cylc suite-state "${SUITE_NAME}" --task=foo --status=failed --point=1 \ --interval=1 --max-polls=10 || exit 1 -# Check scan output. +# Check scan --full output. SRV_D="$(cylc get-global-config --print-run-dir)/${SUITE_NAME}/.service" HOST="$(sed -n 's/^CYLC_SUITE_HOST=//p' "${SRV_D}/contact")" PORT="$(sed -n 's/^CYLC_SUITE_PORT=//p' "${SRV_D}/contact")" @@ -46,17 +46,41 @@ cmp_ok 'scan.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." - URL: - (no URL) Group: (no group) Description: - "Stalls when the first task fails." + "Stalls when the first task fails. + Here we test out a multi-line description!" + URL: + (no URL) + another_metadata: + "1" + custom_metadata: + "something_custom" Task state totals: failed:1 waiting:1 1 failed:1 waiting:1 __END__ +# Check scan --describe output. +cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' +cmp_ok 'scan.out' <<__END__ +${SUITE_NAME} ${USER}@${HOST}:${PORT} + Title: + "Authentication test suite." + Group: + (no group) + Description: + "Stalls when the first task fails. + Here we test out a multi-line description!" + URL: + (no URL) + another_metadata: + "1" + custom_metadata: + "something_custom" +__END__ + # "cylc show" (suite info) OK. TEST_NAME="${TEST_NAME_BASE}-show1" run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" diff --git a/tests/authentication/06-suite-override.t b/tests/authentication/06-suite-override.t index 43099c45d0..7c44946640 100644 --- a/tests/authentication/06-suite-override.t +++ b/tests/authentication/06-suite-override.t @@ -19,7 +19,7 @@ # (Suite overrides global privilege 'identity'.) . $(dirname $0)/test_header -set_test_number 9 +set_test_number 10 install_suite "${TEST_NAME_BASE}" override @@ -43,6 +43,7 @@ mv "${SRV_D}/passphrase" "${SRV_D}/passphrase.DIS" HOST="$(sed -n 's/^CYLC_SUITE_HOST=//p' "${SRV_D}/contact")" PORT="$(sed -n 's/^CYLC_SUITE_PORT=//p' "${SRV_D}/contact")" + cylc scan --comms-timeout=5 -fb -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' cmp_ok 'scan.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} @@ -60,6 +61,20 @@ ${SUITE_NAME} ${USER}@${HOST}:${PORT} 1 failed:1 waiting:1 __END__ +cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' +cmp_ok 'scan.out' <<__END__ +${SUITE_NAME} ${USER}@${HOST}:${PORT} + Title: + "Authentication test suite." + URL: + (no URL) + Group: + (no group) + Description: + "Stalls when the first task fails. + Suite overrides global authentication settings." +__END__ + # "cylc show" (suite info) OK. TEST_NAME="${TEST_NAME_BASE}-show1" run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" diff --git a/tests/authentication/07-sha-hash.t b/tests/authentication/07-sha-hash.t index d8e65e1dcb..d0f4f7e0b5 100644 --- a/tests/authentication/07-sha-hash.t +++ b/tests/authentication/07-sha-hash.t @@ -18,7 +18,7 @@ # Test authentication - using SHA1 hash option. . $(dirname $0)/test_header -set_test_number 9 +set_test_number 10 install_suite "${TEST_NAME_BASE}" basic @@ -40,7 +40,7 @@ unset CYLC_CONF_PATH cylc suite-state "${SUITE_NAME}" --task=foo --status=failed --point=1 \ --interval=1 --max-polls=10 || exit 1 -# Check scan output. +# Check scan --full output. SRV_D="$(cylc get-global-config --print-run-dir)/${SUITE_NAME}/.service" HOST="$(sed -n 's/^CYLC_SUITE_HOST=//p' "${SRV_D}/contact")" PORT="$(sed -n 's/^CYLC_SUITE_PORT=//p' "${SRV_D}/contact")" @@ -49,17 +49,41 @@ cmp_ok 'scan.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." - URL: - (no URL) Group: (no group) Description: - "Stalls when the first task fails." + "Stalls when the first task fails. + Here we test out a multi-line description!" + URL: + (no URL) + another_metadata: + "1" + custom_metadata: + "something_custom" Task state totals: failed:1 waiting:1 1 failed:1 waiting:1 __END__ +# Check scan --describe output. +cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}">'scan.out' 2>'/dev/null' +cmp_ok 'scan.out' <<__END__ +${SUITE_NAME} ${USER}@${HOST}:${PORT} + Title: + "Authentication test suite." + Group: + (no group) + Description: + "Stalls when the first task fails. + Here we test out a multi-line description!" + URL: + (no URL) + another_metadata: + "1" + custom_metadata: + "something_custom" +__END__ + # "cylc show" (suite info) OK. TEST_NAME="${TEST_NAME_BASE}-show1" run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" diff --git a/tests/authentication/basic/suite.rc b/tests/authentication/basic/suite.rc index ae487afafc..07b4603012 100644 --- a/tests/authentication/basic/suite.rc +++ b/tests/authentication/basic/suite.rc @@ -1,6 +1,9 @@ [meta] title = Authentication test suite. - description = Stalls when the first task fails. + description = """Stalls when the first task fails. + Here we test out a multi-line description!""" + custom_metadata = something_custom + another_metadata = 1 [cylc] [[events]] timeout = PT30S diff --git a/tests/cylc-scan/00-simple.t b/tests/cylc-scan/00-simple.t index 6c6515bbb1..afb988627a 100644 --- a/tests/cylc-scan/00-simple.t +++ b/tests/cylc-scan/00-simple.t @@ -18,50 +18,15 @@ # Test cylc scan is picking up running suite . $(dirname $0)/test_header #------------------------------------------------------------------------------- -set_test_number 6 +set_test_number 2 create_test_globalrc -host_port_ref () { - SUITE=$1 - FQDN=$(python -c "import sys -sys.path.insert(0, '$CYLC_HOME') -import cylc.hostuserutil -print cylc.hostuserutil.get_fqdn_by_host('`hostname`')") - PORT=$(sed -n 's/CYLC_SUITE_PORT=//p' \ - "${HOME}/cylc-run/${SUITE}/.service/contact") - echo "$SUITE `whoami`@$FQDN:$PORT" -} #------------------------------------------------------------------------------- -install_suite $TEST_NAME_BASE ctb-cylc-scan-simple +install_suite $TEST_NAME_BASE simple #------------------------------------------------------------------------------- TEST_NAME=$TEST_NAME_BASE-validate run_ok $TEST_NAME cylc validate $SUITE_NAME #------------------------------------------------------------------------------- TEST_NAME=$TEST_NAME_BASE-run -run_ok $TEST_NAME cylc run $SUITE_NAME --hold -SCAN_LINE=$(host_port_ref "${SUITE_NAME}") +suite_run_ok $TEST_NAME cylc run --reference-test --debug --no-detach $SUITE_NAME #------------------------------------------------------------------------------- -TEST_NAME=$TEST_NAME_BASE-basic -run_ok $TEST_NAME cylc scan --no-bold -n "$SUITE_NAME" -cmp_ok "$TEST_NAME.stdout" <<<${SCAN_LINE} -#------------------------------------------------------------------------------- -TEST_NAME=$TEST_NAME_BASE-describe -run_ok $TEST_NAME cylc scan --no-bold --describe -n "$SUITE_NAME" -cmp_ok "$TEST_NAME.stdout" <<__SHOW_OUTPUT__ -${SCAN_LINE} - Title: - "A simple test" - Group: - (no group) - Description: - "A simple test to simply test whether cylc scan is - doing the right thing - let's see what happens." - URL: - (no URL) - datum: - "metadatum" - another_datum: - "another_metadatum" -__SHOW_OUTPUT__ -#------------------------------------------------------------------------------- -cylc stop $SUITE_NAME --now purge_suite $SUITE_NAME diff --git a/tests/cylc-scan/ctb-cylc-scan-simple/reference.log b/tests/cylc-scan/simple/reference.log similarity index 100% rename from tests/cylc-scan/ctb-cylc-scan-simple/reference.log rename to tests/cylc-scan/simple/reference.log diff --git a/tests/cylc-scan/ctb-cylc-scan-simple/suite.rc b/tests/cylc-scan/simple/suite.rc similarity index 100% rename from tests/cylc-scan/ctb-cylc-scan-simple/suite.rc rename to tests/cylc-scan/simple/suite.rc From 18eef440df5a8ad627ce848266056f82e43ca54c Mon Sep 17 00:00:00 2001 From: Sadie Bartholomew Date: Mon, 18 Dec 2017 16:22:01 +0000 Subject: [PATCH 14/21] Explicit testing of new --json plus -d and -r scan options --- bin/cylc-scan | 61 ++++++++++++++++++++---- tests/authentication/00-identity.t | 29 +++++++++-- tests/authentication/01-description.t | 32 +++++++++++-- tests/authentication/02-state-totals.t | 36 ++++++++++++-- tests/authentication/03-full-read.t | 36 ++++++++++++-- tests/authentication/04-shutdown.t | 36 ++++++++++++-- tests/authentication/05-full-control.t | 36 ++++++++++++-- tests/authentication/06-suite-override.t | 36 ++++++++++++-- tests/authentication/07-sha-hash.t | 36 ++++++++++++-- 9 files changed, 290 insertions(+), 48 deletions(-) diff --git a/bin/cylc-scan b/bin/cylc-scan index 207037b8b8..9f0f0dfeaf 100755 --- a/bin/cylc-scan +++ b/bin/cylc-scan @@ -47,6 +47,7 @@ if "--use-ssh" in sys.argv[1:]: sys.exit(0) import re +import json from cylc.cfgspec.globalcfg import GLOBAL_CFG from cylc.hostuserutil import get_user @@ -147,8 +148,13 @@ def main(): parser.add_option( "-r", "--raw", "--raw-format", - help='Parsable format ("suite|owner|host|property|value")', - action="store_true", default=False, dest="raw_format" + help='Parsable format ("suite|owner|host|property|value").', + action="store_true", default=False, dest="raw_format") + + parser.add_option( + "-j", "--json", "--json-format", + help="JSON format.", + action="store_true", default=False, dest="json_format" ) options, args = parser.parse_args() @@ -170,10 +176,16 @@ def main(): if options.color: options.print_totals = True - if options.raw_format and (options.old_format or options.describe or - options.print_totals or options.color): - parser.error( - "--raw-format cannot be used with other format options.") + # Check for incompatible formatting options + incompatibles = [options.raw_format, options.json_format] + format_ops = [options.old_format, options.describe, options.print_totals, + options.color] + if incompatibles.count(True) == 2: + parser.error("--raw and --json are incompatible formatting options.") + if incompatibles.count(True) == 1 and format_ops.count(True) >= 1: + parser.error("Incompatible format options: --%s forbidden with any " + "of --old-format, --describe, --state-totals or --color." + % ("raw" if incompatibles[0] else "json")) state_legend = "" if options.color: @@ -237,10 +249,14 @@ def main(): # KEY_UPDATE_TIME and Compat:<=7.5.0 value = suite_identity.get(key) if value: + # Remove new-line characters for readability + value_to_output = str(value).replace("\n", " ") + # Remove whitespace from multiline indents for readability + while " " in value_to_output: + value_to_output = value_to_output.replace(" ", " ") print "%s|%s|%s|%s|%s" % ( name, owner, host, key, - str(value).replace("\n", " ") - ) + value_to_output) totals = suite_identity.get(KEY_STATES) if totals is None: continue @@ -253,6 +269,35 @@ def main(): print "%s|%s|%s|%s|%s" % (name, owner, host, key, state_line) continue + if options.json_format: + json_info = {"name": name, "owner": owner, "host": host, + "port": port} + for key in [KEY_TITLE, KEY_DESCRIPTION, KEY_UPDATE_TIME]: + try: + value = suite_identity[KEY_META].get(key) + except KeyError: + # KEY_UPDATE_TIME and Compat:<=7.5.0 + value = suite_identity.get(key) + if value: + # Escape new-line characters for valid JSON + value_to_dump = str(value).replace("\n", "\\n") + # Remove whitespace from multiline indents for readability + while " " in value_to_dump: + value_to_dump = value_to_dump.replace(" ", " ") + additions = {key: value_to_dump} + json_info.update(additions) + totals = suite_identity.get(KEY_STATES) + if totals is None: + continue + point_state_lines = get_point_state_count_lines( + *totals, use_color=options.color) + for point, state_line in point_state_lines: + if point: + more_additions = {KEY_STATES: {point: state_line}} + json_info.update(more_additions) + print json.dumps(json_info) + continue + line = '%s %s@%s:%s' % (name, owner, host, port) if options.describe or options.print_totals: if skip_one: diff --git a/tests/authentication/00-identity.t b/tests/authentication/00-identity.t index fe5853c5a2..5b1ba765f5 100644 --- a/tests/authentication/00-identity.t +++ b/tests/authentication/00-identity.t @@ -18,7 +18,7 @@ # Test authentication - privilege 'identity'. . $(dirname $0)/test_header -set_test_number 15 +set_test_number 17 install_suite "${TEST_NAME_BASE}" basic @@ -70,20 +70,39 @@ mv "${SRV_D}/passphrase" "${SRV_D}/passphrase.DIS" # Check scan --full output. cylc scan --comms-timeout=5 -fb -n "${SUITE_NAME}" 'localhost' \ - >'scan.out' 2>'/dev/null' -cmp_ok scan.out << __END__ + >'scan-f.out' 2>'/dev/null' +cmp_ok scan-f.out << __END__ ${SUITE_NAME} ${USER}@localhost:${PORT} (description and state totals withheld) __END__ # Check scan --describe output. cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}" 'localhost' \ - >'scan.out' 2>'/dev/null' -cmp_ok scan.out << __END__ + >'scan-d.out' 2>'/dev/null' +cmp_ok scan-d.out << __END__ ${SUITE_NAME} ${USER}@localhost:${PORT} (description and state totals withheld) __END__ +# Check scan --raw output. +cylc scan --comms-timeout=5 -rb -n "${SUITE_NAME}" 'localhost' \ + >'scan-r.out' 2>'/dev/null' +cmp_ok scan-r.out << __END__ +${SUITE_NAME}|${USER}|localhost|port|${PORT} +__END__ + +# Check scan --json output. +cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" 'localhost' \ + >'scan-j.out' 2>'/dev/null' +cmp_json_ok 'scan-j.out' 'scan-j.out' << __END__ +{ + "host": "localhost", + "name": "${SUITE_NAME}", + "owner": "${USER}", + "port": ${PORT} +} +__END__ + # "cylc show" should be denied. TEST_NAME="${TEST_NAME_BASE}-show" run_fail "${TEST_NAME}" cylc show "${SUITE_NAME}" diff --git a/tests/authentication/01-description.t b/tests/authentication/01-description.t index 3b3028437f..02210bdbf2 100644 --- a/tests/authentication/01-description.t +++ b/tests/authentication/01-description.t @@ -18,7 +18,7 @@ # Test authentication - privilege 'description'. . $(dirname $0)/test_header -set_test_number 8 +set_test_number 10 install_suite "${TEST_NAME_BASE}" basic @@ -43,8 +43,8 @@ mv "${SRV_D}/passphrase" "${SRV_D}/passphrase.DIS" # Check scan --full output. HOST="$(sed -n 's/^CYLC_SUITE_HOST=//p' "${SRV_D}/contact")" PORT="$(sed -n 's/^CYLC_SUITE_PORT=//p' "${SRV_D}/contact")" -cylc scan --comms-timeout=5 -fb -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' -cmp_ok 'scan.out' <<__END__ +cylc scan --comms-timeout=5 -fb -n "${SUITE_NAME}" >'scan-f.out' 2>'/dev/null' +cmp_ok 'scan-f.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." @@ -63,8 +63,8 @@ ${SUITE_NAME} ${USER}@${HOST}:${PORT} __END__ # Check scan --describe output. -cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' -cmp_ok 'scan.out' <<__END__ +cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}" >'scan-d.out' 2>'/dev/null' +cmp_ok 'scan-d.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." @@ -81,6 +81,28 @@ ${SUITE_NAME} ${USER}@${HOST}:${PORT} "something_custom" __END__ +# Check scan --raw output. +cylc scan --comms-timeout=5 -rb -n "${SUITE_NAME}" >'scan-r.out' 2>'/dev/null' +cmp_ok 'scan-r.out' <<__END__ +${SUITE_NAME}|${USER}|${HOST}|port|${PORT} +${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. +${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. \ +Here we test out a multi-line description! +__END__ + +# Check scan --json output. +cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' +cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ +{ + "host": "${HOST}", + "name": "${SUITE_NAME}", + "title": "Authentication test suite.", + "owner": "${USER}", + "port": "${PORT}", + "description": "Stalls when the first task fails.\\\n Here we test out a multi-line description!" +} +__END__ + # "cylc show" (suite info) OK. TEST_NAME="${TEST_NAME_BASE}-show1" run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" diff --git a/tests/authentication/02-state-totals.t b/tests/authentication/02-state-totals.t index 315827a67e..99fd88f9a9 100644 --- a/tests/authentication/02-state-totals.t +++ b/tests/authentication/02-state-totals.t @@ -18,7 +18,7 @@ # Test authentication - privilege 'state-totals'. . $(dirname $0)/test_header -set_test_number 8 +set_test_number 10 install_suite "${TEST_NAME_BASE}" basic @@ -43,8 +43,8 @@ mv "${SRV_D}/passphrase" "${SRV_D}/passphrase.DIS" # Check scan --full output. HOST="$(sed -n 's/^CYLC_SUITE_HOST=//p' "${SRV_D}/contact")" PORT="$(sed -n 's/^CYLC_SUITE_PORT=//p' "${SRV_D}/contact")" -cylc scan --comms-timeout=5 -fb -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' -cmp_ok 'scan.out' <<__END__ +cylc scan --comms-timeout=5 -fb -n "${SUITE_NAME}" >'scan-f.out' 2>'/dev/null' +cmp_ok 'scan-f.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." @@ -65,8 +65,8 @@ ${SUITE_NAME} ${USER}@${HOST}:${PORT} __END__ # Check scan --describe output. -cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' -cmp_ok 'scan.out' <<__END__ +cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}" >'scan-d.out' 2>'/dev/null' +cmp_ok 'scan-d.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." @@ -83,6 +83,32 @@ ${SUITE_NAME} ${USER}@${HOST}:${PORT} "something_custom" __END__ +# Check scan --raw output. +cylc scan --comms-timeout=5 -rb -n "${SUITE_NAME}" >'scan-r.out' 2>'/dev/null' +cmp_ok 'scan-r.out' <<__END__ +${SUITE_NAME}|${USER}|${HOST}|port|${PORT} +${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. +${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! +${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:1 +${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 +__END__ + +# Check scan --json output. +cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' +cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ +{ + "states": { + "1": "failed:1 waiting:1" + }, + "host": "${HOST}", + "name": "${SUITE_NAME}", + "title": "Authentication test suite.", + "owner": "${USER}", + "port": "${PORT}", + "description": "Stalls when the first task fails.\\\n Here we test out a multi-line description!" +} +__END__ + # "cylc show" (suite info) OK. TEST_NAME="${TEST_NAME_BASE}-show1" run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" diff --git a/tests/authentication/03-full-read.t b/tests/authentication/03-full-read.t index 367e3de96c..33687bc3cb 100644 --- a/tests/authentication/03-full-read.t +++ b/tests/authentication/03-full-read.t @@ -18,7 +18,7 @@ # Test authentication - privilege 'full-read'. . $(dirname $0)/test_header -set_test_number 9 +set_test_number 11 install_suite "${TEST_NAME_BASE}" basic @@ -43,8 +43,8 @@ mv "${SRV_D}/passphrase" "${SRV_D}/passphrase.DIS" # Check scan --full output. HOST="$(sed -n 's/^CYLC_SUITE_HOST=//p' "${SRV_D}/contact")" PORT="$(sed -n 's/^CYLC_SUITE_PORT=//p' "${SRV_D}/contact")" -cylc scan --comms-timeout=5 -fb -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' -cmp_ok 'scan.out' <<__END__ +cylc scan --comms-timeout=5 -fb -n "${SUITE_NAME}" >'scan-f.out' 2>'/dev/null' +cmp_ok 'scan-f.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." @@ -65,8 +65,8 @@ ${SUITE_NAME} ${USER}@${HOST}:${PORT} __END__ # Check scan --describe output. -cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' -cmp_ok 'scan.out' <<__END__ +cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}" >'scan-d.out' 2>'/dev/null' +cmp_ok 'scan-d.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." @@ -83,6 +83,32 @@ ${SUITE_NAME} ${USER}@${HOST}:${PORT} "something_custom" __END__ +# Check scan --raw output. +cylc scan --comms-timeout=5 -rb -n "${SUITE_NAME}" >'scan-r.out' 2>'/dev/null' +cmp_ok 'scan-r.out' <<__END__ +${SUITE_NAME}|${USER}|${HOST}|port|${PORT} +${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. +${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! +${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:1 +${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 +__END__ + +# Check scan --json output. +cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' +cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ +{ + "states": { + "1": "failed:1 waiting:1" + }, + "host": "${HOST}", + "name": "${SUITE_NAME}", + "title": "Authentication test suite.", + "owner": "${USER}", + "port": "${PORT}", + "description": "Stalls when the first task fails.\\\n Here we test out a multi-line description!" +} +__END__ + # "cylc show" (suite info) OK. TEST_NAME="${TEST_NAME_BASE}-show1" run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" diff --git a/tests/authentication/04-shutdown.t b/tests/authentication/04-shutdown.t index c9d79bbbc5..b4981b935b 100644 --- a/tests/authentication/04-shutdown.t +++ b/tests/authentication/04-shutdown.t @@ -18,7 +18,7 @@ # Test authentication - privilege 'shutdown'. . $(dirname $0)/test_header -set_test_number 10 +set_test_number 12 install_suite "${TEST_NAME_BASE}" basic @@ -43,8 +43,8 @@ mv "${SRV_D}/passphrase" "${SRV_D}/passphrase.DIS" HOST="$(sed -n 's/^CYLC_SUITE_HOST=//p' "${SRV_D}/contact")" PORT="$(sed -n 's/^CYLC_SUITE_PORT=//p' "${SRV_D}/contact")" -cylc scan --comms-timeout=5 -fb -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' -cmp_ok 'scan.out' <<__END__ +cylc scan --comms-timeout=5 -fb -n "${SUITE_NAME}" >'scan-f.out' 2>'/dev/null' +cmp_ok 'scan-f.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." @@ -64,8 +64,8 @@ ${SUITE_NAME} ${USER}@${HOST}:${PORT} 1 failed:1 waiting:1 __END__ -cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' -cmp_ok 'scan.out' <<__END__ +cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}" >'scan-d.out' 2>'/dev/null' +cmp_ok 'scan-d.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." @@ -82,6 +82,32 @@ ${SUITE_NAME} ${USER}@${HOST}:${PORT} "something_custom" __END__ +# Check scan --raw output. +cylc scan --comms-timeout=5 -rb -n "${SUITE_NAME}" >'scan-r.out' 2>'/dev/null' +cmp_ok 'scan-r.out' <<__END__ +${SUITE_NAME}|${USER}|${HOST}|port|${PORT} +${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. +${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! +${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:1 +${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 +__END__ + +# Check scan --json output. +cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' +cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ +{ + "states": { + "1": "failed:1 waiting:1" + }, + "host": "${HOST}", + "name": "${SUITE_NAME}", + "title": "Authentication test suite.", + "owner": "${USER}", + "port": "${PORT}", + "description": "Stalls when the first task fails.\\\n Here we test out a multi-line description!" +} +__END__ + # "cylc show" (suite info) OK. TEST_NAME="${TEST_NAME_BASE}-show1" run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" diff --git a/tests/authentication/05-full-control.t b/tests/authentication/05-full-control.t index c234e43fa1..2fffc3cf54 100644 --- a/tests/authentication/05-full-control.t +++ b/tests/authentication/05-full-control.t @@ -18,7 +18,7 @@ # Test authentication - privilege 'full-control' (with passphrase). . $(dirname $0)/test_header -set_test_number 10 +set_test_number 12 install_suite "${TEST_NAME_BASE}" basic @@ -41,8 +41,8 @@ cylc suite-state "${SUITE_NAME}" --task=foo --status=failed --point=1 \ SRV_D="$(cylc get-global-config --print-run-dir)/${SUITE_NAME}/.service" HOST="$(sed -n 's/^CYLC_SUITE_HOST=//p' "${SRV_D}/contact")" PORT="$(sed -n 's/^CYLC_SUITE_PORT=//p' "${SRV_D}/contact")" -cylc scan --comms-timeout=5 -fb -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' -cmp_ok 'scan.out' <<__END__ +cylc scan --comms-timeout=5 -fb -n "${SUITE_NAME}" >'scan-f.out' 2>'/dev/null' +cmp_ok 'scan-f.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." @@ -63,8 +63,8 @@ ${SUITE_NAME} ${USER}@${HOST}:${PORT} __END__ # Check scan --describe output. -cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' -cmp_ok 'scan.out' <<__END__ +cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}" >'scan-d.out' 2>'/dev/null' +cmp_ok 'scan-d.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." @@ -81,6 +81,32 @@ ${SUITE_NAME} ${USER}@${HOST}:${PORT} "something_custom" __END__ +# Check scan --raw output. +cylc scan --comms-timeout=5 -rb -n "${SUITE_NAME}" >'scan-r.out' 2>'/dev/null' +cmp_ok 'scan-r.out' <<__END__ +${SUITE_NAME}|${USER}|${HOST}|port|${PORT} +${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. +${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! +${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:1 +${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 +__END__ + +# Check scan --json output. +cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' +cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ +{ + "states": { + "1": "failed:1 waiting:1" + }, + "host": "${HOST}", + "name": "${SUITE_NAME}", + "title": "Authentication test suite.", + "owner": "${USER}", + "port": "${PORT}", + "description": "Stalls when the first task fails.\\\n Here we test out a multi-line description!" +} +__END__ + # "cylc show" (suite info) OK. TEST_NAME="${TEST_NAME_BASE}-show1" run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" diff --git a/tests/authentication/06-suite-override.t b/tests/authentication/06-suite-override.t index 7c44946640..9bf2bbf497 100644 --- a/tests/authentication/06-suite-override.t +++ b/tests/authentication/06-suite-override.t @@ -19,7 +19,7 @@ # (Suite overrides global privilege 'identity'.) . $(dirname $0)/test_header -set_test_number 10 +set_test_number 12 install_suite "${TEST_NAME_BASE}" override @@ -44,8 +44,8 @@ mv "${SRV_D}/passphrase" "${SRV_D}/passphrase.DIS" HOST="$(sed -n 's/^CYLC_SUITE_HOST=//p' "${SRV_D}/contact")" PORT="$(sed -n 's/^CYLC_SUITE_PORT=//p' "${SRV_D}/contact")" -cylc scan --comms-timeout=5 -fb -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' -cmp_ok 'scan.out' <<__END__ +cylc scan --comms-timeout=5 -fb -n "${SUITE_NAME}" >'scan-f.out' 2>'/dev/null' +cmp_ok 'scan-f.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." @@ -61,8 +61,8 @@ ${SUITE_NAME} ${USER}@${HOST}:${PORT} 1 failed:1 waiting:1 __END__ -cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}" >'scan.out' 2>'/dev/null' -cmp_ok 'scan.out' <<__END__ +cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}" >'scan-d.out' 2>'/dev/null' +cmp_ok 'scan-d.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." @@ -75,6 +75,32 @@ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Suite overrides global authentication settings." __END__ +# Check scan --raw output. +cylc scan --comms-timeout=5 -rb -n "${SUITE_NAME}" >'scan-r.out' 2>'/dev/null' +cmp_ok 'scan-r.out' <<__END__ +${SUITE_NAME}|${USER}|${HOST}|port|${PORT} +${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. +${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Suite overrides global authentication settings. +${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:1 +${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 +__END__ + +# Check scan --json output. +cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' +cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ +{ + "states": { + "1": "failed:1 waiting:1" + }, + "host": "${HOST}", + "name": "${SUITE_NAME}", + "title": "Authentication test suite.", + "owner": "${USER}", + "port": "${PORT}", + "description": "Stalls when the first task fails.\\\nSuite overrides global authentication settings." +} +__END__ + # "cylc show" (suite info) OK. TEST_NAME="${TEST_NAME_BASE}-show1" run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" diff --git a/tests/authentication/07-sha-hash.t b/tests/authentication/07-sha-hash.t index d0f4f7e0b5..0faec15ff0 100644 --- a/tests/authentication/07-sha-hash.t +++ b/tests/authentication/07-sha-hash.t @@ -18,7 +18,7 @@ # Test authentication - using SHA1 hash option. . $(dirname $0)/test_header -set_test_number 10 +set_test_number 12 install_suite "${TEST_NAME_BASE}" basic @@ -44,8 +44,8 @@ cylc suite-state "${SUITE_NAME}" --task=foo --status=failed --point=1 \ SRV_D="$(cylc get-global-config --print-run-dir)/${SUITE_NAME}/.service" HOST="$(sed -n 's/^CYLC_SUITE_HOST=//p' "${SRV_D}/contact")" PORT="$(sed -n 's/^CYLC_SUITE_PORT=//p' "${SRV_D}/contact")" -cylc scan --comms-timeout=5 -fb -n "${SUITE_NAME}">'scan.out' 2>'/dev/null' -cmp_ok 'scan.out' <<__END__ +cylc scan --comms-timeout=5 -fb -n "${SUITE_NAME}">'scan-f.out' 2>'/dev/null' +cmp_ok 'scan-f.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." @@ -66,8 +66,8 @@ ${SUITE_NAME} ${USER}@${HOST}:${PORT} __END__ # Check scan --describe output. -cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}">'scan.out' 2>'/dev/null' -cmp_ok 'scan.out' <<__END__ +cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}">'scan-d.out' 2>'/dev/null' +cmp_ok 'scan-d.out' <<__END__ ${SUITE_NAME} ${USER}@${HOST}:${PORT} Title: "Authentication test suite." @@ -84,6 +84,32 @@ ${SUITE_NAME} ${USER}@${HOST}:${PORT} "something_custom" __END__ +# Check scan --raw output. +cylc scan --comms-timeout=5 -rb -n "${SUITE_NAME}" >'scan-r.out' 2>'/dev/null' +cmp_ok 'scan-r.out' <<__END__ +${SUITE_NAME}|${USER}|${HOST}|port|${PORT} +${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. +${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! +${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:1 +${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 +__END__ + +# Check scan --json output. +cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' +cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ +{ + "states": { + "1": "failed:1 waiting:1" + }, + "host": "${HOST}", + "name": "${SUITE_NAME}", + "title": "Authentication test suite.", + "owner": "${USER}", + "port": "${PORT}", + "description": "Stalls when the first task fails.\\\n Here we test out a multi-line description!" +} +__END__ + # "cylc show" (suite info) OK. TEST_NAME="${TEST_NAME_BASE}-show1" run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" From ae05b98729abc87f5ef6b9cbf83fb5d8d08e208f Mon Sep 17 00:00:00 2001 From: Sadie Bartholomew Date: Wed, 3 Jan 2018 17:13:16 +0000 Subject: [PATCH 15/21] Responding to feedback and fix tests --- bin/cylc-scan | 48 ++++++++++++------------ bin/cylc-show | 6 +-- tests/authentication/00-identity.t | 9 +++-- tests/authentication/01-description.t | 13 ++++--- tests/authentication/02-state-totals.t | 22 ++++++----- tests/authentication/03-full-read.t | 22 ++++++----- tests/authentication/04-shutdown.t | 22 ++++++----- tests/authentication/05-full-control.t | 22 ++++++----- tests/authentication/06-suite-override.t | 22 ++++++----- tests/authentication/07-sha-hash.t | 22 ++++++----- 10 files changed, 118 insertions(+), 90 deletions(-) diff --git a/bin/cylc-scan b/bin/cylc-scan index 9f0f0dfeaf..e4aa6f1bc7 100755 --- a/bin/cylc-scan +++ b/bin/cylc-scan @@ -226,6 +226,7 @@ def main(): if not args: return + json_dict = {} skip_one = True for host, port, suite_identity in scan_many(args, options.comms_timeout): name = suite_identity[KEY_NAME] @@ -249,14 +250,10 @@ def main(): # KEY_UPDATE_TIME and Compat:<=7.5.0 value = suite_identity.get(key) if value: - # Remove new-line characters for readability - value_to_output = str(value).replace("\n", " ") - # Remove whitespace from multiline indents for readability - while " " in value_to_output: - value_to_output = value_to_output.replace(" ", " ") - print "%s|%s|%s|%s|%s" % ( - name, owner, host, key, - value_to_output) + print "%s|%s|%s|%s|%s" % (name, owner, host, key, + ' '.join([x.strip() for x in + str(value).split('\n') + if x])) totals = suite_identity.get(KEY_STATES) if totals is None: continue @@ -270,8 +267,7 @@ def main(): continue if options.json_format: - json_info = {"name": name, "owner": owner, "host": host, - "port": port} + suite_inf = {"owner": owner, "host": host, "port": port} for key in [KEY_TITLE, KEY_DESCRIPTION, KEY_UPDATE_TIME]: try: value = suite_identity[KEY_META].get(key) @@ -279,23 +275,25 @@ def main(): # KEY_UPDATE_TIME and Compat:<=7.5.0 value = suite_identity.get(key) if value: - # Escape new-line characters for valid JSON - value_to_dump = str(value).replace("\n", "\\n") - # Remove whitespace from multiline indents for readability - while " " in value_to_dump: - value_to_dump = value_to_dump.replace(" ", " ") - additions = {key: value_to_dump} - json_info.update(additions) + suite_inf.update({key: ' '.join([x.strip() for x in + str(value).split('\n') if x])}) totals = suite_identity.get(KEY_STATES) if totals is None: continue point_state_lines = get_point_state_count_lines( *totals, use_color=options.color) for point, state_line in point_state_lines: + state_dict = {} if point: - more_additions = {KEY_STATES: {point: state_line}} - json_info.update(more_additions) - print json.dumps(json_info) + for state_inf in state_line.split(' '): + if ':' in state_inf: + key, value = state_inf.split(':') + try: + state_dict.update({key: int(value)}) + except (TypeError, ValueError): + state_dict.update({key: str(value)}) + suite_inf.update({KEY_STATES: {point: state_dict}}) + json_dict[name] = suite_inf continue line = '%s %s@%s:%s' % (name, owner, host, port) @@ -330,16 +328,16 @@ def main(): else: print (indent + bold("Title:\n") + indent * 2 + '"%s"' % meta_items["title"]) - for metaitem in meta_items.keys(): + for metaitem, metavalue in meta_items.items(): if metaitem != "title": if metaitem == "description" or metaitem == "group": print indent + bold(metaitem.capitalize() + ":") else: print indent + bold(metaitem + ":") - if not meta_items[metaitem]: + if not metavalue: lines = "(no %s)" % metaitem else: - lines = '"%s"' % meta_items[metaitem] + lines = '"%s"' % metavalue line1 = True for line in lines.split('\n'): line = line.lstrip() @@ -364,6 +362,10 @@ def main(): point_prefix = "%s " % point print indent * 2 + "%s%s" % (point_prefix, state_line) + # Need a single filtered JSON structure => only dump after final iteration + if options.json_format: + print json.dumps(json_dict, indent=4) + def get_point_state_count_lines(state_count_totals, state_count_cycles, use_color=False): diff --git a/bin/cylc-show b/bin/cylc-show index 3d3485ad53..c27988fbcf 100755 --- a/bin/cylc-show +++ b/bin/cylc-show @@ -64,7 +64,7 @@ def main(): # Print suite info. suite_info = pclient.get_info('get_suite_info') if options.json: - print json.dumps(suite_info) + print json.dumps(suite_info, indent=4) else: for key, value in sorted(suite_info.items(), reverse=True): print '%s: %s' % (key, value or "(not given)") @@ -76,7 +76,7 @@ def main(): if task_names: results = pclient.get_info('get_task_info', names=task_names) if options.json: - print json.dumps(results) + print json.dumps(results, indent=4) else: for task_name, result in sorted(results.items()): if len(results) > 1: @@ -89,7 +89,7 @@ def main(): 'get_task_requisites', items=task_ids, list_prereqs=options.list_prereqs) if options.json: - print json.dumps(results) + print json.dumps(results, indent=4) else: for task_id, result in sorted(results.items()): if len(results) > 1: diff --git a/tests/authentication/00-identity.t b/tests/authentication/00-identity.t index 5b1ba765f5..13c6bca29c 100644 --- a/tests/authentication/00-identity.t +++ b/tests/authentication/00-identity.t @@ -96,10 +96,11 @@ cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" 'localhost' \ >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' << __END__ { - "host": "localhost", - "name": "${SUITE_NAME}", - "owner": "${USER}", - "port": ${PORT} + "${SUITE_NAME}": { + "owner": "${USER}", + "host": "localhost", + "port": ${PORT} + } } __END__ diff --git a/tests/authentication/01-description.t b/tests/authentication/01-description.t index 02210bdbf2..5767ac6b03 100644 --- a/tests/authentication/01-description.t +++ b/tests/authentication/01-description.t @@ -94,12 +94,13 @@ __END__ cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ { - "host": "${HOST}", - "name": "${SUITE_NAME}", - "title": "Authentication test suite.", - "owner": "${USER}", - "port": "${PORT}", - "description": "Stalls when the first task fails.\\\n Here we test out a multi-line description!" + "${SUITE_NAME}": { + "owner": "${USER}", + "host": "${HOST}", + "port": "${PORT}", + "title": "Authentication test suite.", + "description": "Stalls when the first task fails. Here we test out a multi-line description!" + } } __END__ diff --git a/tests/authentication/02-state-totals.t b/tests/authentication/02-state-totals.t index 99fd88f9a9..1b32840eb8 100644 --- a/tests/authentication/02-state-totals.t +++ b/tests/authentication/02-state-totals.t @@ -97,15 +97,19 @@ __END__ cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ { - "states": { - "1": "failed:1 waiting:1" - }, - "host": "${HOST}", - "name": "${SUITE_NAME}", - "title": "Authentication test suite.", - "owner": "${USER}", - "port": "${PORT}", - "description": "Stalls when the first task fails.\\\n Here we test out a multi-line description!" + "${SUITE_NAME}": { + "owner": "${USER}", + "host": "${HOST}", + "port": "${PORT}", + "title": "Authentication test suite.", + "description": "Stalls when the first task fails. Here we test out a multi-line description!", + "states": { + "1": { + "failed": 1, + "waiting": 1 + } + } + } } __END__ diff --git a/tests/authentication/03-full-read.t b/tests/authentication/03-full-read.t index 33687bc3cb..8fac6751fd 100644 --- a/tests/authentication/03-full-read.t +++ b/tests/authentication/03-full-read.t @@ -97,15 +97,19 @@ __END__ cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ { - "states": { - "1": "failed:1 waiting:1" - }, - "host": "${HOST}", - "name": "${SUITE_NAME}", - "title": "Authentication test suite.", - "owner": "${USER}", - "port": "${PORT}", - "description": "Stalls when the first task fails.\\\n Here we test out a multi-line description!" + "${SUITE_NAME}": { + "owner": "${USER}", + "host": "${HOST}", + "port": "${PORT}", + "title": "Authentication test suite.", + "description": "Stalls when the first task fails. Here we test out a multi-line description!", + "states": { + "1": { + "failed": 1, + "waiting": 1 + } + } + } } __END__ diff --git a/tests/authentication/04-shutdown.t b/tests/authentication/04-shutdown.t index b4981b935b..dd6568682b 100644 --- a/tests/authentication/04-shutdown.t +++ b/tests/authentication/04-shutdown.t @@ -96,15 +96,19 @@ __END__ cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ { - "states": { - "1": "failed:1 waiting:1" - }, - "host": "${HOST}", - "name": "${SUITE_NAME}", - "title": "Authentication test suite.", - "owner": "${USER}", - "port": "${PORT}", - "description": "Stalls when the first task fails.\\\n Here we test out a multi-line description!" + "${SUITE_NAME}": { + "owner": "${USER}", + "host": "${HOST}", + "port": "${PORT}", + "title": "Authentication test suite.", + "description": "Stalls when the first task fails. Here we test out a multi-line description!", + "states": { + "1": { + "failed": 1, + "waiting": 1 + } + } + } } __END__ diff --git a/tests/authentication/05-full-control.t b/tests/authentication/05-full-control.t index 2fffc3cf54..01b9dcf4ae 100644 --- a/tests/authentication/05-full-control.t +++ b/tests/authentication/05-full-control.t @@ -95,15 +95,19 @@ __END__ cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ { - "states": { - "1": "failed:1 waiting:1" - }, - "host": "${HOST}", - "name": "${SUITE_NAME}", - "title": "Authentication test suite.", - "owner": "${USER}", - "port": "${PORT}", - "description": "Stalls when the first task fails.\\\n Here we test out a multi-line description!" + "${SUITE_NAME}": { + "owner": "${USER}", + "host": "${HOST}", + "port": "${PORT}", + "title": "Authentication test suite.", + "description": "Stalls when the first task fails. Here we test out a multi-line description!", + "states": { + "1": { + "failed": 1, + "waiting": 1 + } + } + } } __END__ diff --git a/tests/authentication/06-suite-override.t b/tests/authentication/06-suite-override.t index 9bf2bbf497..79a0b2fa80 100644 --- a/tests/authentication/06-suite-override.t +++ b/tests/authentication/06-suite-override.t @@ -89,15 +89,19 @@ __END__ cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ { - "states": { - "1": "failed:1 waiting:1" - }, - "host": "${HOST}", - "name": "${SUITE_NAME}", - "title": "Authentication test suite.", - "owner": "${USER}", - "port": "${PORT}", - "description": "Stalls when the first task fails.\\\nSuite overrides global authentication settings." + "${SUITE_NAME}": { + "owner": "${USER}", + "host": "${HOST}", + "port": "${PORT}", + "title": "Authentication test suite.", + "description": "Stalls when the first task fails. Suite overrides global authentication settings.", + "states": { + "1": { + "failed": 1, + "waiting": 1 + } + } + } } __END__ diff --git a/tests/authentication/07-sha-hash.t b/tests/authentication/07-sha-hash.t index 0faec15ff0..54f14c1580 100644 --- a/tests/authentication/07-sha-hash.t +++ b/tests/authentication/07-sha-hash.t @@ -98,15 +98,19 @@ __END__ cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ { - "states": { - "1": "failed:1 waiting:1" - }, - "host": "${HOST}", - "name": "${SUITE_NAME}", - "title": "Authentication test suite.", - "owner": "${USER}", - "port": "${PORT}", - "description": "Stalls when the first task fails.\\\n Here we test out a multi-line description!" + "${SUITE_NAME}": { + "owner": "${USER}", + "host": "${HOST}", + "port": "${PORT}", + "title": "Authentication test suite.", + "description": "Stalls when the first task fails. Here we test out a multi-line description!", + "states": { + "1": { + "failed": 1, + "waiting": 1 + } + } + } } __END__ From 0791616fb9d6853e1cdaf236bb79ce472be67d18 Mon Sep 17 00:00:00 2001 From: Sadie Bartholomew Date: Thu, 4 Jan 2018 14:37:48 +0000 Subject: [PATCH 16/21] Removal of use_color argument as review response --- bin/cylc-scan | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/cylc-scan b/bin/cylc-scan index e4aa6f1bc7..7648a943e7 100755 --- a/bin/cylc-scan +++ b/bin/cylc-scan @@ -280,8 +280,7 @@ def main(): totals = suite_identity.get(KEY_STATES) if totals is None: continue - point_state_lines = get_point_state_count_lines( - *totals, use_color=options.color) + point_state_lines = get_point_state_count_lines(*totals) for point, state_line in point_state_lines: state_dict = {} if point: From a1728dafe3e61c30cba8a7ef1b87285b9b25b9bb Mon Sep 17 00:00:00 2001 From: Sadie Bartholomew Date: Fri, 5 Jan 2018 11:03:29 +0000 Subject: [PATCH 17/21] Respond to feedback with meta key and flow amendments --- bin/cylc-scan | 86 ++++++++++++++---------- tests/authentication/00-identity.t | 8 +-- tests/authentication/01-description.t | 14 ++-- tests/authentication/02-state-totals.t | 32 +++++---- tests/authentication/03-full-read.t | 32 +++++---- tests/authentication/04-shutdown.t | 32 +++++---- tests/authentication/05-full-control.t | 32 +++++---- tests/authentication/06-suite-override.t | 28 ++++---- tests/authentication/07-sha-hash.t | 32 +++++---- 9 files changed, 166 insertions(+), 130 deletions(-) diff --git a/bin/cylc-scan b/bin/cylc-scan index 7648a943e7..d9b1ce385b 100755 --- a/bin/cylc-scan +++ b/bin/cylc-scan @@ -243,17 +243,20 @@ def main(): if options.raw_format: print "%s|%s|%s|port|%s" % (name, owner, host, port) - for key in [KEY_TITLE, KEY_DESCRIPTION, KEY_UPDATE_TIME]: - try: - value = suite_identity[KEY_META].get(key) - except KeyError: - # KEY_UPDATE_TIME and Compat:<=7.5.0 - value = suite_identity.get(key) - if value: - print "%s|%s|%s|%s|%s" % (name, owner, host, key, - ' '.join([x.strip() for x in - str(value).split('\n') - if x])) + try: + items = suite_identity[KEY_META] + except KeyError: + # Compat:<=7.5.0 + items = { + "title": suite_identity.get(KEY_TITLE), + "description": suite_identity.get(KEY_DESCRIPTION), + "update time": suite_identity.get(KEY_UPDATE_TIME) + } + for key in items: + if not items[key] == "" and items[key] is not None: + cleaned = ' '.join([x.strip() for x + in str(items[key]).split('\n') if x]) + print "%s|%s|%s|%s|%s" % (name, owner, host, key, cleaned) totals = suite_identity.get(KEY_STATES) if totals is None: continue @@ -268,30 +271,43 @@ def main(): if options.json_format: suite_inf = {"owner": owner, "host": host, "port": port} - for key in [KEY_TITLE, KEY_DESCRIPTION, KEY_UPDATE_TIME]: - try: - value = suite_identity[KEY_META].get(key) - except KeyError: - # KEY_UPDATE_TIME and Compat:<=7.5.0 - value = suite_identity.get(key) - if value: - suite_inf.update({key: ' '.join([x.strip() for x in - str(value).split('\n') if x])}) - totals = suite_identity.get(KEY_STATES) - if totals is None: - continue - point_state_lines = get_point_state_count_lines(*totals) - for point, state_line in point_state_lines: - state_dict = {} - if point: - for state_inf in state_line.split(' '): - if ':' in state_inf: - key, value = state_inf.split(':') - try: - state_dict.update({key: int(value)}) - except (TypeError, ValueError): - state_dict.update({key: str(value)}) - suite_inf.update({KEY_STATES: {point: state_dict}}) + try: + meta_items = suite_identity[KEY_META] + except KeyError: + # Compat:<=7.5.0 + meta_items = { + "title": suite_identity.get(KEY_TITLE), + "description": suite_identity.get(KEY_DESCRIPTION), + "update time": suite_identity.get(KEY_UPDATE_TIME) + } + clean_meta_items = {} + for key in meta_items: + if not meta_items[key] == "" and meta_items[key] is not None: + if '\n' in meta_items[key]: + clean_meta_items.update({ + key: ' '.join([x.strip() for x in + str(meta_items[key]).split('\n') if + x])}) + else: + clean_meta_items.update({key: meta_items[key]}) + suite_inf.update({"meta": clean_meta_items}) + totals = suite_identity.get(KEY_STATES) + point_inf = {} + if totals is None: + continue + point_state_lines = get_point_state_count_lines(*totals) + for point, state_line in point_state_lines: + state_dict = {} + if point: + for state_inf in state_line.split(' '): + if ':' in state_inf: + key, value = state_inf.split(':') + try: + state_dict.update({key: int(value)}) + except (TypeError, ValueError): + state_dict.update({key: str(value)}) + point_inf.update({point: state_dict}) + suite_inf.update({KEY_STATES: point_inf}) json_dict[name] = suite_inf continue diff --git a/tests/authentication/00-identity.t b/tests/authentication/00-identity.t index 13c6bca29c..f44965b169 100644 --- a/tests/authentication/00-identity.t +++ b/tests/authentication/00-identity.t @@ -95,13 +95,7 @@ __END__ cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" 'localhost' \ >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' << __END__ -{ - "${SUITE_NAME}": { - "owner": "${USER}", - "host": "localhost", - "port": ${PORT} - } -} +{} __END__ # "cylc show" should be denied. diff --git a/tests/authentication/01-description.t b/tests/authentication/01-description.t index 5767ac6b03..3734d4a347 100644 --- a/tests/authentication/01-description.t +++ b/tests/authentication/01-description.t @@ -85,23 +85,17 @@ __END__ cylc scan --comms-timeout=5 -rb -n "${SUITE_NAME}" >'scan-r.out' 2>'/dev/null' cmp_ok 'scan-r.out' <<__END__ ${SUITE_NAME}|${USER}|${HOST}|port|${PORT} -${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. \ Here we test out a multi-line description! +${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. +${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 +${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom __END__ # Check scan --json output. cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ -{ - "${SUITE_NAME}": { - "owner": "${USER}", - "host": "${HOST}", - "port": "${PORT}", - "title": "Authentication test suite.", - "description": "Stalls when the first task fails. Here we test out a multi-line description!" - } -} +{} __END__ # "cylc show" (suite info) OK. diff --git a/tests/authentication/02-state-totals.t b/tests/authentication/02-state-totals.t index 1b32840eb8..5435963dd4 100644 --- a/tests/authentication/02-state-totals.t +++ b/tests/authentication/02-state-totals.t @@ -87,8 +87,10 @@ __END__ cylc scan --comms-timeout=5 -rb -n "${SUITE_NAME}" >'scan-r.out' 2>'/dev/null' cmp_ok 'scan-r.out' <<__END__ ${SUITE_NAME}|${USER}|${HOST}|port|${PORT} -${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! +${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. +${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 +${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom ${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:1 ${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 __END__ @@ -97,18 +99,22 @@ __END__ cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ { - "${SUITE_NAME}": { - "owner": "${USER}", - "host": "${HOST}", - "port": "${PORT}", - "title": "Authentication test suite.", - "description": "Stalls when the first task fails. Here we test out a multi-line description!", - "states": { - "1": { - "failed": 1, - "waiting": 1 - } - } + "${SUITE_NAME}":{ + "owner":"${USER}", + "states":{ + "1":{ + "failed":1, + "waiting":1 + } + }, + "host":"${HOST}", + "meta":{ + "another_metadata":"1", + "custom_metadata":"something_custom", + "description":"Stalls when the first task fails. Here we test out a multi-line description!", + "title":"Authentication test suite." + }, + "port":"${PORT}" } } __END__ diff --git a/tests/authentication/03-full-read.t b/tests/authentication/03-full-read.t index 8fac6751fd..caa950f1ce 100644 --- a/tests/authentication/03-full-read.t +++ b/tests/authentication/03-full-read.t @@ -87,8 +87,10 @@ __END__ cylc scan --comms-timeout=5 -rb -n "${SUITE_NAME}" >'scan-r.out' 2>'/dev/null' cmp_ok 'scan-r.out' <<__END__ ${SUITE_NAME}|${USER}|${HOST}|port|${PORT} -${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! +${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. +${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 +${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom ${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:1 ${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 __END__ @@ -97,18 +99,22 @@ __END__ cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ { - "${SUITE_NAME}": { - "owner": "${USER}", - "host": "${HOST}", - "port": "${PORT}", - "title": "Authentication test suite.", - "description": "Stalls when the first task fails. Here we test out a multi-line description!", - "states": { - "1": { - "failed": 1, - "waiting": 1 - } - } + "${SUITE_NAME}":{ + "owner":"${USER}", + "states":{ + "1":{ + "failed":1, + "waiting":1 + } + }, + "host":"${HOST}", + "meta":{ + "another_metadata":"1", + "custom_metadata":"something_custom", + "description":"Stalls when the first task fails. Here we test out a multi-line description!", + "title":"Authentication test suite." + }, + "port":"${PORT}" } } __END__ diff --git a/tests/authentication/04-shutdown.t b/tests/authentication/04-shutdown.t index dd6568682b..3e3d947b62 100644 --- a/tests/authentication/04-shutdown.t +++ b/tests/authentication/04-shutdown.t @@ -86,8 +86,10 @@ __END__ cylc scan --comms-timeout=5 -rb -n "${SUITE_NAME}" >'scan-r.out' 2>'/dev/null' cmp_ok 'scan-r.out' <<__END__ ${SUITE_NAME}|${USER}|${HOST}|port|${PORT} -${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! +${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. +${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 +${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom ${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:1 ${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 __END__ @@ -96,18 +98,22 @@ __END__ cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ { - "${SUITE_NAME}": { - "owner": "${USER}", - "host": "${HOST}", - "port": "${PORT}", - "title": "Authentication test suite.", - "description": "Stalls when the first task fails. Here we test out a multi-line description!", - "states": { - "1": { - "failed": 1, - "waiting": 1 - } - } + "${SUITE_NAME}":{ + "owner":"${USER}", + "states":{ + "1":{ + "failed":1, + "waiting":1 + } + }, + "host":"${HOST}", + "meta":{ + "another_metadata":"1", + "custom_metadata":"something_custom", + "description":"Stalls when the first task fails. Here we test out a multi-line description!", + "title":"Authentication test suite." + }, + "port":"${PORT}" } } __END__ diff --git a/tests/authentication/05-full-control.t b/tests/authentication/05-full-control.t index 01b9dcf4ae..64726780fc 100644 --- a/tests/authentication/05-full-control.t +++ b/tests/authentication/05-full-control.t @@ -85,8 +85,10 @@ __END__ cylc scan --comms-timeout=5 -rb -n "${SUITE_NAME}" >'scan-r.out' 2>'/dev/null' cmp_ok 'scan-r.out' <<__END__ ${SUITE_NAME}|${USER}|${HOST}|port|${PORT} -${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! +${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. +${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 +${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom ${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:1 ${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 __END__ @@ -95,18 +97,22 @@ __END__ cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ { - "${SUITE_NAME}": { - "owner": "${USER}", - "host": "${HOST}", - "port": "${PORT}", - "title": "Authentication test suite.", - "description": "Stalls when the first task fails. Here we test out a multi-line description!", - "states": { - "1": { - "failed": 1, - "waiting": 1 - } - } + "${SUITE_NAME}":{ + "owner":"${USER}", + "states":{ + "1":{ + "failed":1, + "waiting":1 + } + }, + "host":"${HOST}", + "meta":{ + "another_metadata":"1", + "custom_metadata":"something_custom", + "description":"Stalls when the first task fails. Here we test out a multi-line description!", + "title":"Authentication test suite." + }, + "port":"${PORT}" } } __END__ diff --git a/tests/authentication/06-suite-override.t b/tests/authentication/06-suite-override.t index 79a0b2fa80..be3b7056f5 100644 --- a/tests/authentication/06-suite-override.t +++ b/tests/authentication/06-suite-override.t @@ -79,8 +79,8 @@ __END__ cylc scan --comms-timeout=5 -rb -n "${SUITE_NAME}" >'scan-r.out' 2>'/dev/null' cmp_ok 'scan-r.out' <<__END__ ${SUITE_NAME}|${USER}|${HOST}|port|${PORT} -${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Suite overrides global authentication settings. +${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:1 ${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 __END__ @@ -89,18 +89,20 @@ __END__ cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ { - "${SUITE_NAME}": { - "owner": "${USER}", - "host": "${HOST}", - "port": "${PORT}", - "title": "Authentication test suite.", - "description": "Stalls when the first task fails. Suite overrides global authentication settings.", - "states": { - "1": { - "failed": 1, - "waiting": 1 - } - } + "${SUITE_NAME}":{ + "owner":"${USER}", + "states":{ + "1":{ + "failed":1, + "waiting":1 + } + }, + "host":"${HOST}", + "meta":{ + "description":"Stalls when the first task fails. Suite overrides global authentication settings.", + "title":"Authentication test suite." + }, + "port":"${PORT}" } } __END__ diff --git a/tests/authentication/07-sha-hash.t b/tests/authentication/07-sha-hash.t index 54f14c1580..610acbf860 100644 --- a/tests/authentication/07-sha-hash.t +++ b/tests/authentication/07-sha-hash.t @@ -88,8 +88,10 @@ __END__ cylc scan --comms-timeout=5 -rb -n "${SUITE_NAME}" >'scan-r.out' 2>'/dev/null' cmp_ok 'scan-r.out' <<__END__ ${SUITE_NAME}|${USER}|${HOST}|port|${PORT} -${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! +${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. +${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 +${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom ${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:1 ${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 __END__ @@ -98,18 +100,22 @@ __END__ cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ { - "${SUITE_NAME}": { - "owner": "${USER}", - "host": "${HOST}", - "port": "${PORT}", - "title": "Authentication test suite.", - "description": "Stalls when the first task fails. Here we test out a multi-line description!", - "states": { - "1": { - "failed": 1, - "waiting": 1 - } - } + "${SUITE_NAME}":{ + "owner":"${USER}", + "states":{ + "1":{ + "failed":1, + "waiting":1 + } + }, + "host":"${HOST}", + "meta":{ + "another_metadata":"1", + "custom_metadata":"something_custom", + "description":"Stalls when the first task fails. Here we test out a multi-line description!", + "title":"Authentication test suite." + }, + "port":"${PORT}" } } __END__ From 3916091101d5ed9d32fcd30f201d4b5de6197691 Mon Sep 17 00:00:00 2001 From: Sadie Bartholomew Date: Fri, 5 Jan 2018 18:54:26 +0000 Subject: [PATCH 18/21] Add further cycle point to tests and optimise last changes --- bin/cylc-scan | 35 +++++++++++------------- tests/authentication/00-identity.t | 8 +++++- tests/authentication/01-description.t | 14 +++++++++- tests/authentication/02-state-totals.t | 9 ++++-- tests/authentication/03-full-read.t | 9 ++++-- tests/authentication/04-shutdown.t | 10 +++++-- tests/authentication/05-full-control.t | 9 ++++-- tests/authentication/06-suite-override.t | 9 ++++-- tests/authentication/07-sha-hash.t | 9 ++++-- tests/authentication/basic/suite.rc | 10 ++++++- tests/authentication/override/suite.rc | 10 ++++++- 11 files changed, 97 insertions(+), 35 deletions(-) diff --git a/bin/cylc-scan b/bin/cylc-scan index d9b1ce385b..8898c21760 100755 --- a/bin/cylc-scan +++ b/bin/cylc-scan @@ -244,24 +244,23 @@ def main(): if options.raw_format: print "%s|%s|%s|port|%s" % (name, owner, host, port) try: - items = suite_identity[KEY_META] + meta_items = suite_identity[KEY_META] except KeyError: # Compat:<=7.5.0 - items = { + meta_items = { "title": suite_identity.get(KEY_TITLE), "description": suite_identity.get(KEY_DESCRIPTION), "update time": suite_identity.get(KEY_UPDATE_TIME) } - for key in items: - if not items[key] == "" and items[key] is not None: + for key, value in meta_items.items(): + if value: cleaned = ' '.join([x.strip() for x - in str(items[key]).split('\n') if x]) + in str(value).split('\n') if x]) print "%s|%s|%s|%s|%s" % (name, owner, host, key, cleaned) totals = suite_identity.get(KEY_STATES) if totals is None: continue - point_state_lines = get_point_state_count_lines( - *totals, use_color=options.color) + point_state_lines = get_point_state_count_lines(*totals) for point, state_line in point_state_lines: key = KEY_STATES if point: @@ -281,20 +280,18 @@ def main(): "update time": suite_identity.get(KEY_UPDATE_TIME) } clean_meta_items = {} - for key in meta_items: - if not meta_items[key] == "" and meta_items[key] is not None: - if '\n' in meta_items[key]: - clean_meta_items.update({ - key: ' '.join([x.strip() for x in - str(meta_items[key]).split('\n') if - x])}) - else: - clean_meta_items.update({key: meta_items[key]}) - suite_inf.update({"meta": clean_meta_items}) + for key, value in meta_items.items(): + if value: + clean_meta_items.update({ + key: ' '.join([x.strip() for x in + str(value).split('\n') if x])}) + if len(clean_meta_items) != 0: + suite_inf.update({"meta": clean_meta_items}) totals = suite_identity.get(KEY_STATES) - point_inf = {} if totals is None: + json_dict[name] = suite_inf continue + point_inf = {} point_state_lines = get_point_state_count_lines(*totals) for point, state_line in point_state_lines: state_dict = {} @@ -307,7 +304,7 @@ def main(): except (TypeError, ValueError): state_dict.update({key: str(value)}) point_inf.update({point: state_dict}) - suite_inf.update({KEY_STATES: point_inf}) + suite_inf.update({KEY_STATES: point_inf}) json_dict[name] = suite_inf continue diff --git a/tests/authentication/00-identity.t b/tests/authentication/00-identity.t index f44965b169..2fcd26cded 100644 --- a/tests/authentication/00-identity.t +++ b/tests/authentication/00-identity.t @@ -95,7 +95,13 @@ __END__ cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" 'localhost' \ >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' << __END__ -{} +{ + "${SUITE_NAME}":{ + "owner":"${USER}", + "host":"localhost", + "port":${PORT} + } +} __END__ # "cylc show" should be denied. diff --git a/tests/authentication/01-description.t b/tests/authentication/01-description.t index 3734d4a347..5311776e63 100644 --- a/tests/authentication/01-description.t +++ b/tests/authentication/01-description.t @@ -95,7 +95,19 @@ __END__ # Check scan --json output. cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ -{} +{ + "${SUITE_NAME}":{ + "owner":"${USER}", + "host":"${HOST}", + "meta":{ + "another_metadata":"1", + "custom_metadata":"something_custom", + "description":"Stalls when the first task fails. Here we test out a multi-line description!", + "title":"Authentication test suite." + }, + "port":"${PORT}" + } +} __END__ # "cylc show" (suite info) OK. diff --git a/tests/authentication/02-state-totals.t b/tests/authentication/02-state-totals.t index 5435963dd4..b07ae3f1e5 100644 --- a/tests/authentication/02-state-totals.t +++ b/tests/authentication/02-state-totals.t @@ -60,8 +60,9 @@ ${SUITE_NAME} ${USER}@${HOST}:${PORT} custom_metadata: "something_custom" Task state totals: - failed:1 waiting:1 + failed:1 waiting:2 1 failed:1 waiting:1 + 2 waiting:1 __END__ # Check scan --describe output. @@ -91,8 +92,9 @@ ${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here ${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 ${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom -${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:1 +${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:2 ${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 +${SUITE_NAME}|${USER}|${HOST}|states:2|waiting:1 __END__ # Check scan --json output. @@ -105,6 +107,9 @@ cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ "1":{ "failed":1, "waiting":1 + }, + "2":{ + "waiting":1 } }, "host":"${HOST}", diff --git a/tests/authentication/03-full-read.t b/tests/authentication/03-full-read.t index caa950f1ce..5e052c1c63 100644 --- a/tests/authentication/03-full-read.t +++ b/tests/authentication/03-full-read.t @@ -60,8 +60,9 @@ ${SUITE_NAME} ${USER}@${HOST}:${PORT} custom_metadata: "something_custom" Task state totals: - failed:1 waiting:1 + failed:1 waiting:2 1 failed:1 waiting:1 + 2 waiting:1 __END__ # Check scan --describe output. @@ -91,8 +92,9 @@ ${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here ${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 ${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom -${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:1 +${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:2 ${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 +${SUITE_NAME}|${USER}|${HOST}|states:2|waiting:1 __END__ # Check scan --json output. @@ -105,6 +107,9 @@ cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ "1":{ "failed":1, "waiting":1 + }, + "2":{ + "waiting":1 } }, "host":"${HOST}", diff --git a/tests/authentication/04-shutdown.t b/tests/authentication/04-shutdown.t index 3e3d947b62..8ea2ea6211 100644 --- a/tests/authentication/04-shutdown.t +++ b/tests/authentication/04-shutdown.t @@ -60,8 +60,9 @@ ${SUITE_NAME} ${USER}@${HOST}:${PORT} custom_metadata: "something_custom" Task state totals: - failed:1 waiting:1 + failed:1 waiting:2 1 failed:1 waiting:1 + 2 waiting:1 __END__ cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}" >'scan-d.out' 2>'/dev/null' @@ -90,8 +91,9 @@ ${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here ${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 ${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom -${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:1 +${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:2 ${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 +${SUITE_NAME}|${USER}|${HOST}|states:2|waiting:1 __END__ # Check scan --json output. @@ -104,6 +106,9 @@ cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ "1":{ "failed":1, "waiting":1 + }, + "2":{ + "waiting":1 } }, "host":"${HOST}", @@ -141,3 +146,4 @@ TEST_NAME="${TEST_NAME_BASE}-stop" run_ok "${TEST_NAME}" cylc stop --debug --max-polls=20 --interval=1 "${SUITE_NAME}" purge_suite "${SUITE_NAME}" exit + \ No newline at end of file diff --git a/tests/authentication/05-full-control.t b/tests/authentication/05-full-control.t index 64726780fc..5001320556 100644 --- a/tests/authentication/05-full-control.t +++ b/tests/authentication/05-full-control.t @@ -58,8 +58,9 @@ ${SUITE_NAME} ${USER}@${HOST}:${PORT} custom_metadata: "something_custom" Task state totals: - failed:1 waiting:1 + failed:1 waiting:2 1 failed:1 waiting:1 + 2 waiting:1 __END__ # Check scan --describe output. @@ -89,8 +90,9 @@ ${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here ${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 ${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom -${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:1 +${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:2 ${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 +${SUITE_NAME}|${USER}|${HOST}|states:2|waiting:1 __END__ # Check scan --json output. @@ -103,6 +105,9 @@ cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ "1":{ "failed":1, "waiting":1 + }, + "2":{ + "waiting":1 } }, "host":"${HOST}", diff --git a/tests/authentication/06-suite-override.t b/tests/authentication/06-suite-override.t index be3b7056f5..e3898f861c 100644 --- a/tests/authentication/06-suite-override.t +++ b/tests/authentication/06-suite-override.t @@ -57,8 +57,9 @@ ${SUITE_NAME} ${USER}@${HOST}:${PORT} "Stalls when the first task fails. Suite overrides global authentication settings." Task state totals: - failed:1 waiting:1 + failed:1 waiting:2 1 failed:1 waiting:1 + 2 waiting:1 __END__ cylc scan --comms-timeout=5 -db -n "${SUITE_NAME}" >'scan-d.out' 2>'/dev/null' @@ -81,8 +82,9 @@ cmp_ok 'scan-r.out' <<__END__ ${SUITE_NAME}|${USER}|${HOST}|port|${PORT} ${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Suite overrides global authentication settings. ${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. -${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:1 +${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:2 ${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 +${SUITE_NAME}|${USER}|${HOST}|states:2|waiting:1 __END__ # Check scan --json output. @@ -95,6 +97,9 @@ cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ "1":{ "failed":1, "waiting":1 + }, + "2":{ + "waiting":1 } }, "host":"${HOST}", diff --git a/tests/authentication/07-sha-hash.t b/tests/authentication/07-sha-hash.t index 610acbf860..6f2bb6a617 100644 --- a/tests/authentication/07-sha-hash.t +++ b/tests/authentication/07-sha-hash.t @@ -61,8 +61,9 @@ ${SUITE_NAME} ${USER}@${HOST}:${PORT} custom_metadata: "something_custom" Task state totals: - failed:1 waiting:1 + failed:1 waiting:2 1 failed:1 waiting:1 + 2 waiting:1 __END__ # Check scan --describe output. @@ -92,8 +93,9 @@ ${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here ${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 ${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom -${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:1 +${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:2 ${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 +${SUITE_NAME}|${USER}|${HOST}|states:2|waiting:1 __END__ # Check scan --json output. @@ -106,6 +108,9 @@ cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ "1":{ "failed":1, "waiting":1 + }, + "2":{ + "waiting":1 } }, "host":"${HOST}", diff --git a/tests/authentication/basic/suite.rc b/tests/authentication/basic/suite.rc index 07b4603012..4e201b15f6 100644 --- a/tests/authentication/basic/suite.rc +++ b/tests/authentication/basic/suite.rc @@ -9,10 +9,18 @@ timeout = PT30S abort on timeout = True [scheduling] + cycling mode = integer + initial cycle point = 1 + final cycle point = 2 [[dependencies]] - graph = foo => bar + [[[R/1]]] + graph = foo => bar + [[[R/2]]] + graph = bar[-P1] => pub [runtime] [[foo]] script = /bin/false [[bar]] script = /bin/true + [[pub]] + script = /bin/true diff --git a/tests/authentication/override/suite.rc b/tests/authentication/override/suite.rc index b853aff283..c16c726728 100644 --- a/tests/authentication/override/suite.rc +++ b/tests/authentication/override/suite.rc @@ -9,10 +9,18 @@ Suite overrides global authentication settings.""" [[authentication]] public = shutdown [scheduling] + cycling mode = integer + initial cycle point = 1 + final cycle point = 2 [[dependencies]] - graph = foo => bar + [[[R/1]]] + graph = foo => bar + [[[R/2]]] + graph = bar[-P1] => pub [runtime] [[foo]] script = /bin/false [[bar]] script = /bin/true + [[pub]] + script = /bin/true From d35e296e709414615e767cfb4b20a4f3d271083e Mon Sep 17 00:00:00 2001 From: Sadie Bartholomew Date: Mon, 8 Jan 2018 18:52:03 +0000 Subject: [PATCH 19/21] Refactoring format options --- bin/cylc-scan | 58 +++++++++----------------- tests/authentication/01-description.t | 4 +- tests/authentication/02-state-totals.t | 4 +- tests/authentication/03-full-read.t | 4 +- tests/authentication/04-shutdown.t | 4 +- tests/authentication/05-full-control.t | 4 +- tests/authentication/07-sha-hash.t | 4 +- 7 files changed, 31 insertions(+), 51 deletions(-) diff --git a/bin/cylc-scan b/bin/cylc-scan index 8898c21760..fb83d19db2 100755 --- a/bin/cylc-scan +++ b/bin/cylc-scan @@ -241,27 +241,33 @@ def main(): print name, owner, host, port continue - if options.raw_format: - print "%s|%s|%s|port|%s" % (name, owner, host, port) + if options.raw_format or options.json_format or options.describe: + # Extracting required data for these options before processing try: meta_items = suite_identity[KEY_META] except KeyError: # Compat:<=7.5.0 meta_items = { "title": suite_identity.get(KEY_TITLE), - "description": suite_identity.get(KEY_DESCRIPTION), - "update time": suite_identity.get(KEY_UPDATE_TIME) + "description": suite_identity.get(KEY_DESCRIPTION) } - for key, value in meta_items.items(): - if value: - cleaned = ' '.join([x.strip() for x - in str(value).split('\n') if x]) - print "%s|%s|%s|%s|%s" % (name, owner, host, key, cleaned) - totals = suite_identity.get(KEY_STATES) + if not options.describe: + suite_identity.get(KEY_UPDATE_TIME) + clean_meta_items = {} + for key, value in meta_items.items(): + if value: + clean_meta_items.update({ + key: ' '.join([x.strip() for x in + str(value).split('\n') if x])}) + totals = suite_identity.get(KEY_STATES) + + if options.raw_format: + print "%s|%s|%s|port|%s" % (name, owner, host, port) + for key, value in clean_meta_items.items(): + print "%s|%s|%s|%s|%s" % (name, owner, host, key, value) if totals is None: continue - point_state_lines = get_point_state_count_lines(*totals) - for point, state_line in point_state_lines: + for point, state_line in get_point_state_count_lines(*totals): key = KEY_STATES if point: key = "%s:%s" % (KEY_STATES, point) @@ -270,30 +276,13 @@ def main(): if options.json_format: suite_inf = {"owner": owner, "host": host, "port": port} - try: - meta_items = suite_identity[KEY_META] - except KeyError: - # Compat:<=7.5.0 - meta_items = { - "title": suite_identity.get(KEY_TITLE), - "description": suite_identity.get(KEY_DESCRIPTION), - "update time": suite_identity.get(KEY_UPDATE_TIME) - } - clean_meta_items = {} - for key, value in meta_items.items(): - if value: - clean_meta_items.update({ - key: ' '.join([x.strip() for x in - str(value).split('\n') if x])}) if len(clean_meta_items) != 0: suite_inf.update({"meta": clean_meta_items}) - totals = suite_identity.get(KEY_STATES) if totals is None: json_dict[name] = suite_inf continue point_inf = {} - point_state_lines = get_point_state_count_lines(*totals) - for point, state_line in point_state_lines: + for point, state_line in get_point_state_count_lines(*totals): state_dict = {} if point: for state_inf in state_line.split(' '): @@ -321,15 +310,6 @@ def main(): print line if options.describe: - try: - meta_items = suite_identity[KEY_META] - except KeyError: - # Compat:<=7.5.0 - meta_items = { - "title": suite_identity.get(KEY_TITLE), - "description": suite_identity.get(KEY_DESCRIPTION) - } - # Deal with title first for case of just withheld statement if meta_items["title"] is None: print indent + bold("(description and state totals " diff --git a/tests/authentication/01-description.t b/tests/authentication/01-description.t index 5311776e63..66a8e2e4b2 100644 --- a/tests/authentication/01-description.t +++ b/tests/authentication/01-description.t @@ -85,11 +85,11 @@ __END__ cylc scan --comms-timeout=5 -rb -n "${SUITE_NAME}" >'scan-r.out' 2>'/dev/null' cmp_ok 'scan-r.out' <<__END__ ${SUITE_NAME}|${USER}|${HOST}|port|${PORT} +${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 +${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom ${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. \ Here we test out a multi-line description! ${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. -${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 -${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom __END__ # Check scan --json output. diff --git a/tests/authentication/02-state-totals.t b/tests/authentication/02-state-totals.t index b07ae3f1e5..69df265485 100644 --- a/tests/authentication/02-state-totals.t +++ b/tests/authentication/02-state-totals.t @@ -88,10 +88,10 @@ __END__ cylc scan --comms-timeout=5 -rb -n "${SUITE_NAME}" >'scan-r.out' 2>'/dev/null' cmp_ok 'scan-r.out' <<__END__ ${SUITE_NAME}|${USER}|${HOST}|port|${PORT} -${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! -${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 ${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom +${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! +${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:2 ${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 ${SUITE_NAME}|${USER}|${HOST}|states:2|waiting:1 diff --git a/tests/authentication/03-full-read.t b/tests/authentication/03-full-read.t index 5e052c1c63..ddc8fae09d 100644 --- a/tests/authentication/03-full-read.t +++ b/tests/authentication/03-full-read.t @@ -88,10 +88,10 @@ __END__ cylc scan --comms-timeout=5 -rb -n "${SUITE_NAME}" >'scan-r.out' 2>'/dev/null' cmp_ok 'scan-r.out' <<__END__ ${SUITE_NAME}|${USER}|${HOST}|port|${PORT} -${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! -${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 ${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom +${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! +${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:2 ${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 ${SUITE_NAME}|${USER}|${HOST}|states:2|waiting:1 diff --git a/tests/authentication/04-shutdown.t b/tests/authentication/04-shutdown.t index 8ea2ea6211..786efaaab1 100644 --- a/tests/authentication/04-shutdown.t +++ b/tests/authentication/04-shutdown.t @@ -87,10 +87,10 @@ __END__ cylc scan --comms-timeout=5 -rb -n "${SUITE_NAME}" >'scan-r.out' 2>'/dev/null' cmp_ok 'scan-r.out' <<__END__ ${SUITE_NAME}|${USER}|${HOST}|port|${PORT} -${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! -${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 ${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom +${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! +${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:2 ${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 ${SUITE_NAME}|${USER}|${HOST}|states:2|waiting:1 diff --git a/tests/authentication/05-full-control.t b/tests/authentication/05-full-control.t index 5001320556..c4dd2e799d 100644 --- a/tests/authentication/05-full-control.t +++ b/tests/authentication/05-full-control.t @@ -86,10 +86,10 @@ __END__ cylc scan --comms-timeout=5 -rb -n "${SUITE_NAME}" >'scan-r.out' 2>'/dev/null' cmp_ok 'scan-r.out' <<__END__ ${SUITE_NAME}|${USER}|${HOST}|port|${PORT} -${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! -${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 ${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom +${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! +${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:2 ${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 ${SUITE_NAME}|${USER}|${HOST}|states:2|waiting:1 diff --git a/tests/authentication/07-sha-hash.t b/tests/authentication/07-sha-hash.t index 6f2bb6a617..a5e07895af 100644 --- a/tests/authentication/07-sha-hash.t +++ b/tests/authentication/07-sha-hash.t @@ -89,10 +89,10 @@ __END__ cylc scan --comms-timeout=5 -rb -n "${SUITE_NAME}" >'scan-r.out' 2>'/dev/null' cmp_ok 'scan-r.out' <<__END__ ${SUITE_NAME}|${USER}|${HOST}|port|${PORT} -${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! -${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 ${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom +${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! +${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. ${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:2 ${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 ${SUITE_NAME}|${USER}|${HOST}|states:2|waiting:1 From 681c7211d928c7900a540f10ad26828a0a210104 Mon Sep 17 00:00:00 2001 From: Sadie Bartholomew Date: Fri, 26 Jan 2018 13:02:32 +0000 Subject: [PATCH 20/21] Scan revamp to 'raw' JSON, amend tests bar update-time --- bin/cylc-scan | 50 +++++---------- tests/authentication/00-identity.t | 17 +++--- tests/authentication/01-description.t | 34 +++++++---- tests/authentication/02-state-totals.t | 78 +++++++++++++++++------- tests/authentication/03-full-read.t | 78 +++++++++++++++++------- tests/authentication/04-shutdown.t | 78 +++++++++++++++++------- tests/authentication/05-full-control.t | 78 +++++++++++++++++------- tests/authentication/06-suite-override.t | 74 ++++++++++++++++------ tests/authentication/07-sha-hash.t | 78 +++++++++++++++++------- tests/lib/bash/test_header | 21 ++++++- 10 files changed, 404 insertions(+), 182 deletions(-) diff --git a/bin/cylc-scan b/bin/cylc-scan index fb83d19db2..0cbba9c254 100755 --- a/bin/cylc-scan +++ b/bin/cylc-scan @@ -226,7 +226,7 @@ def main(): if not args: return - json_dict = {} + json_filter = [] skip_one = True for host, port, suite_identity in scan_many(args, options.comms_timeout): name = suite_identity[KEY_NAME] @@ -236,12 +236,15 @@ def main(): continue if not cres['suite-owner'].match(owner): continue + if options.json_format: + json_filter.append([host, port, suite_identity]) + continue if options.old_format: print name, owner, host, port continue - if options.raw_format or options.json_format or options.describe: + if options.raw_format or options.describe: # Extracting required data for these options before processing try: meta_items = suite_identity[KEY_META] @@ -251,17 +254,16 @@ def main(): "title": suite_identity.get(KEY_TITLE), "description": suite_identity.get(KEY_DESCRIPTION) } - if not options.describe: - suite_identity.get(KEY_UPDATE_TIME) - clean_meta_items = {} - for key, value in meta_items.items(): - if value: - clean_meta_items.update({ - key: ' '.join([x.strip() for x in - str(value).split('\n') if x])}) - totals = suite_identity.get(KEY_STATES) if options.raw_format: + suite_identity.get(KEY_UPDATE_TIME) + clean_meta_items = {} + for key, value in meta_items.items(): + if value: + clean_meta_items.update({ + key: ' '.join([x.strip() for x in + str(value).split('\n') if x])}) + totals = suite_identity.get(KEY_STATES) print "%s|%s|%s|port|%s" % (name, owner, host, port) for key, value in clean_meta_items.items(): print "%s|%s|%s|%s|%s" % (name, owner, host, key, value) @@ -274,29 +276,6 @@ def main(): print "%s|%s|%s|%s|%s" % (name, owner, host, key, state_line) continue - if options.json_format: - suite_inf = {"owner": owner, "host": host, "port": port} - if len(clean_meta_items) != 0: - suite_inf.update({"meta": clean_meta_items}) - if totals is None: - json_dict[name] = suite_inf - continue - point_inf = {} - for point, state_line in get_point_state_count_lines(*totals): - state_dict = {} - if point: - for state_inf in state_line.split(' '): - if ':' in state_inf: - key, value = state_inf.split(':') - try: - state_dict.update({key: int(value)}) - except (TypeError, ValueError): - state_dict.update({key: str(value)}) - point_inf.update({point: state_dict}) - suite_inf.update({KEY_STATES: point_inf}) - json_dict[name] = suite_inf - continue - line = '%s %s@%s:%s' % (name, owner, host, port) if options.describe or options.print_totals: if skip_one: @@ -354,9 +333,8 @@ def main(): point_prefix = "%s " % point print indent * 2 + "%s%s" % (point_prefix, state_line) - # Need a single filtered JSON structure => only dump after final iteration if options.json_format: - print json.dumps(json_dict, indent=4) + print json.dumps(json_filter, indent=4) def get_point_state_count_lines(state_count_totals, state_count_cycles, diff --git a/tests/authentication/00-identity.t b/tests/authentication/00-identity.t index 2fcd26cded..257df9e8a4 100644 --- a/tests/authentication/00-identity.t +++ b/tests/authentication/00-identity.t @@ -95,13 +95,16 @@ __END__ cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" 'localhost' \ >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' << __END__ -{ - "${SUITE_NAME}":{ - "owner":"${USER}", - "host":"localhost", - "port":${PORT} - } -} +[ + [ + "localhost", + ${PORT}, + { + "owner":"${USER}", + "name":"${SUITE_NAME}" + } + ] +] __END__ # "cylc show" should be denied. diff --git a/tests/authentication/01-description.t b/tests/authentication/01-description.t index 66a8e2e4b2..a0a9e3dc5d 100644 --- a/tests/authentication/01-description.t +++ b/tests/authentication/01-description.t @@ -95,19 +95,27 @@ __END__ # Check scan --json output. cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ -{ - "${SUITE_NAME}":{ - "owner":"${USER}", - "host":"${HOST}", - "meta":{ - "another_metadata":"1", - "custom_metadata":"something_custom", - "description":"Stalls when the first task fails. Here we test out a multi-line description!", - "title":"Authentication test suite." - }, - "port":"${PORT}" - } -} +[ + [ + "${HOST}", + "${PORT}", + { + "group":"", + "description":"Stalls when the first task fails.\n Here we test out a multi-line description!", + "title":"Authentication test suite.", + "meta":{ + "group":"", + "description":"Stalls when the first task fails.\n Here we test out a multi-line description!", + "title":"Authentication test suite.", + "URL":"", + "another_metadata":"1", + "custom_metadata":"something_custom" + }, + "owner":"${USER}", + "name":"${SUITE_NAME}" + } + ] +] __END__ # "cylc show" (suite info) OK. diff --git a/tests/authentication/02-state-totals.t b/tests/authentication/02-state-totals.t index 69df265485..b4a354a44d 100644 --- a/tests/authentication/02-state-totals.t +++ b/tests/authentication/02-state-totals.t @@ -100,28 +100,64 @@ __END__ # Check scan --json output. cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ -{ - "${SUITE_NAME}":{ - "owner":"${USER}", - "states":{ - "1":{ - "failed":1, - "waiting":1 +[ + [ + "${HOST}", + "${PORT}", + { + "group":"", + "description":"Stalls when the first task fails.\n Here we test out a multi-line description!", + "title":"Authentication test suite.", + "states":[ + { + "failed":1, + "waiting":2 + }, + { + "1":{ + "failed":1, + "waiting":1 + }, + "2":{ + "waiting":1 + } + } + ], + "tasks-by-state":{ + "failed":[ + [ + 1516968607.0, + "foo", + "1" + ] + ], + "waiting":[ + [ + 0, + "pub", + "2" + ], + [ + 0, + "bar", + "1" + ] + ] }, - "2":{ - "waiting":1 - } - }, - "host":"${HOST}", - "meta":{ - "another_metadata":"1", - "custom_metadata":"something_custom", - "description":"Stalls when the first task fails. Here we test out a multi-line description!", - "title":"Authentication test suite." - }, - "port":"${PORT}" - } -} + "meta":{ + "group":"", + "description":"Stalls when the first task fails.\n Here we test out a multi-line description!", + "title":"Authentication test suite.", + "URL":"", + "another_metadata":"1", + "custom_metadata":"something_custom" + }, + "owner":"${USER}", + "update-time":1516968608.1505351, + "name":"${SUITE_NAME}" + } + ] +] __END__ # "cylc show" (suite info) OK. diff --git a/tests/authentication/03-full-read.t b/tests/authentication/03-full-read.t index ddc8fae09d..9ffc740d31 100644 --- a/tests/authentication/03-full-read.t +++ b/tests/authentication/03-full-read.t @@ -100,28 +100,64 @@ __END__ # Check scan --json output. cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ -{ - "${SUITE_NAME}":{ - "owner":"${USER}", - "states":{ - "1":{ - "failed":1, - "waiting":1 +[ + [ + "${HOST}", + "${PORT}", + { + "group":"", + "description":"Stalls when the first task fails.\n Here we test out a multi-line description!", + "title":"Authentication test suite.", + "states":[ + { + "failed":1, + "waiting":2 + }, + { + "1":{ + "failed":1, + "waiting":1 + }, + "2":{ + "waiting":1 + } + } + ], + "tasks-by-state":{ + "failed":[ + [ + 1516968607.0, + "foo", + "1" + ] + ], + "waiting":[ + [ + 0, + "pub", + "2" + ], + [ + 0, + "bar", + "1" + ] + ] }, - "2":{ - "waiting":1 - } - }, - "host":"${HOST}", - "meta":{ - "another_metadata":"1", - "custom_metadata":"something_custom", - "description":"Stalls when the first task fails. Here we test out a multi-line description!", - "title":"Authentication test suite." - }, - "port":"${PORT}" - } -} + "meta":{ + "group":"", + "description":"Stalls when the first task fails.\n Here we test out a multi-line description!", + "title":"Authentication test suite.", + "URL":"", + "another_metadata":"1", + "custom_metadata":"something_custom" + }, + "owner":"${USER}", + "update-time":1516968608.1505351, + "name":"${SUITE_NAME}" + } + ] +] __END__ # "cylc show" (suite info) OK. diff --git a/tests/authentication/04-shutdown.t b/tests/authentication/04-shutdown.t index 786efaaab1..4b642c4166 100644 --- a/tests/authentication/04-shutdown.t +++ b/tests/authentication/04-shutdown.t @@ -99,28 +99,64 @@ __END__ # Check scan --json output. cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ -{ - "${SUITE_NAME}":{ - "owner":"${USER}", - "states":{ - "1":{ - "failed":1, - "waiting":1 +[ + [ + "${HOST}", + "${PORT}", + { + "group":"", + "description":"Stalls when the first task fails.\n Here we test out a multi-line description!", + "title":"Authentication test suite.", + "states":[ + { + "failed":1, + "waiting":2 + }, + { + "1":{ + "failed":1, + "waiting":1 + }, + "2":{ + "waiting":1 + } + } + ], + "tasks-by-state":{ + "failed":[ + [ + 1516968607.0, + "foo", + "1" + ] + ], + "waiting":[ + [ + 0, + "pub", + "2" + ], + [ + 0, + "bar", + "1" + ] + ] }, - "2":{ - "waiting":1 - } - }, - "host":"${HOST}", - "meta":{ - "another_metadata":"1", - "custom_metadata":"something_custom", - "description":"Stalls when the first task fails. Here we test out a multi-line description!", - "title":"Authentication test suite." - }, - "port":"${PORT}" - } -} + "meta":{ + "group":"", + "description":"Stalls when the first task fails.\n Here we test out a multi-line description!", + "title":"Authentication test suite.", + "URL":"", + "another_metadata":"1", + "custom_metadata":"something_custom" + }, + "owner":"${USER}", + "update-time":1516968608.1505351, + "name":"${SUITE_NAME}" + } + ] +] __END__ # "cylc show" (suite info) OK. diff --git a/tests/authentication/05-full-control.t b/tests/authentication/05-full-control.t index c4dd2e799d..0486b57440 100644 --- a/tests/authentication/05-full-control.t +++ b/tests/authentication/05-full-control.t @@ -98,28 +98,64 @@ __END__ # Check scan --json output. cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ -{ - "${SUITE_NAME}":{ - "owner":"${USER}", - "states":{ - "1":{ - "failed":1, - "waiting":1 +[ + [ + "${HOST}", + "${PORT}", + { + "group":"", + "description":"Stalls when the first task fails.\n Here we test out a multi-line description!", + "title":"Authentication test suite.", + "states":[ + { + "failed":1, + "waiting":2 + }, + { + "1":{ + "failed":1, + "waiting":1 + }, + "2":{ + "waiting":1 + } + } + ], + "tasks-by-state":{ + "failed":[ + [ + 1516968607.0, + "foo", + "1" + ] + ], + "waiting":[ + [ + 0, + "pub", + "2" + ], + [ + 0, + "bar", + "1" + ] + ] }, - "2":{ - "waiting":1 - } - }, - "host":"${HOST}", - "meta":{ - "another_metadata":"1", - "custom_metadata":"something_custom", - "description":"Stalls when the first task fails. Here we test out a multi-line description!", - "title":"Authentication test suite." - }, - "port":"${PORT}" - } -} + "meta":{ + "group":"", + "description":"Stalls when the first task fails.\n Here we test out a multi-line description!", + "title":"Authentication test suite.", + "URL":"", + "another_metadata":"1", + "custom_metadata":"something_custom" + }, + "owner":"${USER}", + "update-time":1516968608.1505351, + "name":"${SUITE_NAME}" + } + ] +] __END__ # "cylc show" (suite info) OK. diff --git a/tests/authentication/06-suite-override.t b/tests/authentication/06-suite-override.t index e3898f861c..270f9b2d8d 100644 --- a/tests/authentication/06-suite-override.t +++ b/tests/authentication/06-suite-override.t @@ -90,26 +90,62 @@ __END__ # Check scan --json output. cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ -{ - "${SUITE_NAME}":{ - "owner":"${USER}", - "states":{ - "1":{ - "failed":1, - "waiting":1 +[ + [ + "${HOST}", + "${PORT}", + { + "group":"", + "description":"Stalls when the first task fails.\nSuite overrides global authentication settings.", + "title":"Authentication test suite.", + "states":[ + { + "failed":1, + "waiting":2 + }, + { + "1":{ + "failed":1, + "waiting":1 + }, + "2":{ + "waiting":1 + } + } + ], + "tasks-by-state":{ + "failed":[ + [ + 1516968607.0, + "foo", + "1" + ] + ], + "waiting":[ + [ + 0, + "pub", + "2" + ], + [ + 0, + "bar", + "1" + ] + ] }, - "2":{ - "waiting":1 - } - }, - "host":"${HOST}", - "meta":{ - "description":"Stalls when the first task fails. Suite overrides global authentication settings.", - "title":"Authentication test suite." - }, - "port":"${PORT}" - } -} + "meta":{ + "URL":"", + "group":"", + "description":"Stalls when the first task fails.\nSuite overrides global authentication settings.", + "title":"Authentication test suite." + }, + "owner":"${USER}", + "update-time":1516968608.1505351, + "name":"${SUITE_NAME}" + } + ] +] __END__ # "cylc show" (suite info) OK. diff --git a/tests/authentication/07-sha-hash.t b/tests/authentication/07-sha-hash.t index a5e07895af..07c5094acd 100644 --- a/tests/authentication/07-sha-hash.t +++ b/tests/authentication/07-sha-hash.t @@ -101,28 +101,64 @@ __END__ # Check scan --json output. cylc scan --comms-timeout=5 -jb -n "${SUITE_NAME}" >'scan-j.out' 2>'/dev/null' cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ -{ - "${SUITE_NAME}":{ - "owner":"${USER}", - "states":{ - "1":{ - "failed":1, - "waiting":1 +[ + [ + "${HOST}", + "${PORT}", + { + "group":"", + "description":"Stalls when the first task fails.\n Here we test out a multi-line description!", + "title":"Authentication test suite.", + "states":[ + { + "failed":1, + "waiting":2 + }, + { + "1":{ + "failed":1, + "waiting":1 + }, + "2":{ + "waiting":1 + } + } + ], + "tasks-by-state":{ + "failed":[ + [ + 1516968607.0, + "foo", + "1" + ] + ], + "waiting":[ + [ + 0, + "pub", + "2" + ], + [ + 0, + "bar", + "1" + ] + ] }, - "2":{ - "waiting":1 - } - }, - "host":"${HOST}", - "meta":{ - "another_metadata":"1", - "custom_metadata":"something_custom", - "description":"Stalls when the first task fails. Here we test out a multi-line description!", - "title":"Authentication test suite." - }, - "port":"${PORT}" - } -} + "meta":{ + "group":"", + "description":"Stalls when the first task fails.\n Here we test out a multi-line description!", + "title":"Authentication test suite.", + "URL":"", + "another_metadata":"1", + "custom_metadata":"something_custom" + }, + "owner":"${USER}", + "update-time":1516968608.1505351, + "name":"${SUITE_NAME}" + } + ] +] __END__ # "cylc show" (suite info) OK. diff --git a/tests/lib/bash/test_header b/tests/lib/bash/test_header index f6a8f35c25..8e61eedc0a 100644 --- a/tests/lib/bash/test_header +++ b/tests/lib/bash/test_header @@ -220,16 +220,33 @@ cmp_json_ok() { run_ok "$1" python -c "import sys import json -output = json.load(open(sys.argv[1], 'r')) +# no need for try/except as get traceback anyway in stderr if sys.argv[2]: expected_output = json.load(open(sys.argv[2], 'r')) else: expected_output = json.load(sys.stdin) +output = json.load(open(sys.argv[1], 'r')) +# convert unicode for useful stderr - '^?*?^' chosen as arbitrary placeholder +def process(some_output): + if isinstance(some_output, unicode): + return str(some_output.encode('utf-8')).replace('\'', '^?*?^') + if isinstance(some_output, list): + return [process(item) for item in some_output] + if isinstance(some_output, dict): + return dict((process(key), process(value)) for key, value in + some_output.items()) + return some_output + +# check for match; if different process for convenient stderr lifting -> tests if output == expected_output: sys.exit(0) else: - sys.exit('%s not equal to %s' % (output, expected_output))" "$2" "${3:-}" + # Python's return always includes single quotes; JSON needs double + print_string = '%s\n not equal to \n%s' % ( + str(process(output)).replace('\'', '\"'), + str(process(expected_output)).replace('\'', '\"')) + sys.exit(print_string.replace('^?*?^', '\''))" "$2" "${3:-}" } contains_ok() { From 5cb878b37586c1174713bf531be4dc4b13916a36 Mon Sep 17 00:00:00 2001 From: Sadie Bartholomew Date: Fri, 26 Jan 2018 19:13:14 +0000 Subject: [PATCH 21/21] Tailor tests to account for arbitary floats with regex --- tests/authentication/02-state-totals.t | 4 +-- tests/authentication/03-full-read.t | 4 +-- tests/authentication/04-shutdown.t | 4 +-- tests/authentication/05-full-control.t | 4 +-- tests/authentication/06-suite-override.t | 14 ++++---- tests/authentication/07-sha-hash.t | 4 +-- tests/lib/bash/test_header | 46 +++++++++++++----------- 7 files changed, 43 insertions(+), 37 deletions(-) diff --git a/tests/authentication/02-state-totals.t b/tests/authentication/02-state-totals.t index b4a354a44d..74eb2a2899 100644 --- a/tests/authentication/02-state-totals.t +++ b/tests/authentication/02-state-totals.t @@ -126,7 +126,7 @@ cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ "tasks-by-state":{ "failed":[ [ - 1516968607.0, + "", "foo", "1" ] @@ -153,7 +153,7 @@ cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ "custom_metadata":"something_custom" }, "owner":"${USER}", - "update-time":1516968608.1505351, + "update-time":"", "name":"${SUITE_NAME}" } ] diff --git a/tests/authentication/03-full-read.t b/tests/authentication/03-full-read.t index 9ffc740d31..073f305f74 100644 --- a/tests/authentication/03-full-read.t +++ b/tests/authentication/03-full-read.t @@ -126,7 +126,7 @@ cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ "tasks-by-state":{ "failed":[ [ - 1516968607.0, + "", "foo", "1" ] @@ -153,7 +153,7 @@ cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ "custom_metadata":"something_custom" }, "owner":"${USER}", - "update-time":1516968608.1505351, + "update-time":"", "name":"${SUITE_NAME}" } ] diff --git a/tests/authentication/04-shutdown.t b/tests/authentication/04-shutdown.t index 4b642c4166..409896d39b 100644 --- a/tests/authentication/04-shutdown.t +++ b/tests/authentication/04-shutdown.t @@ -125,7 +125,7 @@ cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ "tasks-by-state":{ "failed":[ [ - 1516968607.0, + "", "foo", "1" ] @@ -152,7 +152,7 @@ cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ "custom_metadata":"something_custom" }, "owner":"${USER}", - "update-time":1516968608.1505351, + "update-time":"", "name":"${SUITE_NAME}" } ] diff --git a/tests/authentication/05-full-control.t b/tests/authentication/05-full-control.t index 0486b57440..82d565d644 100644 --- a/tests/authentication/05-full-control.t +++ b/tests/authentication/05-full-control.t @@ -124,7 +124,7 @@ cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ "tasks-by-state":{ "failed":[ [ - 1516968607.0, + "", "foo", "1" ] @@ -151,7 +151,7 @@ cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ "custom_metadata":"something_custom" }, "owner":"${USER}", - "update-time":1516968608.1505351, + "update-time":"", "name":"${SUITE_NAME}" } ] diff --git a/tests/authentication/06-suite-override.t b/tests/authentication/06-suite-override.t index 270f9b2d8d..b564e6ba6b 100644 --- a/tests/authentication/06-suite-override.t +++ b/tests/authentication/06-suite-override.t @@ -116,7 +116,7 @@ cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ "tasks-by-state":{ "failed":[ [ - 1516968607.0, + "", "foo", "1" ] @@ -134,14 +134,14 @@ cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ ] ] }, - "meta":{ - "URL":"", - "group":"", - "description":"Stalls when the first task fails.\nSuite overrides global authentication settings.", - "title":"Authentication test suite." + "meta": { + "URL": "", + "group": "", + "description": "Stalls when the first task fails.\nSuite overrides global authentication settings.", + "title": "Authentication test suite." }, "owner":"${USER}", - "update-time":1516968608.1505351, + "update-time":"", "name":"${SUITE_NAME}" } ] diff --git a/tests/authentication/07-sha-hash.t b/tests/authentication/07-sha-hash.t index 07c5094acd..a4b7de991b 100644 --- a/tests/authentication/07-sha-hash.t +++ b/tests/authentication/07-sha-hash.t @@ -127,7 +127,7 @@ cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ "tasks-by-state":{ "failed":[ [ - 1516968607.0, + "", "foo", "1" ] @@ -154,7 +154,7 @@ cmp_json_ok 'scan-j.out' 'scan-j.out' <<__END__ "custom_metadata":"something_custom" }, "owner":"${USER}", - "update-time":1516968608.1505351, + "update-time":"", "name":"${SUITE_NAME}" } ] diff --git a/tests/lib/bash/test_header b/tests/lib/bash/test_header index 8e61eedc0a..37b90090f4 100644 --- a/tests/lib/bash/test_header +++ b/tests/lib/bash/test_header @@ -218,35 +218,41 @@ cmp_ok() { cmp_json_ok() { run_ok "$1" python -c "import sys +import re import json +from cylc.task_state import TASK_STATUSES_ORDERED -# no need for try/except as get traceback anyway in stderr +# Load data; no need for JSON object try/except as get stderr traceback anyway +output = json.load(open(sys.argv[1], 'r')) if sys.argv[2]: expected_output = json.load(open(sys.argv[2], 'r')) else: expected_output = json.load(sys.stdin) -output = json.load(open(sys.argv[1], 'r')) -# convert unicode for useful stderr - '^?*?^' chosen as arbitrary placeholder -def process(some_output): - if isinstance(some_output, unicode): - return str(some_output.encode('utf-8')).replace('\'', '^?*?^') - if isinstance(some_output, list): - return [process(item) for item in some_output] - if isinstance(some_output, dict): - return dict((process(key), process(value)) for key, value in - some_output.items()) - return some_output - -# check for match; if different process for convenient stderr lifting -> tests -if output == expected_output: +# Replace arbitary time floats in JSON structures for effective comparison +def float_repl(some_output, marker): + new_output = re.sub(r'((\'|\")update-time(\'|\")\s*:\s*)\d*\.\d*', + r'\1' + marker, str(some_output)) + # Deal with cylc scan TIME_FIELDS (see cylc.state_summary_mgr) + for status in TASK_STATUSES_ORDERED: + new_regex = (r'((\'|\")%s(\'|\")\s*:\s*\[\n*\s*\[\n*\s*)\d*\.\d*' % + status) + new_output = re.sub(new_regex, r'\1' + marker, str(new_output)) + return new_output + +# Unicode (u'...') for 'loaded' outputs; double-quoted strings for 'dumps' +u_marker = \"u''\" +s_marker = '\"\"' + +# Check for match; if different, process for easy stderr lifting -> tests +if float_repl(output, u_marker) == float_repl(expected_output, u_marker): sys.exit(0) else: - # Python's return always includes single quotes; JSON needs double - print_string = '%s\n not equal to \n%s' % ( - str(process(output)).replace('\'', '\"'), - str(process(expected_output)).replace('\'', '\"')) - sys.exit(print_string.replace('^?*?^', '\''))" "$2" "${3:-}" + sys.exit('LOADED:\n%s\n!=\n%s\n OR DUMPED:\n%s\n!=\n%s' % ( + float_repl(output, u_marker), float_repl(expected_output, u_marker), + float_repl(json.dumps(output, indent=4), s_marker), + float_repl(json.dumps(expected_output, indent=4), + s_marker)))" "$2" "${3:-}" } contains_ok() {