From bdfd289abf8b82ea78151f241ad6183a0015466b Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Tue, 29 Jan 2019 14:51:18 +1300 Subject: [PATCH 1/8] Support alt cylc-run location. Conflicts: bin/cylc-register lib/cylc/suite_srv_files_mgr.py --- bin/cylc-register | 12 +++++++++--- lib/cylc/suite_srv_files_mgr.py | 12 ++++++++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/bin/cylc-register b/bin/cylc-register index 976e4db9f3..06825db9c9 100755 --- a/bin/cylc-register +++ b/bin/cylc-register @@ -67,10 +67,14 @@ def main(): ("[PATH]", "Suite definition directory (defaults to $PWD)")]) parser.add_option( - "--redirect", help="Allow an existing suite name and run directory " - "to be used with another suite.", + "--redirect", help="Allow an existing suite name and run directory" + " to be used with another suite.", action="store_true", default=False, dest="redirect") + parser.add_option( + "--run-dir", help="Symlink $HOME/cylc-run/REG to RUNDIR/REG.", + action="store", metavar="RUNDIR", default=None, dest="rundir") + opts, args = parser.parse_args() if len(args) == 2: reg, src = args @@ -79,7 +83,9 @@ def main(): src = None else: reg = src = None - SuiteSrvFilesManager().register(reg, src, redirect=opts.redirect) + + SuiteSrvFilesManager().register( + reg, src, redirect=opts.redirect, rundir=opts.rundir) if __name__ == "__main__": diff --git a/lib/cylc/suite_srv_files_mgr.py b/lib/cylc/suite_srv_files_mgr.py index d08e4941d3..9b48922e46 100644 --- a/lib/cylc/suite_srv_files_mgr.py +++ b/lib/cylc/suite_srv_files_mgr.py @@ -429,7 +429,7 @@ def parse_suite_arg(self, options, arg): name = os.path.basename(os.path.dirname(arg)) return name, path - def register(self, reg=None, source=None, redirect=False): + def register(self, reg=None, source=None, redirect=False, rundir=None): """Register a suite, or renew its registration. Create suite service directory and symlink to suite source location. @@ -468,7 +468,15 @@ def register(self, reg=None, source=None, redirect=False): # Create service dir if necessary. srv_d = self.get_suite_srv_dir(reg) - mkdir_p(srv_d) + if rundir is None: + os.makedirs(srv_d, exist_ok=True) + else: + suite_run_d, srv_d_name = os.path.split(srv_d) + alt_suite_run_d = os.path.join(rundir, reg) + alt_srv_d = os.path.join(rundir, reg, srv_d_name) + os.makedirs(alt_srv_d, exist_ok=True) + os.makedirs(os.path.dirname(suite_run_d), exist_ok=True) + os.symlink(alt_suite_run_d, suite_run_d) # See if suite already has a source or not try: From 84006f033d5f8672bdbe93cbeb0a32e4027590e8 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Sun, 30 Jun 2019 02:07:50 +0100 Subject: [PATCH 2/8] Handle existing alt run dirs. --- lib/cylc/suite_srv_files_mgr.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/cylc/suite_srv_files_mgr.py b/lib/cylc/suite_srv_files_mgr.py index 9b48922e46..89ee09153c 100644 --- a/lib/cylc/suite_srv_files_mgr.py +++ b/lib/cylc/suite_srv_files_mgr.py @@ -476,8 +476,21 @@ def register(self, reg=None, source=None, redirect=False, rundir=None): alt_srv_d = os.path.join(rundir, reg, srv_d_name) os.makedirs(alt_srv_d, exist_ok=True) os.makedirs(os.path.dirname(suite_run_d), exist_ok=True) - os.symlink(alt_suite_run_d, suite_run_d) + if os.path.islink(suite_run_d) and not os.path.exists(suite_run_d): + # Remove a bad symlink. + os.unlink(suite_run_d) + if not os.path.exists(suite_run_d): + os.symlink(alt_suite_run_d, suite_run_d) + elif not os.path.islink(suite_run_d): + raise SuiteServiceFileError( + f"Run directory '{suite_run_d}' already exists.") + elif alt_suite_run_d != os.readlink(suite_run_d): + target = os.readlink(suite_run_d) + raise SuiteServiceFileError( + f"Symlink '{suite_run_d}' already points to {target}.") + # (else already the right symlink) + temp = None # See if suite already has a source or not try: orig_source = os.readlink( From cd9ad6739f81d0c2dc0afccb67120a28aa75ce42 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Sun, 30 Jun 2019 13:50:05 +0100 Subject: [PATCH 3/8] Add alt run dir tests. --- tests/registration/00-simple.t | 44 ++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/tests/registration/00-simple.t b/tests/registration/00-simple.t index 040a1d50e9..8ad5ee1d37 100755 --- a/tests/registration/00-simple.t +++ b/tests/registration/00-simple.t @@ -18,7 +18,7 @@ # Test suite registration . "$(dirname "$0")/test_header" -set_test_number 25 +set_test_number 34 init_suite "${TEST_NAME_BASE}" <<'__SUITE_RC__' [meta] @@ -32,7 +32,7 @@ init_suite "${TEST_NAME_BASE}" <<'__SUITE_RC__' __SUITE_RC__ # Unique suite run-dir prefix to avoid messing with real suites. -PRE=cylctb-reg-${CYLC_TEST_TIME_INIT} +PRE=cylctb-${CYLC_TEST_TIME_INIT} # Test fail no suite.rc file. CYLC_RUN_DIR=$(cylc get-global --print-run-dir) @@ -144,5 +144,45 @@ else fail "${TEST_NAME_BASE}-print.stderr" fi +# Test alternate run dir. +# 1. Normal case. +TEST_NAME="${TEST_NAME_BASE}-alt-run-dir" +SRC_DIR="${PWD}/foo" +REG="${PRE}/foo" +RUN_DIR="${CYLC_RUN_DIR}/${REG}" +ALT_RUN_DIR="${PWD}/alt" +mkdir "${SRC_DIR}"; touch "${SRC_DIR}/suite.rc" +run_ok "${TEST_NAME}" cylc register --run-dir="${ALT_RUN_DIR}" "${REG}" "${SRC_DIR}" +contains_ok "${TEST_NAME}.stdout" <<__OUT__ +REGISTERED ${REG} -> ${SRC_DIR} +__OUT__ +run_ok "${TEST_NAME}-check-link" test -L "${RUN_DIR}" +run_ok "${TEST_NAME}-rm-link" rm "${RUN_DIR}" +run_ok "${TEST_NAME}-rm-alt-run-dir" rm -r "${ALT_RUN_DIR}" +rm -r "${SRC_DIR}" + +# Test alternate run dir. +# 2. If reg already exists (as a directory). +TEST_NAME="${TEST_NAME_BASE}-alt-exists1" +mkdir "${RUN_DIR}" +mkdir "${SRC_DIR}"; touch "${SRC_DIR}/suite.rc" +run_fail "${TEST_NAME}" cylc register --run-dir="${ALT_RUN_DIR}" "${REG}" "${SRC_DIR}" +contains_ok "${TEST_NAME}.stderr" <<__OUT__ +SuiteServiceFileError: Run directory '${RUN_DIR}' already exists. +__OUT__ +rm -r $SRC_DIR ${RUN_DIR} + +# Test alternate run dir. +# 3. If reg already exists (as a valid symlink). +TEST_NAME="${TEST_NAME_BASE}-alt-exists2" +mkdir "${SRC_DIR}"; touch "${SRC_DIR}/suite.rc" +mkdir "${PWD}/target" +ln -s "${PWD}/target" "${RUN_DIR}" +run_fail "${TEST_NAME}" cylc register --run-dir="${ALT_RUN_DIR}" "${REG}" "${SRC_DIR}" +contains_ok "${TEST_NAME}.stderr" <<__OUT__ +SuiteServiceFileError: Symlink '${RUN_DIR}' already points to ${PWD}/target. +__OUT__ +rm -r "${SRC_DIR}" "${PWD}/target" "${CYLC_RUN_DIR}/${PRE}" + purge_suite "${SUITE_NAME}" exit From 39f622e73d248de0ab5f1455db65c356f55f43f7 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Sun, 30 Jun 2019 14:22:47 +0100 Subject: [PATCH 4/8] Revert Python 3 from cherry-pick. --- lib/cylc/suite_srv_files_mgr.py | 12 ++++++------ tests/registration/00-simple.t | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/cylc/suite_srv_files_mgr.py b/lib/cylc/suite_srv_files_mgr.py index 89ee09153c..f4e1f5cdb5 100644 --- a/lib/cylc/suite_srv_files_mgr.py +++ b/lib/cylc/suite_srv_files_mgr.py @@ -469,13 +469,13 @@ def register(self, reg=None, source=None, redirect=False, rundir=None): # Create service dir if necessary. srv_d = self.get_suite_srv_dir(reg) if rundir is None: - os.makedirs(srv_d, exist_ok=True) + mkdir_p(srv_d) else: suite_run_d, srv_d_name = os.path.split(srv_d) alt_suite_run_d = os.path.join(rundir, reg) alt_srv_d = os.path.join(rundir, reg, srv_d_name) - os.makedirs(alt_srv_d, exist_ok=True) - os.makedirs(os.path.dirname(suite_run_d), exist_ok=True) + mkdir_p(alt_srv_d) + mkdir_p(os.path.dirname(suite_run_d)) if os.path.islink(suite_run_d) and not os.path.exists(suite_run_d): # Remove a bad symlink. os.unlink(suite_run_d) @@ -483,14 +483,14 @@ def register(self, reg=None, source=None, redirect=False, rundir=None): os.symlink(alt_suite_run_d, suite_run_d) elif not os.path.islink(suite_run_d): raise SuiteServiceFileError( - f"Run directory '{suite_run_d}' already exists.") + "Run directory '%s' already exists." % suite_run_d) elif alt_suite_run_d != os.readlink(suite_run_d): target = os.readlink(suite_run_d) raise SuiteServiceFileError( - f"Symlink '{suite_run_d}' already points to {target}.") + "Symlink '%s' already points to %s." % ( + suite_run_d, target)) # (else already the right symlink) - temp = None # See if suite already has a source or not try: orig_source = os.readlink( diff --git a/tests/registration/00-simple.t b/tests/registration/00-simple.t index 8ad5ee1d37..686bf0d92c 100755 --- a/tests/registration/00-simple.t +++ b/tests/registration/00-simple.t @@ -168,7 +168,7 @@ mkdir "${RUN_DIR}" mkdir "${SRC_DIR}"; touch "${SRC_DIR}/suite.rc" run_fail "${TEST_NAME}" cylc register --run-dir="${ALT_RUN_DIR}" "${REG}" "${SRC_DIR}" contains_ok "${TEST_NAME}.stderr" <<__OUT__ -SuiteServiceFileError: Run directory '${RUN_DIR}' already exists. +Run directory '${RUN_DIR}' already exists. __OUT__ rm -r $SRC_DIR ${RUN_DIR} @@ -180,7 +180,7 @@ mkdir "${PWD}/target" ln -s "${PWD}/target" "${RUN_DIR}" run_fail "${TEST_NAME}" cylc register --run-dir="${ALT_RUN_DIR}" "${REG}" "${SRC_DIR}" contains_ok "${TEST_NAME}.stderr" <<__OUT__ -SuiteServiceFileError: Symlink '${RUN_DIR}' already points to ${PWD}/target. +Symlink '${RUN_DIR}' already points to ${PWD}/target. __OUT__ rm -r "${SRC_DIR}" "${PWD}/target" "${CYLC_RUN_DIR}/${PRE}" From a54626e04b0ac3a4df6e33c738e80a9be807c060 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Sun, 30 Jun 2019 16:23:51 +0100 Subject: [PATCH 5/8] style fix --- lib/cylc/suite_srv_files_mgr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cylc/suite_srv_files_mgr.py b/lib/cylc/suite_srv_files_mgr.py index f4e1f5cdb5..91f3740032 100644 --- a/lib/cylc/suite_srv_files_mgr.py +++ b/lib/cylc/suite_srv_files_mgr.py @@ -488,7 +488,7 @@ def register(self, reg=None, source=None, redirect=False, rundir=None): target = os.readlink(suite_run_d) raise SuiteServiceFileError( "Symlink '%s' already points to %s." % ( - suite_run_d, target)) + suite_run_d, target)) # (else already the right symlink) # See if suite already has a source or not From 6461f5d0e27498882364a6137ad2e1b59b9ada6f Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Thu, 29 Aug 2019 14:31:06 +1200 Subject: [PATCH 6/8] Document alternate run directories and sub-suites. --- bin/cylc-register | 7 ++--- doc/src/running-suites.rst | 52 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/bin/cylc-register b/bin/cylc-register index 06825db9c9..8ea2e7dc70 100755 --- a/bin/cylc-register +++ b/bin/cylc-register @@ -23,10 +23,11 @@ program can then be started, stopped, and targeted by name REG. (Note that "cylc run" can also register suites on the fly). Registration creates a suite run directory "~/cylc-run/REG/" containing a -".service/source" symlink to the suite definition PATH. The .service directory -will also be used for server authentication files at run time. +".service/source" symlink to the suite definition location. The .service +directory is also used for server authentication files at runtime. With the +"--run-dir" option "~/cylc-run/REG" can be symlinked to another location. -Suite names can be hierarchical, corresponding to the path under ~/cylc-run. +Suite names can be hierarchical, corresponding to path under the run directory. % cylc register dogs/fido PATH Register PATH/suite.rc as dogs/fido, with run directory ~/cylc-run/dogs/fido. diff --git a/doc/src/running-suites.rst b/doc/src/running-suites.rst index 7fff475d34..4ee1356d74 100644 --- a/doc/src/running-suites.rst +++ b/doc/src/running-suites.rst @@ -1631,3 +1631,55 @@ See the ``[suite servers]`` configuration section such they can often trigger on time even if the suite is delayed to the point that downstream tasks are late due to their dependence on previous-cycle tasks that are delayed. + + +.. _Alternate Suite Run Directories: + +Alternate Suite Run Directories +------------------------------- + +The ``cylc register`` command normally creates a suite run directory at +the standard location ``~/cylc-run//``. With the ``--run-dir`` +option it can create the run directory at some other location, with a symlink +from ``~/cylc-run/`` to allow access via the standard file path. + +This may be useful for quick-running :ref:`Sub-Suites` that generate large +numbers of files - you could put their run directories on fast local disk or +RAM disk, for performance and housekeeping reasons. + + +.. _Sub-Suites: + +Sub-Suites +---------- + +A single Cylc suite can configure multiple cycling sequences in the graph, but +cycles can't be nested. If you need *cycles within cycles* - e.g. to iterate +over many files generated by each run of a cycling task - current options are: + +- parameterize the sub-cycles + + - this is easy but it makes more tasks-per-cycle, which is the primary + determinant of suite size and server program efficiency + +- run a separate cycling suite over the sub-cycle, inside a main-suite task, + for each main-suite cycle point - i.e. use **sub-suites** + + - this is very efficient, but monitoring and run-directory housekeeping may + be more difficult because it creates multiple suites and run directories + +Sub-suites must be started with ``--no-detach`` so that the containing task +does not finish until the sub-suite does, and they should be non-cycling +or have a ``final cycle point`` so they don't keep on running indefinitely. + +Sub-suite names should normally incorporate the main-suite cycle point (use +``$CYLC_TASK_CYCLE_POINT`` in the ``cylc run`` command line to start the +sub-suite), so that successive sub-suites can run concurrently if necessary and +do not compete for the same run directory. This will generate a new sub-suite +run directory for every main-suite cycle point, so you may want to put +housekeeping tasks in the main suite to extract the useful products from each +sub-suite run and then delete the sub-suite run directory. + +For quick-running sub-suites that generate large numbers of files, consider +using :ref:`Alternate Suite Run Directories` for better performance and easier +housekeeping. From 994bd3609276893b21c7abfa7bbfeb67bb51371f Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Thu, 29 Aug 2019 16:53:25 +1200 Subject: [PATCH 7/8] Update change log. --- CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index ce5936694f..83b13bb904 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -30,6 +30,10 @@ Selected user-facing changes: ### Enhancements +[#3203](https://github.com/cylc/cylc-flow/pull/3203) - support alternate run +directories, particularly for sub-suites. + + [#3303](https://github.com/cylc/cylc-flow/pull/3303) - improve CLI task-globbing help. From 0c50468c4b273ae098d0b9347fb99d03afff4fe3 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Fri, 30 Aug 2019 15:36:47 +1200 Subject: [PATCH 8/8] Tidy up rego tests. --- tests/registration/00-simple.t | 284 ++++++++++++++++++++------------- 1 file changed, 169 insertions(+), 115 deletions(-) diff --git a/tests/registration/00-simple.t b/tests/registration/00-simple.t index 686bf0d92c..1828f1c4c9 100755 --- a/tests/registration/00-simple.t +++ b/tests/registration/00-simple.t @@ -14,175 +14,229 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . + #------------------------------------------------------------------------------ # Test suite registration -. "$(dirname "$0")/test_header" -set_test_number 34 +export RND_SUITE_NAME +export RND_SUITE_SOURCE +export RND_SUITE_RUNDIR +export CYLC_RUN_DIR + +CYLC_RUN_DIR="$(cylc get-global-config --print-run-dir)" + +function make_rnd_suite() { + # Create a randomly-named suite source directory. + # Define its run directory. + RND_SUITE_NAME=x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6) + RND_SUITE_SOURCE="$PWD/${RND_SUITE_NAME}" + mkdir -p "${RND_SUITE_SOURCE}" + touch "${RND_SUITE_SOURCE}/suite.rc" + RND_SUITE_RUNDIR="${CYLC_RUN_DIR}/${RND_SUITE_NAME}" +} + +function purge_rnd_suite() { + # Remove the suite source created by make_rnd_suite(). + # And remove its run-directory too. + RND_SUITE_SOURCE=${1:-$RND_SUITE_SOURCE} + RND_SUITE_RUNDIR=${2:-$RND_SUITE_RUNDIR} + rm -rf "${RND_SUITE_SOURCE}" + rm -rf "${RND_SUITE_RUNDIR}" +} -init_suite "${TEST_NAME_BASE}" <<'__SUITE_RC__' -[meta] - title = the quick brown fox -[scheduling] - [[dependencies]] - graph = a => b => c -[runtime] - [[a,b,c]] - script = true -__SUITE_RC__ +. "$(dirname "$0")/test_header" +set_test_number 37 -# Unique suite run-dir prefix to avoid messing with real suites. -PRE=cylctb-${CYLC_TEST_TIME_INIT} +# Use $SUITE_NAME and $SUITE_RUN_DIR defined by test_header -# Test fail no suite.rc file. -CYLC_RUN_DIR=$(cylc get-global --print-run-dir) -TEST_NAME="${TEST_NAME_BASE}-noreg" -run_fail "${TEST_NAME}" cylc register "${SUITE_NAME}" "${PWD}/zilch" +#------------------------------ +# Test fail no suite source dir +TEST_NAME="${TEST_NAME_BASE}-nodir" +make_rnd_suite +rm -rf "${RND_SUITE_SOURCE}" +run_fail "${TEST_NAME}" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}" contains_ok "${TEST_NAME}.stderr" <<__ERR__ -ERROR: no suite.rc in ${PWD}/zilch +ERROR: no suite.rc in ${RND_SUITE_SOURCE} __ERR__ +purge_rnd_suite + +#--------------------------- +# Test fail no suite.rc file +TEST_NAME="${TEST_NAME_BASE}-nodir" +make_rnd_suite +rm -f "${RND_SUITE_SOURCE}/suite.rc" +run_fail "${TEST_NAME}" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}" +contains_ok "${TEST_NAME}.stderr" <<__ERR__ +ERROR: no suite.rc in ${RND_SUITE_SOURCE} +__ERR__ +purge_rnd_suite -CHEESE=${PRE}-cheese +#------------------------------------------------------- # Test default name: "cylc reg" (suite in $PWD, no args) -TEST_NAME="${TEST_NAME_BASE}-cheese" -mkdir $CHEESE -cd $CHEESE -touch suite.rc +TEST_NAME="${TEST_NAME_BASE}-pwd1" +make_rnd_suite +pushd "${RND_SUITE_SOURCE}" || exit 1 run_ok "${TEST_NAME}" cylc register contains_ok "${TEST_NAME}.stdout" <<__OUT__ -REGISTERED $CHEESE -> ${PWD} +REGISTERED $RND_SUITE_NAME -> ${RND_SUITE_SOURCE} __OUT__ -cd .. -rm -rf "${CYLC_RUN_DIR}/$CHEESE" - -# Test default name: "cylc reg REG" (suite in $PWD) -TEST_NAME="${TEST_NAME_BASE}-toast" -cd $CHEESE -TOAST=${PRE}-toast -run_ok "${TEST_NAME}" cylc register $TOAST +popd || exit 1 +purge_rnd_suite + +#-------------------------------------------------- +# Test default path: "cylc reg REG" (suite in $PWD) +TEST_NAME="${TEST_NAME_BASE}-pwd2" +make_rnd_suite +pushd "${RND_SUITE_SOURCE}" || exit 1 +run_ok "${TEST_NAME}" cylc register "${RND_SUITE_NAME}" contains_ok "${TEST_NAME}.stdout" <<__OUT__ -REGISTERED $TOAST -> ${PWD} +REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_SOURCE} __OUT__ -cd .. -rm -rf "${CYLC_RUN_DIR}/$TOAST" +popd || exit 1 +purge_rnd_suite +#------------------------- # Test "cylc reg REG PATH" -TEST_NAME="${TEST_NAME_BASE}-bagels" -BAGELS=${PRE}-bagels -run_ok "${TEST_NAME}" cylc register $BAGELS $CHEESE +TEST_NAME="${TEST_NAME_BASE}-normal" +make_rnd_suite +run_ok "${TEST_NAME}" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}" contains_ok "${TEST_NAME}.stdout" <<__OUT__ -REGISTERED $BAGELS -> ${PWD}/$CHEESE +REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_SOURCE} __OUT__ -rm -rf "${CYLC_RUN_DIR}/$BAGELS" - -# Test "cylc reg REG ~/cylc-run/REG" -TEST_NAME="${TEST_NAME_BASE}-onion" -ONION="${PRE}-onion" -mkdir -p "${CYLC_RUN_DIR}/${ONION}" -cp -p "${PWD}/suite.rc" "${CYLC_RUN_DIR}/${ONION}/" -run_ok "${TEST_NAME}" cylc register "${ONION}" "${CYLC_RUN_DIR}/${ONION}" +purge_rnd_suite + +#-------------------------------------------------------------------- +# Test register existing run directory: "cylc reg REG ~/cylc-run/REG" +TEST_NAME="${TEST_NAME_BASE}-reg-run-dir" +make_rnd_suite +mkdir -p "${RND_SUITE_RUNDIR}" +cp "${RND_SUITE_SOURCE}/suite.rc" "${RND_SUITE_RUNDIR}" +run_ok "${TEST_NAME}" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_RUNDIR}" contains_ok "${TEST_NAME}.stdout" <<__OUT__ -REGISTERED ${ONION} -> ${CYLC_RUN_DIR}/${ONION} +REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_RUNDIR} __OUT__ -SOURCE="$(readlink "${CYLC_RUN_DIR}/${ONION}/.service/source")" +SOURCE="$(readlink "${RND_SUITE_RUNDIR}/.service/source")" run_ok "${TEST_NAME}-source" test '..' = "${SOURCE}" # Run it twice -run_ok "${TEST_NAME}-2" cylc register "${ONION}" "${CYLC_RUN_DIR}/${ONION}" +run_ok "${TEST_NAME}-2" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_RUNDIR}" contains_ok "${TEST_NAME}-2.stdout" <<__OUT__ -REGISTERED ${ONION} -> ${CYLC_RUN_DIR}/${ONION} +REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_RUNDIR} __OUT__ -SOURCE="$(readlink "${CYLC_RUN_DIR}/${ONION}/.service/source")" -run_ok "${TEST_NAME}-2-source" test '..' = "${SOURCE}" -rm -rf "${CYLC_RUN_DIR}/${ONION}" +SOURCE="$(readlink "${RND_SUITE_RUNDIR}/.service/source")" +run_ok "${TEST_NAME}-source" test '..' = "${SOURCE}" +purge_rnd_suite +#---------------------------------------------------------------- # Test fail "cylc reg REG PATH" where REG already points to PATH2 -YOGHURT=${PRE}-YOGHURT -cp -r $CHEESE $YOGHURT -TEST_NAME="${TEST_NAME_BASE}-cheese" -run_ok "${TEST_NAME}" cylc register $CHEESE $CHEESE -TEST_NAME="${TEST_NAME_BASE}-repurpose1" -run_fail "${TEST_NAME}" cylc register $CHEESE $YOGHURT +TEST_NAME="${TEST_NAME_BASE}-dup1" +make_rnd_suite +run_ok "${TEST_NAME}" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}" +RND_SUITE_NAME1="${RND_SUITE_NAME}" +RND_SUITE_SOURCE1="${RND_SUITE_SOURCE}" +RND_SUITE_RUNDIR1="${RND_SUITE_RUNDIR}" +make_rnd_suite +TEST_NAME="${TEST_NAME_BASE}-dup2" +run_fail "${TEST_NAME}" cylc register "${RND_SUITE_NAME1}" "${RND_SUITE_SOURCE}" contains_ok "${TEST_NAME}.stderr" <<__ERR__ -ERROR: the name '$CHEESE' already points to ${PWD}/$CHEESE. +ERROR: the name '${RND_SUITE_NAME1}' already points to ${RND_SUITE_SOURCE1}. Use --redirect to re-use an existing name and run directory. __ERR__ - -# Test succeed "cylc reg REG PATH" where REG already points to PATH2 -TEST_NAME="${TEST_NAME_BASE}-repurpose2" -cp -r $CHEESE $YOGHURT -run_ok "${TEST_NAME}" cylc register --redirect $CHEESE $YOGHURT +# Now force it +TEST_NAME="${TEST_NAME_BASE}-dup3" +run_ok "${TEST_NAME}" cylc register --redirect "${RND_SUITE_NAME1}" "${RND_SUITE_SOURCE}" sed -i 's/^\t//; s/^.* WARNING - /WARNING - /' "${TEST_NAME}.stderr" contains_ok "${TEST_NAME}.stderr" <<__ERR__ -WARNING - the name '$CHEESE' points to ${PWD}/$CHEESE. -It will now be redirected to ${PWD}/$YOGHURT. -Files in the existing $CHEESE run directory will be overwritten. +WARNING - the name '${RND_SUITE_NAME1}' points to ${RND_SUITE_SOURCE1}. +It will now be redirected to ${RND_SUITE_SOURCE}. +Files in the existing ${RND_SUITE_NAME1} run directory will be overwritten. __ERR__ contains_ok "${TEST_NAME}.stdout" <<__OUT__ -REGISTERED $CHEESE -> ${PWD}/$YOGHURT +REGISTERED ${RND_SUITE_NAME1} -> ${RND_SUITE_SOURCE} __OUT__ -rm -rf "${CYLC_RUN_DIR}/$CHEESE" - -run_ok "${TEST_NAME_BASE}-get-dir" cylc get-directory "${SUITE_NAME}" -cd .. # necessary so the suite is being validated via the database not filepath -run_ok "${TEST_NAME_BASE}-val" cylc validate "${SUITE_NAME}" -cd "${OLDPWD}" +TEST_NAME="${TEST_NAME_BASE}-get-dir" +run_ok "${TEST_NAME}" cylc get-directory "${RND_SUITE_NAME1}" +contains_ok "${TEST_NAME}.stdout" <<__ERR__ +${RND_SUITE_SOURCE} +__ERR__ -run_ok "${TEST_NAME_BASE}-print" cylc print -contains_ok "${TEST_NAME_BASE}-print.stdout" <<__OUT__ -${SUITE_NAME} | the quick brown fox | ${TEST_DIR}/${SUITE_NAME} -__OUT__ +purge_rnd_suite +purge_rnd_suite "${RND_SUITE_SOURCE1}" "${RND_SUITE_RUNDIR1}" -# Filter out errors from 'bad' suites in the 'cylc-run' directory -NONSPECIFIC_ERR2='\[Errno 2\] No such file or directory:' -SPECIFIC_ERR2="$NONSPECIFIC_ERR2 '$HOME/cylc-run/$SUITE_NAME/suite.rc'" -ERR2_COUNT=$(grep -c "$SPECIFIC_ERR2" "${TEST_NAME_BASE}-print.stderr") -if [ "$ERR2_COUNT" -eq "0" ]; then - grep -v -s "$NONSPECIFIC_ERR2" "${TEST_NAME_BASE}-print.stderr" > "${TEST_NAME_BASE}-print-filtered.stderr" - cmp_ok "${TEST_NAME_BASE}-print-filtered.stderr" <'/dev/null' -else - fail "${TEST_NAME_BASE}-print.stderr" -fi - -# Test alternate run dir. +#----------------------- +# Test alternate run dir # 1. Normal case. TEST_NAME="${TEST_NAME_BASE}-alt-run-dir" -SRC_DIR="${PWD}/foo" -REG="${PRE}/foo" -RUN_DIR="${CYLC_RUN_DIR}/${REG}" +make_rnd_suite ALT_RUN_DIR="${PWD}/alt" -mkdir "${SRC_DIR}"; touch "${SRC_DIR}/suite.rc" -run_ok "${TEST_NAME}" cylc register --run-dir="${ALT_RUN_DIR}" "${REG}" "${SRC_DIR}" +run_ok "${TEST_NAME}" \ + cylc register --run-dir="${ALT_RUN_DIR}" "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}" contains_ok "${TEST_NAME}.stdout" <<__OUT__ -REGISTERED ${REG} -> ${SRC_DIR} +REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_SOURCE} __OUT__ -run_ok "${TEST_NAME}-check-link" test -L "${RUN_DIR}" -run_ok "${TEST_NAME}-rm-link" rm "${RUN_DIR}" +run_ok "${TEST_NAME}-check-link" test -L "${RND_SUITE_RUNDIR}" +run_ok "${TEST_NAME}-rm-link" rm "${RND_SUITE_RUNDIR}" run_ok "${TEST_NAME}-rm-alt-run-dir" rm -r "${ALT_RUN_DIR}" -rm -r "${SRC_DIR}" +purge_rnd_suite -# Test alternate run dir. # 2. If reg already exists (as a directory). TEST_NAME="${TEST_NAME_BASE}-alt-exists1" -mkdir "${RUN_DIR}" -mkdir "${SRC_DIR}"; touch "${SRC_DIR}/suite.rc" -run_fail "${TEST_NAME}" cylc register --run-dir="${ALT_RUN_DIR}" "${REG}" "${SRC_DIR}" +make_rnd_suite +ALT_RUN_DIR="${PWD}/alt" +mkdir -p "${RND_SUITE_RUNDIR}" +run_fail "${TEST_NAME}" \ + cylc register --run-dir="${ALT_RUN_DIR}" "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}" contains_ok "${TEST_NAME}.stderr" <<__OUT__ -Run directory '${RUN_DIR}' already exists. +Run directory '${RND_SUITE_RUNDIR}' already exists. __OUT__ -rm -r $SRC_DIR ${RUN_DIR} +purge_rnd_suite -# Test alternate run dir. # 3. If reg already exists (as a valid symlink). TEST_NAME="${TEST_NAME_BASE}-alt-exists2" -mkdir "${SRC_DIR}"; touch "${SRC_DIR}/suite.rc" -mkdir "${PWD}/target" -ln -s "${PWD}/target" "${RUN_DIR}" -run_fail "${TEST_NAME}" cylc register --run-dir="${ALT_RUN_DIR}" "${REG}" "${SRC_DIR}" +make_rnd_suite +ALT_RUN_DIR="${PWD}/alt" +TDIR=$(mktemp -d) +mkdir -p "$(dirname "${RND_SUITE_RUNDIR}")" +ln -s "${TDIR}" "${RND_SUITE_RUNDIR}" +run_fail "${TEST_NAME}" \ + cylc register --run-dir="${ALT_RUN_DIR}" "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}" contains_ok "${TEST_NAME}.stderr" <<__OUT__ -Symlink '${RUN_DIR}' already points to ${PWD}/target. +Symlink '${RND_SUITE_RUNDIR}' already points to ${TDIR}. __OUT__ -rm -r "${SRC_DIR}" "${PWD}/target" "${CYLC_RUN_DIR}/${PRE}" +purge_rnd_suite +rm -rf "${TDIR}" + +#----------------------------------------------------------------------------- +# Now use a real suite + +init_suite "${TEST_NAME_BASE}" <<'__SUITE_RC__' +[meta] + title = the quick brown fox +[scheduling] + [[dependencies]] + graph = a => b => c +[runtime] + [[a,b,c]] + script = true +__SUITE_RC__ + +run_ok "${TEST_NAME_BASE}-val" cylc validate "${SUITE_NAME}" + +run_ok "${TEST_NAME_BASE}-print" cylc print +contains_ok "${TEST_NAME_BASE}-print.stdout" <<__OUT__ +${SUITE_NAME} | the quick brown fox | ${TEST_DIR}/${SUITE_NAME} +__OUT__ + +# Filter out errors from 'bad' suites in the 'cylc-run' directory +NONSPECIFIC_ERR2='\[Errno 2\] No such file or directory:' +SPECIFIC_ERR2="$NONSPECIFIC_ERR2 '$HOME/cylc-run/${SUITE_NAME}/suite.rc'" +ERR2_COUNT="$(grep -c "$SPECIFIC_ERR2" "${TEST_NAME_BASE}-print.stderr")" +if ((ERR2_COUNT == 0)); then + grep -v -s "$NONSPECIFIC_ERR2" "${TEST_NAME_BASE}-print.stderr" > "${TEST_NAME_BASE}-print-filtered.stderr" + cmp_ok "${TEST_NAME_BASE}-print-filtered.stderr" <'/dev/null' +else + fail "${TEST_NAME_BASE}-print.stderr" +fi purge_suite "${SUITE_NAME}" exit