rose suite-clean: improve parent directory clean

lib/python/rose/suite_clean.py

@@ -102,36 +102,51 @@ def _clean(self, suite_name, only_items=None):
                         loc_rel = os.path.join(suite_dir_rel, item)
                     locs.append(os.path.join(item_root, loc_rel))
                     roots.add(item_root)
+            locs.reverse()
             if self.host_selector.is_local_host(auth):
                 # Clean relevant items
                 for loc in locs:
                     loc = os.path.abspath(env_var_process(loc))
-                    for name in sorted(glob(loc)):
+                    for name in sorted(glob(loc), reverse=True):
                         engine.fs_util.delete(name)
-                # Clean empty directories
-                # Change directory to root level to avoid cleaning them as well
-                # For cylc suites, e.g. it can clean up to an empty "cylc-run/"
-                # directory.
-                for root in sorted(roots):
-                    cwd = os.getcwd()
-                    if root:
-                        try:
-                            os.chdir(env_var_process(root))
-                        except OSError:
-                            continue
-                    # Reverse sort to ensure that e.g. "share/cycle/" is
-                    # cleaned before "share/"
-                    for name in sorted(self.CLEANABLE_PATHS, reverse=True):
-                        try:
-                            os.removedirs(os.path.join(suite_dir_rel, name))
-                        except OSError:
-                            pass
-                    try:
-                        os.removedirs(suite_dir_rel)
-                    except OSError:
-                        pass
-                    if root:
-                        os.chdir(cwd)
+                if not only_items:
+                    # Clean empty directories
+                    # Change directory to root level to avoid cleaning them as
+                    # well For cylc suites, e.g. it can clean up to an empty
+                    # "cylc-run/" directory.
+                    for root in sorted(roots):
+                        old_cwd = os.getcwd()
+                        cwd = old_cwd
+                        if root:
+                            try:
+                                os.chdir(env_var_process(root))
+                                cwd = os.getcwd()
+                            except OSError:
+                                continue
+                        # Reverse sort to ensure that e.g. "share/cycle/" is
+                        # cleaned before "share/"
+                        for name in sorted(self.CLEANABLE_PATHS, reverse=True):
+                            try:
+                                os.removedirs(
+                                    os.path.join(suite_dir_rel, name))
+                            except OSError:
+                                pass
+                            else:
+                                engine.handle_event(FileSystemEvent(
+                                    FileSystemEvent.DELETE,
+                                    os.path.join(cwd, suite_dir_rel, name)))
+                        if os.sep in suite_dir_rel:
+                            dir_rel = os.path.dirname(suite_dir_rel)
+                            try:
+                                os.removedirs(dir_rel)
+                            except OSError:
+                                pass
+                            else:
+                                engine.handle_event(FileSystemEvent(
+                                    FileSystemEvent.DELETE,
+                                    os.path.join(cwd, dir_rel)))
+                        if root:
+                            os.chdir(old_cwd)
             else:
                 # Invoke bash as a login shell. The root location of a path may
                 # be in $DIR syntax, which can only be expanded correctly in a
@@ -141,28 +156,32 @@ def _clean(self, suite_name, only_items=None):
                 # desirable lines.
                 command = engine.popen.get_cmd("ssh", auth, "bash", "-l", "-c")
                 sh_command = (
-                    "echo %(uuid)s; ls -d %(locs)s|sort; rm -fr %(locs)s"
+                    "echo %(uuid)s; ls -d -r %(locs)s; rm -fr %(locs)s"
                 ) % {
                     "locs": engine.popen.list_to_shell_str(locs),
                     "uuid": uuid_str,
                 }
-                # Clean empty directories
-                # Change directory to root level to avoid cleaning them as well
-                # For cylc suites, e.g. it can clean up to an empty "cylc-run/"
-                # directory.
-                for root in roots:
-                    names = []
-                    # Reverse sort to ensure that e.g. "share/cycle/" is
-                    # cleaned before "share/"
-                    for name in sorted(self.CLEANABLE_PATHS, reverse=True):
-                        names.append(os.path.join(suite_dir_rel, name))
-                    sh_command += (
-                        "; " +
-                        "(cd %(root)s; rmdir -p %(names)s 2>/dev/null || true)"
-                    ) % {
-                        "root": root,
-                        "names": engine.popen.list_to_shell_str(names),
-                    }
+                if not only_items:
+                    # Clean empty directories
+                    # Change directory to root level to avoid cleaning them as
+                    # well For cylc suites, e.g. it can clean up to an empty
+                    # "cylc-run/" directory.
+                    for root in roots:
+                        names = []
+                        # Reverse sort to ensure that e.g. "share/cycle/" is
+                        # cleaned before "share/"
+                        for name in sorted(self.CLEANABLE_PATHS, reverse=True):
+                            names.append(os.path.join(suite_dir_rel, name))
+                        if os.sep in suite_dir_rel:
+                            names.append(os.path.dirname(suite_dir_rel))
+                        sh_command += (
+                            "; " +
+                            "(cd %(root)s; " +
+                            "rmdir -p %(names)s 2>/dev/null || true)"
+                        ) % {
+                            "root": root,
+                            "names": engine.popen.list_to_shell_str(names),
+                        }
                 command.append(quote(sh_command))
                 is_after_uuid_str = False
                 for line in engine.popen(*command)[0].splitlines():

t/rose-suite-clean/01-running.t

@@ -75,6 +75,9 @@ run_pass "$TEST_KEY" rose suite-clean -y $NAME
 sed -i '/\/\.cylc\//d' "$TEST_KEY.out"
 file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__
 [INFO] delete: $HOME/cylc-run/$NAME/log/rose-suite-run.host
+[INFO] delete: $SUITE_RUN_DIR/work/
+[INFO] delete: $SUITE_RUN_DIR/share/cycle/
+[INFO] delete: $SUITE_RUN_DIR/share/
 [INFO] delete: $SUITE_RUN_DIR/
 __OUT__
 file_cmp "$TEST_KEY.err" "$TEST_KEY.err" </dev/null

t/rose-suite-clean/02-only-1.t

@@ -85,15 +85,15 @@ run_suite
 run_pass "$TEST_KEY" rose suite-clean -y -n "$NAME" --only=share --only=work
 if [[ -n "$JOB_HOST" ]]; then
     file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__
-[INFO] delete: $SUITE_RUN_DIR/share/
 [INFO] delete: $SUITE_RUN_DIR/work/
-[INFO] delete: $JOB_HOST:cylc-run/$NAME/share
+[INFO] delete: $SUITE_RUN_DIR/share/
 [INFO] delete: $JOB_HOST:cylc-run/$NAME/work
+[INFO] delete: $JOB_HOST:cylc-run/$NAME/share
 __OUT__
 else
     file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__
-[INFO] delete: $SUITE_RUN_DIR/share/
 [INFO] delete: $SUITE_RUN_DIR/work/
+[INFO] delete: $SUITE_RUN_DIR/share/
 __OUT__
 fi
 #-------------------------------------------------------------------------------

t/rose-suite-clean/03-only-2.t

@@ -61,12 +61,12 @@ if [[ -n "$JOB_HOST" ]]; then
     # We do not know the relative sort order of $SUITE_RUN_DIR and $JOB_HOST.
     LANG=C sort >"$TEST_KEY.expected.out" <<__OUT__
 [INFO] delete: $HOME/cylc-run/$NAME/log/rose-suite-run.host
-[INFO] delete: $SUITE_RUN_DIR/share/cycle/20000101T0000Z/
-[INFO] delete: $SUITE_RUN_DIR/share/cycle/20100101T0000Z/
 [INFO] delete: $SUITE_RUN_DIR/share/cycle/20200101T0000Z/
-[INFO] delete: $JOB_HOST:cylc-run/$NAME/share/cycle/20000101T0000Z
-[INFO] delete: $JOB_HOST:cylc-run/$NAME/share/cycle/20100101T0000Z
+[INFO] delete: $SUITE_RUN_DIR/share/cycle/20100101T0000Z/
+[INFO] delete: $SUITE_RUN_DIR/share/cycle/20000101T0000Z/
 [INFO] delete: $JOB_HOST:cylc-run/$NAME/share/cycle/20200101T0000Z
+[INFO] delete: $JOB_HOST:cylc-run/$NAME/share/cycle/20100101T0000Z
+[INFO] delete: $JOB_HOST:cylc-run/$NAME/share/cycle/20000101T0000Z
 __OUT__
 else
     cat >"$TEST_KEY.expected.out" <<__OUT__
@@ -84,30 +84,30 @@ run_pass "$TEST_KEY" \
     --only=etc/*/greet-earth.out
 if [[ -n "$JOB_HOST" ]]; then
     file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__
-[INFO] delete: $SUITE_RUN_DIR/etc/20000101T0000Z/greet-earth.out
-[INFO] delete: $SUITE_RUN_DIR/etc/20050101T0000Z/greet-earth.out
-[INFO] delete: $SUITE_RUN_DIR/etc/20100101T0000Z/greet-earth.out
-[INFO] delete: $SUITE_RUN_DIR/etc/20150101T0000Z/greet-earth.out
-[INFO] delete: $SUITE_RUN_DIR/etc/20200101T0000Z/greet-earth.out
-[INFO] delete: $SUITE_RUN_DIR/share/cycle/20050101T0000Z/hello-world.out
 [INFO] delete: $SUITE_RUN_DIR/share/cycle/20150101T0000Z/hello-world.out
-[INFO] delete: $JOB_HOST:cylc-run/$NAME/etc/20000101T0000Z/greet-earth.out
-[INFO] delete: $JOB_HOST:cylc-run/$NAME/etc/20050101T0000Z/greet-earth.out
-[INFO] delete: $JOB_HOST:cylc-run/$NAME/etc/20100101T0000Z/greet-earth.out
-[INFO] delete: $JOB_HOST:cylc-run/$NAME/etc/20150101T0000Z/greet-earth.out
-[INFO] delete: $JOB_HOST:cylc-run/$NAME/etc/20200101T0000Z/greet-earth.out
-[INFO] delete: $JOB_HOST:cylc-run/$NAME/share/cycle/20050101T0000Z/hello-world.out
+[INFO] delete: $SUITE_RUN_DIR/share/cycle/20050101T0000Z/hello-world.out
+[INFO] delete: $SUITE_RUN_DIR/etc/20200101T0000Z/greet-earth.out
+[INFO] delete: $SUITE_RUN_DIR/etc/20150101T0000Z/greet-earth.out
+[INFO] delete: $SUITE_RUN_DIR/etc/20100101T0000Z/greet-earth.out
+[INFO] delete: $SUITE_RUN_DIR/etc/20050101T0000Z/greet-earth.out
+[INFO] delete: $SUITE_RUN_DIR/etc/20000101T0000Z/greet-earth.out
 [INFO] delete: $JOB_HOST:cylc-run/$NAME/share/cycle/20150101T0000Z/hello-world.out
+[INFO] delete: $JOB_HOST:cylc-run/$NAME/share/cycle/20050101T0000Z/hello-world.out
+[INFO] delete: $JOB_HOST:cylc-run/$NAME/etc/20200101T0000Z/greet-earth.out
+[INFO] delete: $JOB_HOST:cylc-run/$NAME/etc/20150101T0000Z/greet-earth.out
+[INFO] delete: $JOB_HOST:cylc-run/$NAME/etc/20100101T0000Z/greet-earth.out
+[INFO] delete: $JOB_HOST:cylc-run/$NAME/etc/20050101T0000Z/greet-earth.out
+[INFO] delete: $JOB_HOST:cylc-run/$NAME/etc/20000101T0000Z/greet-earth.out
 __OUT__
 else
     file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__
-[INFO] delete: $SUITE_RUN_DIR/etc/20000101T0000Z/greet-earth.out
-[INFO] delete: $SUITE_RUN_DIR/etc/20050101T0000Z/greet-earth.out
-[INFO] delete: $SUITE_RUN_DIR/etc/20100101T0000Z/greet-earth.out
-[INFO] delete: $SUITE_RUN_DIR/etc/20150101T0000Z/greet-earth.out
-[INFO] delete: $SUITE_RUN_DIR/etc/20200101T0000Z/greet-earth.out
-[INFO] delete: $SUITE_RUN_DIR/share/cycle/20050101T0000Z/hello-world.out
 [INFO] delete: $SUITE_RUN_DIR/share/cycle/20150101T0000Z/hello-world.out
+[INFO] delete: $SUITE_RUN_DIR/share/cycle/20050101T0000Z/hello-world.out
+[INFO] delete: $SUITE_RUN_DIR/etc/20200101T0000Z/greet-earth.out
+[INFO] delete: $SUITE_RUN_DIR/etc/20150101T0000Z/greet-earth.out
+[INFO] delete: $SUITE_RUN_DIR/etc/20100101T0000Z/greet-earth.out
+[INFO] delete: $SUITE_RUN_DIR/etc/20050101T0000Z/greet-earth.out
+[INFO] delete: $SUITE_RUN_DIR/etc/20000101T0000Z/greet-earth.out
 __OUT__
 fi
 #-------------------------------------------------------------------------------

t/rose-suite-clean/04-only-3.t

@@ -50,11 +50,13 @@ NAME=$(basename $SUITE_RUN_DIR)
 run_suite
 TEST_KEY="$TEST_KEY_BASE-work"
 run_pass "$TEST_KEY" rose suite-clean -y -n "$NAME" --only=work
-LANG=C sort >"$TEST_KEY.out.expected" <<__OUT__
-[INFO] delete: $HOME/cylc-run/$NAME/log/rose-suite-run.host
-[INFO] delete: $SUITE_RUN_DIR/work
+{
+    echo "[INFO] delete: $HOME/cylc-run/$NAME/log/rose-suite-run.host"
+    LANG=C sort -r <<__OUT__
 [INFO] delete: $ROOT_DIR_WORK/cylc-run/$NAME/work/
+[INFO] delete: $SUITE_RUN_DIR/work
 __OUT__
+} >"$TEST_KEY.out.expected"
 file_cmp  "$TEST_KEY.out" "$TEST_KEY.out" "$TEST_KEY.out.expected"
 #-------------------------------------------------------------------------------
 rose suite-clean -q -y --name="$NAME"

t/rose-suite-clean/05-only-4.t

@@ -52,9 +52,9 @@ TEST_KEY="$TEST_KEY_BASE-work"
 run_pass "$TEST_KEY" rose suite-clean -y -n "$NAME" --only=work/20?00101T0000Z
 file_cmp  "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__
 [INFO] delete: $HOME/cylc-run/$NAME/log/rose-suite-run.host
-[INFO] delete: $SUITE_RUN_DIR/work/20000101T0000Z/
-[INFO] delete: $SUITE_RUN_DIR/work/20100101T0000Z/
 [INFO] delete: $SUITE_RUN_DIR/work/20200101T0000Z/
+[INFO] delete: $SUITE_RUN_DIR/work/20100101T0000Z/
+[INFO] delete: $SUITE_RUN_DIR/work/20000101T0000Z/
 __OUT__
 #-------------------------------------------------------------------------------
 rose suite-clean -q -y --name="$NAME"

t/rose-suite-clean/06-only-5.t

@@ -60,7 +60,7 @@ run_pass "$TEST_KEY" rose suite-clean -y -n "$NAME" --only=work
 {
     echo "[INFO] delete: ${HOME}/cylc-run/${NAME}/log/rose-suite-run.host"
     echo "[INFO] delete: $SUITE_RUN_DIR/work/"
-    LANG=C sort <<__OUT__
+    LANG=C sort -r <<__OUT__
 [INFO] delete: $JOB_HOST:cylc-run/$NAME/work
 [INFO] delete: $JOB_HOST:$JOB_HOST_WORK/cylc-run/$NAME/work
 __OUT__

t/rose-suite-clean/07-only-6.t

@@ -57,12 +57,12 @@ TEST_KEY="$TEST_KEY_BASE-work"
 run_pass "$TEST_KEY" rose suite-clean -y -n "$NAME" --only=work/20?00101T0000Z
 file_cmp  "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__
 [INFO] delete: ${HOME}/cylc-run/${NAME}/log/rose-suite-run.host
-[INFO] delete: $SUITE_RUN_DIR/work/20000101T0000Z/
-[INFO] delete: $SUITE_RUN_DIR/work/20100101T0000Z/
 [INFO] delete: $SUITE_RUN_DIR/work/20200101T0000Z/
-[INFO] delete: $JOB_HOST:cylc-run/$NAME/work/20000101T0000Z
-[INFO] delete: $JOB_HOST:cylc-run/$NAME/work/20100101T0000Z
+[INFO] delete: $SUITE_RUN_DIR/work/20100101T0000Z/
+[INFO] delete: $SUITE_RUN_DIR/work/20000101T0000Z/
 [INFO] delete: $JOB_HOST:cylc-run/$NAME/work/20200101T0000Z
+[INFO] delete: $JOB_HOST:cylc-run/$NAME/work/20100101T0000Z
+[INFO] delete: $JOB_HOST:cylc-run/$NAME/work/20000101T0000Z
 __OUT__
 #-------------------------------------------------------------------------------
 rose suite-clean -q -y --name="$NAME"

t/rose-suite-clean/08-rmdir.t

@@ -51,7 +51,7 @@ fi
 export ROSE_CONF_PATH=
 export ROSE_TEST_ROOT_DIR="${PWD}/root.d"
 set -e
-rose suite-run -C "${PWD}/${NAME}" --no-gcontrol --host='localhost' -- --debug
+rose suite-run -q -C "${PWD}/${NAME}" --no-gcontrol --host='localhost' -- --debug
 # Prove that the directories exist before clean
 test -d "${HOME}/cylc-run/${NAME}"
 test -d "${PWD}/root.d/cylc-run/${NAME}"

t/rose-suite-clean/09-rmdir-2.t

@@ -0,0 +1,86 @@
+#!/bin/bash
+#-------------------------------------------------------------------------------
+# (C) British Crown Copyright 2012-6 Met Office.
+#
+# This file is part of Rose, a framework for meteorological suites.
+#
+# Rose is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Rose is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Rose. If not, see <http://www.gnu.org/licenses/>.
+#-------------------------------------------------------------------------------
+# Test "rose suite-clean", clean up parent directories.
+#-------------------------------------------------------------------------------
+. "$(dirname "$0")/test_header"
+
+mkdir -p "${HOME}/cylc-run"
+SUITE_RUN_DIR0="$(mktemp -d --tmpdir="${HOME}/cylc-run" 'rose-test-battery.XXXXXX')"
+NAME0="$(basename "${SUITE_RUN_DIR0}")"
+SUITE_RUN_DIR="${SUITE_RUN_DIR0}/foo/bar"
+mkdir -p "${SUITE_RUN_DIR}"
+NAME="${NAME0}/foo/bar"
+
+cp -pr "${TEST_SOURCE_DIR}/${TEST_KEY_BASE}" 'source'
+
+JOB_HOST="$(rose config 't' 'job-host')"
+JOB_HOST_RUN_ROOT="$(rose config 't' 'job-host-run-root')"
+JOB_HOST_OPT=
+if [[ -n "${JOB_HOST}" && -n "${JOB_HOST_RUN_ROOT}" ]]; then
+    export JOB_HOST="$(rose host-select -q "${JOB_HOST}")"
+    export JOB_HOST_RUN_ROOT
+    cat >>'source/rose-suite.conf' <<__CONF__
+root-dir{share/cycle}=${JOB_HOST}=${JOB_HOST_RUN_ROOT}
+                     =*=\${ROSE_TEST_ROOT_DIR}
+root-dir{work}=${JOB_HOST}=${JOB_HOST_RUN_ROOT}
+              =*=\${ROSE_TEST_ROOT_DIR}
+
+[jinja2:suite.rc]
+JOB_HOST="${JOB_HOST}"
+__CONF__
+    tests 9
+else
+    tests 5
+fi
+
+# Run suite, create lots of directories
+export ROSE_CONF_PATH=
+export ROSE_TEST_ROOT_DIR="${PWD}/root.d"
+set -e
+rose suite-run --name="${NAME}" -q -C "${PWD}/source" --no-gcontrol \
+    --host='localhost' -- --debug
+# Prove that the directories exist before clean
+test -d "${HOME}/cylc-run/${NAME}"
+test -d "${PWD}/root.d/cylc-run/${NAME}"
+if [[ -n "${JOB_HOST}" && -n "${JOB_HOST_RUN_ROOT}" ]]; then
+    SSH='ssh -n -oBatchMode=yes'
+    ${SSH} "${JOB_HOST}" \
+        "bash -l -c 'ls -d cylc-run/${NAME} ${JOB_HOST_RUN_ROOT}/cylc-run/${NAME}'"
+fi
+set +e
+
+TEST_KEY="${TEST_KEY_BASE}"
+run_pass "${TEST_KEY}" rose suite-clean -y -v -v --debug "${NAME}"
+run_fail "${TEST_KEY}-test-d-cylc-run" test -d "${HOME}/cylc-run/${NAME}"
+run_fail "${TEST_KEY}-test-d-cylc-run-0" test -d "${HOME}/cylc-run/${NAME0}"
+run_fail "${TEST_KEY}-test-d-root-x" test -d "${PWD}/root.d/cylc-run/${NAME}"
+run_fail "${TEST_KEY}-test-d-root-x-0" test -d "${PWD}/root.d/cylc-run/${NAME0}"
+if [[ -n "${JOB_HOST}" && -n "${JOB_HOST_RUN_ROOT}" ]]; then
+    run_pass "${TEST_KEY}-test-d-cylc-run-at-${JOB_HOST}" ${SSH} "${JOB_HOST}" \
+        "bash -l -c '! test -d cylc-run/${NAME}'"
+    run_pass "${TEST_KEY}-test-d-cylc-run0-at-${JOB_HOST}" ${SSH} "${JOB_HOST}" \
+        "bash -l -c '! test -d cylc-run/${NAME0}'"
+    run_pass "${TEST_KEY}-test-d-root-at-${JOB_HOST}" ${SSH} "${JOB_HOST}" \
+        "bash -l -c '! test -d ${JOB_HOST_RUN_ROOT}/cylc-run/${NAME}'"
+    run_pass "${TEST_KEY}-test-d-root0-at-${JOB_HOST}" ${SSH} "${JOB_HOST}" \
+        "bash -l -c '! test -d ${JOB_HOST_RUN_ROOT}/cylc-run/${NAME0}'"
+fi
+#-------------------------------------------------------------------------------
+exit 0

t/rose-suite-clean/09-rmdir-2/app/t-1/rose-app.conf

@@ -0,0 +1,2 @@
+[command]
+default=echo 'Hello World' | tee 'hello.txt' "${ROSE_DATA}/hello.txt" "${ROSE_DATAC}/hello.txt"

t/rose-suite-clean/09-rmdir-2/rose-suite.conf

@@ -0,0 +1,2 @@
+root-dir{share/cycle}=*=$ROSE_TEST_ROOT_DIR
+root-dir{work}=*=$ROSE_TEST_ROOT_DIR