Merge remote-tracking branch 'origin/topic/etyp/recurse-cfg-upwards'

* origin/topic/etyp/recurse-cfg-upwards:
  Allow executing tests from outside the suite
  Check if current directory has tests before error
  Check parent directories for `btest.cfg`

Author: evantypanski

CHANGES

@@ -1,3 +1,40 @@
+1.2-15 | 2025-04-30 10:11:52 -0400
+
+  * Allow executing tests from outside the suite (Evan Typanski, Corelight)
+
+    If btest.cfg is in the `testing/btest` relative directory, this allows
+    the user to call btest with `btest testing/btest/tests`, which will use
+    the btest.cfg file in testing/btest. It will also check its parents, so
+    you may use `btest testing/btest/language/while.zeek` - for example.
+
+    The first complication is that previously-valid uses of btest should
+    remain the same. This is done by always checking the current directory
+    for btest.cfg first.
+
+    The second is if there is a conflict between two tests' btest.cfg file.
+    For example, the following may choose two different btest.cfg files:
+
+    .
+    └── multiple-cfg
+        ├── Baseline
+        ├── btest.cfg
+        └── tests
+            ├── no-cfg
+            │   └── test2.test
+            └── with-cfg
+                ├── btest.cfg
+                └── tests
+                    └── test1.test
+
+    If there is a conflicting config file, then btest should raise an error,
+    as it can currently only execute with one config file per run.
+
+  * Check if current directory has tests before erroring (Evan Typanski, Corelight)
+
+    After this, btest can execute tests relative to the current directory.
+
+  * Check parent directories for `btest.cfg` (Evan Typanski, Corelight)
+
 1.2-11 | 2025-04-28 11:42:33 +0200
 
   * Enable lints around return statements (Benjamin Bannier, Corelight)

README

@@ -2,7 +2,7 @@
 ..
 .. Version number is filled in automatically.
 
-.. |version| replace:: 1.2-11
+.. |version| replace:: 1.2-15
 
 ==================================================
 BTest - A Generic Driver for Powerful System Tests
@@ -414,10 +414,11 @@ Configuration
 -------------
 
 Specifics of ``btest``'s execution can be tuned with a configuration
-file, which by default is ``btest.cfg`` if that's found in the
-current directory. It can alternatively be specified with the
-``--config`` command line option, or a ``BTEST_CFG`` environment
-variable. The configuration file is
+file, which by default is ``btest.cfg`` if that's found in the current
+directory or parent directories. It can alternatively be specified
+with the ``--config`` command line option (which will not search
+parent directories), or a ``BTEST_CFG`` environment variable.
+The configuration file is
 "INI-style", and an example comes with the distribution, see
 ``btest.cfg.example``. A configuration file has one main section,
 ``btest``, that defines most options; as well as an optional section

VERSION

@@ -1 +1 @@
-1.2-11
+1.2-15

btest

@@ -56,15 +56,14 @@ else:
     import multiprocessing.managers as mp_managers
     import multiprocessing.sharedctypes as mp_sharedctypes
 
-VERSION = "1.2-11"  # Automatically filled in.
+VERSION = "1.2-15"  # Automatically filled in.
 
 Name = "btest"
 Config = None
 
-try:
-    ConfigDefault = os.environ["BTEST_CFG"]
-except KeyError:
-    ConfigDefault = "btest.cfg"
+DEFAULT_CONFIG_NAME = "btest.cfg"
+
+ConfigDefault = os.environ.get("BTEST_CFG", DEFAULT_CONFIG_NAME)
 
 
 def normalize_path(path):
@@ -2443,7 +2442,7 @@ class LinuxTimer(TimerBase):
 
 
 # Walk the given directory and return all test files.
-def findTests(paths, expand_globs=False):
+def findTests(paths, *, cwd=None, expand_globs=False):
     tests = []
 
     ignore_files = getOption("IgnoreFiles", "").split()
@@ -2452,14 +2451,20 @@ def findTests(paths, expand_globs=False):
     expanded = set()
 
     for p in paths:
-        p = os.path.join(TestBase, p)
+        anchored = os.path.join(TestBase, p)
 
         if expand_globs:
-            for d in glob.glob(p):
+            for d in glob.glob(anchored):
                 if os.path.isdir(d):
                     expanded.add(d)
         else:
-            expanded.add(p)
+            # Allow relative directories if this one does not exist
+            if cwd and not os.path.exists(anchored):
+                from_cwd = cwd / p
+                if from_cwd.exists():
+                    anchored = str(from_cwd)
+
+            expanded.add(anchored)
 
     for path in expanded:
         rpath = os.path.relpath(path, TestBase)
@@ -2705,6 +2710,27 @@ def outputDocumentation(tests, fmt):
             print()
 
 
+# Finds the config file 'filename', recursing to parent directories from the
+# 'start_dir' if not found.
+def find_config_file(filename, *, start_dir, recurse):
+    # First check current dir
+    f = pathlib.Path(filename)
+    if f.is_file():
+        return str(f.absolute())
+
+    if not recurse:
+        return None
+
+    current_dir = start_dir.absolute()
+
+    for d in [current_dir, *current_dir.parents]:
+        f = d / filename
+        if f.is_file():
+            return str(f)
+
+    return None
+
+
 def parse_options():
     optparser = optparse.OptionParser(
         usage="%prog [options] <directories>", version=VERSION
@@ -2947,9 +2973,6 @@ def parse_options():
         warning("ignoring requested parallelism in interactive-update mode")
         options.threads = 1
 
-    if not os.path.exists(options.config):
-        error(f"configuration file '{options.config}' not found")
-
     return options, parsed_args
 
 
@@ -2988,18 +3011,52 @@ if __name__ == "__main__":
 
     (Options, args) = parse_options()
 
+    found_configs = set()
+
+    if args:
+        for arg in args:
+            if found_config := find_config_file(
+                Options.config,
+                start_dir=pathlib.Path(arg).absolute(),
+                recurse=Options.config == DEFAULT_CONFIG_NAME,
+            ):
+                found_configs.add(found_config)
+    else:
+        if found_config := find_config_file(
+            Options.config,
+            start_dir=pathlib.Path.cwd(),
+            recurse=Options.config == DEFAULT_CONFIG_NAME,
+        ):
+            found_configs.add(found_config)
+
+    if len(found_configs) == 0:
+        error(f"configuration file '{Options.config}' not found")
+
+    if len(found_configs) > 1:
+        conflicting_configs = ", ".join(
+            sorted(f"'{config}'" for config in found_configs)
+        )
+        error(
+            f"cannot execute tests using different configuration files together: {conflicting_configs}"
+        )
+
+    btest_cfg = found_configs.pop()
+
+    dirname = os.path.dirname(btest_cfg)
+
+    # Special case for providing just a single path to btest that happens
+    # to be the TestBase. Reset args, assuming the user wants to run all tests.
+    if len(args) == 1 and os.path.abspath(args[0]) == dirname:
+        args = []
+
     # The defaults come from environment variables, plus a few additional items.
     defaults = {}
     # Changes to defaults should not change os.environ
     defaults.update(os.environ)
     defaults["default_path"] = os.environ["PATH"]
 
-    dirname = os.path.dirname(Options.config)
-    if not dirname:
-        dirname = os.getcwd()
-
     # If the BTEST_TEST_BASE envirnoment var is set, we'll use that as the testbase.
-    # If not, we'll use the current directory.
+    # If not, we'll use the config file's directory.
     TestBase = normalize_path(os.environ.get("BTEST_TEST_BASE", dirname))
     defaults["testbase"] = TestBase
     defaults["baselinedir"] = normalize_path(
@@ -3009,7 +3066,7 @@ if __name__ == "__main__":
 
     # Parse our config
     Config = getcfgparser(defaults)
-    Config.read(Options.config, encoding="utf-8")
+    Config.read(btest_cfg, encoding="utf-8")
 
     defaults["baselinedir"] = getOption("BaselineDir", defaults["baselinedir"])
 
@@ -3041,7 +3098,7 @@ if __name__ == "__main__":
                     # At this point, our defaults have changed, so we
                     # reread the configuration.
                     new_config = getcfgparser(defaults)
-                    new_config.read(Options.config)
+                    new_config.read(btest_cfg)
                     return new_config, value
 
             return Config, default
@@ -3063,6 +3120,7 @@ if __name__ == "__main__":
             transform=lambda x: normalize_path(x),
         )
 
+    orig_cwd = pathlib.Path.cwd()
     os.chdir(TestBase)
 
     if Options.sphinx:
@@ -3198,13 +3256,13 @@ if __name__ == "__main__":
 
     testdirs = getOption("TestDirs", "").split()
     if testdirs:
-        Config.configured_tests = findTests(testdirs, True)
+        Config.configured_tests = findTests(testdirs, expand_globs=True)
 
     if args:
         if Options.tests_file:
             error("cannot specify tests both on command line and with --tests-file")
 
-        tests = findTests(args)
+        tests = findTests(args, cwd=orig_cwd)
 
     else:
         if Options.rerun:

setup.py

@@ -5,6 +5,6 @@
 py_modules = ["btest-sphinx"]
 
 setup(
-    version="1.2.dev11",  # Filled in automatically.
+    version="1.2.dev15",  # Filled in automatically.
     py_modules=py_modules,
 )

testing/Baseline/tests.btest-cfg/nopath

@@ -5,4 +5,4 @@ btest-cfg ... ok
 all 1 tests successful
 btest-cfg ... ok
 all 1 tests successful
-configuration file 'btest.cfg' not found
+configuration file 'nonexistant' not found

testing/Baseline/tests.multiple-cfg-fail/output

@@ -0,0 +1,13 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+## RUNNING NO CFG TEST ALONE
+tests.no-cfg.no-cfg ...
+  > exit 0
+... tests.no-cfg.no-cfg ok
+all 1 tests successful
+## RUNNING WITH CFG TEST ALONE
+tests.with-cfg ...
+  > exit 0
+... tests.with-cfg ok
+all 1 tests successful
+## RUNNING TESTS TOGETHER
+cannot execute tests using different configuration files together: '<...>/btest.cfg', '<...>/btest.cfg'

testing/Baseline/tests.outside-btest-dir/output

@@ -0,0 +1,17 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+## RUNNING DIRECTLY ON DIRECTORY WITH CONFIG:
+tests.one.a-relative-test ...
+  > exit 0
+... tests.one.a-relative-test ok
+tests.two.another-relative-test ...
+  > exit 0
+... tests.two.another-relative-test ok
+all 2 tests successful
+## RUNNING DIRECTLY ON TESTS DIRECTORY:
+tests.one.a-relative-test ...
+  > exit 0
+... tests.one.a-relative-test ok
+tests.two.another-relative-test ...
+  > exit 0
+... tests.two.another-relative-test ok
+all 2 tests successful

testing/Scripts/diff-remove-abspath

@@ -2,4 +2,8 @@
 #
 # Replace absolute paths with the basename.
 
-sed 's#[a-zA-Z:]*/\([^/]\{1,\}/\)\{1,\}\([^/]\{1,\}\)#<...>/\2#g'
+# The drive letter portion of the Windows regex below is adapted from
+# https://github.com/stdlib-js/stdlib/blob/develop/lib/node_modules/%40stdlib/regexp/basename-windows/lib/regexp.js
+sed -E 's#/+#/#g' |
+    sed -E 's#([a-zA-Z]:|[\\/]{2}[^\\/]+[\\/]+[^\\/]+)([\\/])([^	 :\\/]{1,}[\\/]){1,}([^	 :\\/]{1,})#<...>/\4#g' |
+    sed -E 's#/([^	 :/]{1,}/){1,}([^	 :/]{1,})#<...>/\2#g'

testing/btest.tests.cfg

@@ -10,6 +10,7 @@ override=normal
 [btest]
 TmpDir      = `echo .tmp`
 BaselineDir = %(testbase)s/Baseline
+TestDirs    = tests
 
 [environment]
 ORIGPATH=%(default_path)s

testing/tests/btest-cfg.test

@@ -2,7 +2,7 @@
 # %TEST-EXEC: btest -c myfile %INPUT > nopath 2>&1
 # %TEST-EXEC: BTEST_CFG=myfile btest %INPUT >> nopath 2>&1
 # %TEST-EXEC: BTEST_CFG=notexist btest -c myfile %INPUT >> nopath 2>&1
-# %TEST-EXEC-FAIL: btest %INPUT >> nopath 2>&1
+# %TEST-EXEC-FAIL: btest -c nonexistant %INPUT >> nopath 2>&1
 # %TEST-EXEC: btest-diff nopath
 # %TEST-EXEC: mkdir z
 # %TEST-EXEC: mv myfile z/btest.cfg

testing/tests/multiple-cfg-fail.test

@@ -0,0 +1,20 @@
+# %TEST-DOC: Test that you can run btest from outside of the directory with btest.cfg
+#
+# %TEST-EXEC: mkdir -p base/tests/no-cfg
+# %TEST-EXEC: mkdir -p base/tests/with-cfg/tests
+# %TEST-EXEC: cat %INPUT >> base/tests/no-cfg/no-cfg.test
+# %TEST-EXEC: mv btest.cfg base/
+# %TEST-EXEC: echo "## RUNNING NO CFG TEST ALONE" >>output
+# %TEST-EXEC: btest -v base/tests/no-cfg/no-cfg.test >>output 2>&1
+#
+# %TEST-EXEC: cp base/btest.cfg base/tests/with-cfg
+# %TEST-EXEC: cat %INPUT >> base/tests/with-cfg/tests/with-cfg.test
+# %TEST-EXEC: echo "## RUNNING WITH CFG TEST ALONE" >>output
+# %TEST-EXEC: btest -v base/tests/with-cfg/tests/with-cfg.test >>output 2>&1
+#
+# But fail together
+# %TEST-EXEC: echo "## RUNNING TESTS TOGETHER" >>output
+# %TEST-EXEC-FAIL: btest -v base/tests/no-cfg/no-cfg.test base/tests/with-cfg/tests/with-cfg.test >>output 2>&1
+# %TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output
+
+@TEST-EXEC: exit 0

testing/tests/outside-btest-dir.test

@@ -0,0 +1,18 @@
+# %TEST-DOC: Test that you can run btest from outside of the directory with btest.cfg
+#
+# %TEST-EXEC: mkdir -p base/tests/one
+# %TEST-EXEC: mkdir -p base/tests/two
+# %TEST-EXEC: cat %INPUT >> base/tests/one/a-relative-test.test
+# %TEST-EXEC: cat %INPUT >> base/tests/two/another-relative-test.test
+# %TEST-EXEC: mv btest.cfg base/
+# %TEST-EXEC: btest base/tests/one/a-relative-test.test
+# %TEST-EXEC: btest base/tests/one/a-relative-test.test base/tests/two/another-relative-test.test
+#
+# Test the special case of running all tests from outside
+# %TEST-EXEC: echo "## RUNNING DIRECTLY ON DIRECTORY WITH CONFIG:" >>output
+# %TEST-EXEC: btest -v base/ >>output 2>&1
+# %TEST-EXEC: echo "## RUNNING DIRECTLY ON TESTS DIRECTORY:" >>output
+# %TEST-EXEC: btest -v base/tests >>output 2>&1
+# %TEST-EXEC: btest-diff output
+
+@TEST-EXEC: exit 0

testing/tests/resolve-from-cwd.test

@@ -0,0 +1,8 @@
+# %TEST-DOC: Ensure btest can find tests from a path relative to the current directory
+#
+# %TEST-EXEC: mkdir -p my/relative/dir
+# %TEST-EXEC: cat %INPUT >> my/relative/dir/a-relative-test.test
+# %TEST-EXEC: cd my/relative/dir && btest a-relative-test.test
+# %TEST-EXEC: cd my/relative && btest dir/a-relative-test.test
+
+@TEST-EXEC: exit 0