[Bug]: CI bug failure

[shahzebsiddiqui]

Bug Description

This is the error result from the CI job https://github.com/buildtesters/buildtest/actions/runs/3161848912/jobs/5147874338.

The test fails periodically since we have tests not loaded in the buildspec cache since they are stored in non-standard locations

tests/cli/test_buildspec.py::test_buildspec_summary PASSED               [ 37%]
tests/cli/test_buildspec.py::test_buildspec_show PASSED                  [ 38%]
tests/cli/test_buildspec.py::test_buildspec_show_fail FAILED             [ 39%]

=================================== FAILURES ===================================
___________________________ test_buildspec_show_fail ___________________________

    @pytest.mark.cli
    def test_buildspec_show_fail():
    
        # Query some random test name that doesn't exist
        with pytest.raises(BuildTestError):
            random_testname = "".join(random.choices(string.ascii_letters, k=10))
            show_failed_buildspecs(
                configuration=configuration, test_names=[random_testname]
            )
    
        # Query a test that is NOT in state=FAIL
        with pytest.raises(BuildTestError):
            results = Report()
            pass_test = random.sample(results.get_test_by_state(state="PASS"), 1)
            show_failed_buildspecs(configuration=configuration, test_names=[pass_test])
    
        report = Report()
        # get a random failed test from report file to be used for showing content of buildspec file
        fail_tests = random.sample(report.get_test_by_state(state="FAIL"), 1)
        # running buildtest buildspec show-fail <test> --theme monokai
>       show_failed_buildspecs(
            configuration=configuration, test_names=fail_tests, theme="monokai"
        )

tests/cli/test_buildspec.py:346: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
buildtest/cli/buildspec.py:1116: in show_failed_buildspecs
    show_buildspecs(failed_tests, configuration, theme)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

test_names = ['slurm_down_nodes_reason']
configuration = <buildtest.config.SiteConfiguration object at 0x7f5dd4afb5e0>
theme = 'monokai'

    def show_buildspecs(test_names, configuration, theme=None):
        """This is the entry point for ``buildtest buildspec show`` command which will print content of
        buildspec based on name of test.
    
        Args:
            test_names (list): List of test names to show content of file
            configuration (buildtest.config.SiteConfiguration): Instance of SiteConfiguration class
            theme (str, optional): Color theme to choose. This is the Pygments style (https://pygments.org/docs/styles/#getting-a-list-of-available-styles) which is specified by ``--theme`` option
        """
        cache = BuildspecCache(configuration=configuration)
        theme = theme or "monokai"
    
        error = False
        visited = set()
        for name in test_names:
            if name not in cache.get_names():
    
                console.print(f"[red]Unable to find test {name} in cache")
                error = True
                continue
    
            buildspec = cache.lookup_buildspec_by_name(name)
            if buildspec not in visited:
                visited.add(buildspec)
    
                console.rule(buildspec)
                with open(buildspec) as fd:
                    syntax = Syntax(fd.read(), "yaml", theme=theme)
                console.print(syntax)
    
        if error:
>           raise BuildTestError(
                f"Please select one of the following test: {cache.get_names()}"
            )
E           buildtest.exceptions.BuildTestError: "Please select one of the following test: ['csh_shell', 'create_burst_buffer_executors', 'sleep', 'gcc_version', 'skip', 'unskipped', 'bash_env_variables', 'csh_env_declaration', 'tcsh_env_declaration', 'wrongexecutor', '_bin_sh_shell', '_bin_bash_shell', 'bash_shell', 'sh_shell', 'shell_options', 'circle_area', 'bash_login_shebang', 'bash_nonlogin_shebang', 'string_tag', 'list_of_strings_tags', 'metric_regex_example', 'foo_bar', 'python_hello', 'add_numbers', 'variables_bash', 'hello_world', 'summary_example', 'always_pass', 'always_fail', 'test_fail_returncode_match', 'test_pass_returncode_mismatch', 'exit1_fail', 'exit1_pass', 'returncode_list_mismatch', 'returncode_int_match', 'status_regex_pass', 'status_regex_fail', 'timelimit_min_max', 'timelimit_min', 'timelimit_max', 'timelimit_min_fail', 'timelimit_max_fail', 'status_returncode_by_executors', 'executors_sbatch_declaration', 'executors_vars_env_declaration', 'multiple_executors', 'runtime_test', 'runtime_test_pass', 'runtime_test_fail', 'test1', 'test2', 'test3', 'pass_test', 'fail_test', 'pass_and_fail_test', 'final_test', 'jobA', 'jobB', 'jobC', 'systemd_default_target', 'root_disk_usage', 'kernel_swapusage', 'ulimit_filelock_unlimited', 'ulimit_cputime_unlimited', 'ulimit_stacksize_unlimited', 'ulimit_vmsize_unlimited', 'ulimit_filedescriptor_4096', 'ulimit_max_user_process_2048', 'runImage', 'pullImage_dockerhub', 'pullImage_sylabscloud', 'pullImage_shub', 'inspect_image', 'build_sif_from_dockerimage', 'build_sandbox_image', 'build_remoteimages', 'pbs_sleep', 'show_host_groups', 'display_lsf_hosts', 'display_hosts_format', 'bhosts_version', 'show_lsf_queues', 'show_lsf_queues_formatted', 'show_lsf_queues_current_user', 'show_lsf_user_groups', 'show_lsf_configuration', 'show_lsf_models', 'show_lsf_resources', 'lsf_version', 'current_user_queue', 'show_all_jobs', 'nodes_state_down', 'nodes_state_reboot', 'nodes_state_allocated', 'nodes_state_completing', 'nodes_state_idle', 'node_down_fail_list_reason', 'dead_nodes', 'get_partitions', 'sinfo_version', 'show_accounts', 'show_users', 'show_qos', 'show_tres', 'slurm_config', 'show_partition', 'qsub_version', 'qselect_version', 'cqsub_version', 'qdel_version', 'qmove_version', 'show_jobs', 'show_queues']"

buildtest/cli/buildspec.py:1087: BuildTestError
----------------------------- Captured stdout call -----------------------------
kjJvplXlZe is not in one of the following failed test: ['sleep', 'sh_shell', 'exit1_fail', '_bin_sh_shell', 'shell_options', 'slurm_not_responding_nodes', 'fail_test', 'slurm_down_nodes_reason', 'tcsh_env_declaration', 'returncode_list_mismatch']
Unable to find test kjJvplXlZe in cache
['final_test'] is not in one of the following failed test: ['sleep', 'sh_shell', 'exit1_fail', '_bin_sh_shell', 'shell_options', 'slurm_not_responding_nodes', 'fail_test', 'slurm_down_nodes_reason', 'tcsh_env_declaration', 'returncode_list_mismatch']
Unable to find test ['final_test'] in cache
Unable to find test slurm_down_nodes_reason in cache
------------------------------ Captured log call -------------------------------
DEBUG    buildtest.cli.report:report.py:255 Loading report file: /home/runner/work/buildtest/buildtest/var/report.json
DEBUG    buildtest.cli.report:report.py:255 Loading report file: /home/runner/work/buildtest/buildtest/var/report.json
DEBUG    buildtest.cli.report:report.py:255 Loading report file: /home/runner/work/buildtest/buildtest/var/report.json
DEBUG    buildtest.cli.report:report.py:255 Loading report file: /home/runner/work/buildtest/buildtest/var/report.json
DEBUG    buildtest.cli.report:report.py:255 Loading report file: /home/runner/work/buildtest/buildtest/var/report.json
=============================== warnings summary ===============================
buildtest/executors/cobalt.py:150
  /home/runner/work/buildtest/buildtest/buildtest/executors/cobalt.py:150: DeprecationWarning: invalid escape sequence \d
    """This method is responsible for moving output and error file in the run

-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
============================= slowest 20 durations =============================
23.74s call     tests/cli/test_build.py::test_jobdeps
9.72s call     tests/cli/test_buildspec.py::test_func_buildspec_find
3.63s call     tests/cli/test_build.py::test_build_buildspecs
3.24s call     tests/cli/test_build.py::test_build_by_tags
3.21s call     tests/cli/test_build.py::test_timeout
3.07s call     tests/test_stats.py::test_stats
1.17s call     tests/cli/test_build.py::test_buildspec_tag_executor
0.97s call     tests/cli/test_build.py::test_build_csh_executor
0.67s call     tests/cli/test_build.py::test_build_filter_check
0.33s call     tests/cli/test_buildspec.py::test_buildspec_find_format
0.32s call     tests/cli/test_build.py::test_build_rebuild
0.31s call     tests/cli/test_build.py::test_build_rerun
0.17s call     tests/cli/test_build.py::test_build_executor_type
0.16s call     tests/buildsystem/test_base.py::test_BuildspecParser
0.15s call     tests/cli/test_buildspec.py::test_buildspec_summary
0.12s call     tests/cli/test_build.py::test_remove_stagedi
0.10s call     tests/cli/test_build.py::test_skip_field
0.09s call     tests/cli/test_buildspec.py::test_buildspec_validate
0.08s call     tests/cli/test_build.py::test_build_by_stages
0.04s call     tests/cli/test_buildspec.py::test_buildspec_find_terse
=========================== short test summary info ============================
SKIPPED [1] tests/test_ascent.py:18: This test must run on domain ascent.olcf.ornl.gov
SKIPPED [1] tests/test_ascent.py:53: This test must run on domain ascent.olcf.ornl.gov
SKIPPED [1] tests/test_cori.py:21: This test runs on Cori Login nodes ('cori*')
SKIPPED [1] tests/test_cori.py:49: This test runs on Cori Login nodes ('cori*')
SKIPPED [1] tests/test_cori.py:73: This test runs on Cori Login nodes ('cori*')
SKIPPED [1] tests/test_cori.py:99: This test runs on Cori Login nodes ('cori*')
SKIPPED [1] tests/test_cori.py:113: This test runs on Cori Login nodes ('cori*')
SKIPPED [1] tests/test_jlse.py:17: Test runs only on JLSE Login Nodes with domain name alcf.anl.gov
SKIPPED [1] tests/test_pbs.py:15: Test runs only on PBS Cluster, must have pbsnodes command
SKIPPED [1] tests/buildsystem/test_spack.py:16: Unable to run this test requires docker container:  ghcr.io/buildtesters/buildtest_spack:latest
XFAIL tests/test_exceptions.py::TestBuildTestError::test_exception
  Testing to see if exception of type BuildTestError is raised
XFAIL tests/test_exceptions.py::TestBuildTestError::test_multi_args
  Testing to see if exception of type BuildTestError is raised
FAILED tests/cli/test_buildspec.py::test_buildspec_show_fail - buildtest.exce...
!!!!!!!!!!!!!!!!!!!!!!!!!! stopping after 1 failures !!!!!!!!!!!!!!!!!!!!!!!!!!!
======= 1 failed, 35 passed, 10 skipped, 2 xfailed, 1 warning in 52.34s ========
Error: Process completed with exit code 1.

Version

devel

Relevant log output

No response

Post question in Slack

• I agree that I posted my question in slack before creating this issue

Is there an existing issue

• I confirm there is no existing issue for this issue