diff --git a/pytest_splunk_addon/standard_lib/addon_basic.py b/pytest_splunk_addon/standard_lib/addon_basic.py index 5d5a1a24c..ea1e1b360 100644 --- a/pytest_splunk_addon/standard_lib/addon_basic.py +++ b/pytest_splunk_addon/standard_lib/addon_basic.py @@ -108,7 +108,7 @@ def test_props_fields( assert result @pytest.mark.splunk_addon_searchtime - def test_props_fields_no_dash( + def test_props_fields_no_dash_not_empty( self, splunk_search_util, splunk_app_fields, record_property, caplog ): """ @@ -136,60 +136,21 @@ def test_props_fields_no_dash( ) op = "" for f in splunk_app_fields["fields"]: - search = search + f' {op} {f}="-"' + search = search + f' {op} {f}="-" OR {f}=""' op = "OR" search = search + ")" self.logger.debug(f"Executing the search query: {search}") # run search - result = splunk_search_util.checkQueryCountIsGreaterThanZero( - search, interval=INTERVAL, retries=RETRIES - ) - record_property("search", search) - - assert not result - - @pytest.mark.splunk_addon_searchtime - def test_props_fields_no_empty( - self, splunk_search_util, splunk_app_fields, record_property, caplog - ): - """ - Test case to check props property mentioned in props.conf - - This test case checks negative scenario for field blank value. - Args: - splunk_search_util(SearchUtil): - Object that helps to search on Splunk. - splunk_app_fields(fixture): - Test for stanza field. - record_property(fixture): - Document facts of test cases. - caplog : - fixture to capture logs. - """ - record_property("stanza_name", splunk_app_fields["stanza_name"]) - record_property("stanza_type", splunk_app_fields["stanza_type"]) - record_property("fields", splunk_app_fields["fields"]) - - search = ( - f"search (index=_internal OR index=*)" - f" {splunk_app_fields['stanza_type']}=" - f"{splunk_app_fields['stanza_name']} AND (" - ) - op = "" - for f in splunk_app_fields["fields"]: - search = search + f' {op} {f}=""' - op = "OR" - search = search + ")" + result, results = splunk_search_util.checkQueryCountIsZero(search) - self.logger.debug(f"Executing the search query: {search}") - # run search - result = splunk_search_util.checkQueryCountIsGreaterThanZero( - search, interval=INTERVAL, retries=RETRIES - ) + if not result: + record_property("results", results.as_list) + pp = pprint.PrettyPrinter(indent=4) + pp.pprint(results.as_list[-10:]) record_property("search", search) + assert result - assert not result # This test will check if there is at least one event with specified tags @pytest.mark.splunk_addon_searchtime diff --git a/tests/constants.py b/tests/constants.py new file mode 100644 index 000000000..5d02e04fb --- /dev/null +++ b/tests/constants.py @@ -0,0 +1,302 @@ +# -*- coding: utf-8 -*- +""" +Define the TA_fiction add-on passed test case list. +""" +TA_FICTION_PASSED = [ + "*test_splunk_app_fiction.py::Test_App::test_splunk_internal_errors PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_stanza*source::/opt/splunk/var/log/splunk/splunkd.log* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_stanza*source::...(/opt/splunk/var/log/splunk/)(metrics.log*|health.log*)* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_stanza*sourcetype::splunkd* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*/opt/splunk/var/log/splunk/splunkd.log*_field::component* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*/opt/splunk/var/log/splunk/metrics.log*_field::splunk_server* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*/opt/splunk/var/log/splunk/health.log*_field::splunk_server* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::hotWarmPath* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::EXTRACT-0* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::extractone* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::EXTRACT-one* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::extracttwoA* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::extracttwoB* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::EXTRACT-two* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::hotWarmPath_in* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::EXTRACT-three* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::hotWarmPath_IN* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::EXTRACT-four* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::reportone* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::REPORT-g1::ta_fiction_onefish* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::reporttwofishA* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::reporttwofishB* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::REPORT-g2::ta_fiction_twofish* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::reportred* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::REPORT-g3::ta_fiction_redfish* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::reportblueA* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::reportblueB* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::REPORT-g3::ta_fiction_bluefish* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::day_id* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::event_id* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::end_time* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::start_time* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::REPORT-tsc-delim-fields::tsc-delim-fields* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::component* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::comp* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::REPORT-tsc-sk-regex-format::tsc-sk-regex-format* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::server_contact_mode* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::dest* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::REPORT-tsc-sk-delim-format::tsc-sk-delim-format* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::REPORT-tsc-regex::tsc-regex* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::size1* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::size2* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::REPORT-tsc-regex-format::tsc-regex-format* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::myeval* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::aliasone* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::aliastwo* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::aliasthree* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::aliasfour* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::aliasfive* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::aliassix* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::aliasseven* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::aliaseight* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::aliasnine* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::aliasten* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::aliaseleven* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::aliastwelve* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::context_test1* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::LOOKUP_test_command_spelling_output1* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::context_test2* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::LOOKUPtest_command_spelling_output2* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::LOOKUP-test_command_spelling_output3* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::lookup_test_command_spelling_output4* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::lookuptest_command_spelling_output5* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::lookup-test_command_spelling_output6* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::LOOKUP_test_command_spelling_outputnew* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::LOOKUPtest_command_spelling_outputnew2* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::LOOKUP-test_command_spelling_outputnew3* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::lookup_test_command_spelling_outputnew4* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::lookuptest_command_spelling_outputnew5* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::lookup-test_command_spelling_outputnew6* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::name* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::LOOKUP-test_as_input* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::context_test_alternative* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::LOOKUP-test_as_output* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::context_test_alternative2* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::LOOKUP-test_as_input_output* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::context_test_alternative3* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::status2* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::LOOKUP-test_as_input_output_multiple* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::context_test_alternative4* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::LOOKUP-test_as_input_outputnew_multiple* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::LOOKUP-test_AS_and_as_keyword* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::LOOKUP-test_as_keyword* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::LOOKUP-test_AS_keyword* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::status.test* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::LOOKUP-test_string_outputfield* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::LOOKUP-test_multiple_input* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::LOOKUP-test_multiple_output* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::LOOKUP-test_multiple_input_output* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::LOOKUP-test_multiple_outputnew* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::LOOKUP-test_multiple_input_outputnew* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::context_test* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::status_test* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::test_name* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd_field::type* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::LOOKUP-test_no_output* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*/opt/splunk/var/log/splunk/splunkd.log*_field::component* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*/opt/splunk/var/log/splunk/metrics.log*_field::splunk_server* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*/opt/splunk/var/log/splunk/health.log*_field::splunk_server* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::hotWarmPath* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::EXTRACT-0* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::extractone* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::EXTRACT-one* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::extracttwoA* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::extracttwoB* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::EXTRACT-two* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::hotWarmPath_in* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::EXTRACT-three* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::hotWarmPath_IN* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::EXTRACT-four* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::reportone* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::REPORT-g1::ta_fiction_onefish* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::reporttwofishA* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::reporttwofishB* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::REPORT-g2::ta_fiction_twofish* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::reportred* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::REPORT-g3::ta_fiction_redfish* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::reportblueA* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::reportblueB* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::REPORT-g3::ta_fiction_bluefish* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::day_id* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::event_id* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::end_time* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::start_time* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::REPORT-tsc-delim-fields::tsc-delim-fields* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::component* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::comp* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::REPORT-tsc-sk-regex-format::tsc-sk-regex-format* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::server_contact_mode* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::dest* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::REPORT-tsc-sk-delim-format::tsc-sk-delim-format* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::REPORT-tsc-regex::tsc-regex* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::size1* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::size2* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::REPORT-tsc-regex-format::tsc-regex-format* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::myeval* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::aliasone* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::aliastwo* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::aliasthree* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::aliasfour* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::aliassix* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::aliasfive* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::aliasseven* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::aliaseight* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::aliasnine* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::aliasten* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::aliaseleven* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::aliastwelve* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::context_test1* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP_test_command_spelling_output1* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::context_test2* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUPtest_command_spelling_output2* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_command_spelling_output3* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::lookup_test_command_spelling_output4* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::lookuptest_command_spelling_output5* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::lookup-test_command_spelling_output6* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP_test_command_spelling_outputnew* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUPtest_command_spelling_outputnew2* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_command_spelling_outputnew3* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::lookup_test_command_spelling_outputnew4* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::lookuptest_command_spelling_outputnew5* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::lookup-test_command_spelling_outputnew6* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::name* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_as_input* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::context_test_alternative* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_as_output* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::context_test_alternative2* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_as_input_output* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::context_test_alternative3* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::status2* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_as_input_output_multiple* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::context_test_alternative4* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_as_input_outputnew_multiple* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_AS_and_as_keyword* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_as_keyword* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_AS_keyword* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::status.test* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_string_outputfield* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_multiple_input* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_multiple_output* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_multiple_input_output* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_multiple_outputnew* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_multiple_input_outputnew* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::context_test* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::status_test* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::test_name* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::type* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_no_output* PASSED*", + '*test_splunk_app_fiction.py::Test_App::test_tags*eventtype="for_tags_positive" | tags_positive_event=enabled* PASSED*', + '*test_splunk_app_fiction.py::Test_App::test_tags*eventtype="for_tags_positive" | tags_disabled_event=disabled* PASSED*', + '*test_splunk_app_fiction.py::Test_App::test_tags*source="/opt/splunk/var/log/splunk/splunkd.log" | tags_positive_event=enabled* PASSED*', + '*test_splunk_app_fiction.py::Test_App::test_tags*source="/opt/splunk/var/log/splunk/splunkd.log" | tags_disabled_event=disabled* PASSED*', + "*test_splunk_app_fiction.py::Test_App::test_eventtype*eventtype::is_splunkd* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_eventtype*eventtype::for_tags_positive* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_eventtype*eventtype::is_splunkd-%host%* PASSED*" +] + + +""" +Define the TA_broken_sourcetype add-on passed test case list. +""" +TA_BROKEN_SOURCETYPE_PASSED = [ + "*test_splunk_app_broken_sourcetype.py::Test_App::test_splunk_internal_errors PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_stanza*sourcetype::splunkd* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::one* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::used_in_extract* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::context_test* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::component* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd::LOOKUP-test_nonexistent_lookup* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd::LOOKUP-test_empty_csv* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd::LOOKUP-test_lookup_not_found* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::thread* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::comp* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd::REPORT-tsc-sk-regex-format::tsc-sk-regex-format* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::field1* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::field2* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::name* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*notvalid_field::extractone* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*notvalid::EXTRACT-one* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::one* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::two* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::used_in_extract* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::incorrect_extract* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::EXTRACT-incorrect_extract* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::OUTPUT* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::context_test* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_no_lookup* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::component* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_nonexistent_lookup* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_no_inputfield* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::output* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_wrong_output* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::outputnew* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_wrong_outputnew* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_empty_csv* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::LOOKUP-test_lookup_not_found* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::thread* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::delim_field1* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::delim_field2* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::REPORT-tsc-delim-fields::tsc-delim-fields* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::comp* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::REPORT-tsc-sk-regex-format::tsc-sk-regex-format* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::non_existing_field_as_source_key* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::server_contact_mode* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::dest* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::REPORT-tsc-sk-delim-format::contact_mode_extract* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::field1* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::field2* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::field3* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::REPORT-tsc-regex-format::tsc-regex-format* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::aliasone* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::name* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::nofield* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*splunkd_field::aliasthree* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*notvalid*::notvalid* PASSED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields_no_dash_not_empty*none*::none* PASSED*", + '*test_splunk_app_broken_sourcetype.py::Test_App::test_tags*sourcetype="splunkd" | tags_negative_testing=enabled* PASSED*', +] + +""" +Define the TA_broken_sourcetype add-on failed test case list. +""" +TA_BROKEN_SOURCETYPE_FAILED = [ + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_stanza*sourcetype::notvalid* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_stanza*source::...notvalid...* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_stanza*source::...none...* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*notvalid_field::extractone* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*notvalid::EXTRACT-one* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::two* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::incorrect_extract* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd::EXTRACT-incorrect_extract* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::OUTPUT* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd::LOOKUP-test_no_lookup* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd::LOOKUP-test_no_inputfield* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::output* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd::LOOKUP-test_wrong_output* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::outputnew* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd::LOOKUP-test_wrong_outputnew* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::delim_field1* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::delim_field2* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd::REPORT-tsc-delim-fields::tsc-delim-fields* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::non_existing_field_as_source_key* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::server_contact_mode* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::dest* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd::REPORT-tsc-sk-delim-format::contact_mode_extract* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::field3* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd::REPORT-tsc-regex-format::tsc-regex-format* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::aliasone* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::nofield* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*splunkd_field::aliasthree* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*notvalid*::notvalid* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_props_fields*none*::none* FAILED*", + '*test_splunk_app_broken_sourcetype.py::Test_App::test_tags*RandomField="RandomValue" | RandomTag=enabled* FAILED*', + '*test_splunk_app_broken_sourcetype.py::Test_App::test_tags*source="/opt/splunk/var/log/splunk/splunkd.log" | tags_negative_testing=disabled* FAILED*', + "*test_splunk_app_broken_sourcetype.py::Test_App::test_eventtype*eventtype::is_splunkd* FAILED*", + "*test_splunk_app_broken_sourcetype.py::Test_App::test_eventtype*eventtype::is_splunkd-%host%* FAILED*", +] diff --git a/tests/test_splunk_addon.py b/tests/test_splunk_addon.py index 370f6dd6e..4981dbb2e 100644 --- a/tests/test_splunk_addon.py +++ b/tests/test_splunk_addon.py @@ -3,6 +3,7 @@ import shutil import logging import pytest +from tests import constants logger = logging.getLogger("test_pytest_splunk_addon") @@ -136,7 +137,9 @@ def empty_method(): "\n".join(result.stdout.lines), "\n".join(result.stderr.lines), ) - result.assert_outcomes(passed=294, failed=0) + + result.stdout.fnmatch_lines_random(constants.TA_FICTION_PASSED) + result.assert_outcomes(passed=len(constants.TA_FICTION_PASSED), failed=0) # make sure that that we get a '0' exit code for the testsuite assert result.ret == 0 @@ -157,9 +160,7 @@ def empty_method(): ) shutil.copytree( - os.path.join( - testdir.request.fspath.dirname, "addons/TA_broken_sourcetype" - ), + os.path.join(testdir.request.fspath.dirname, "addons/TA_broken_sourcetype"), os.path.join(testdir.tmpdir, "tests/package"), ) @@ -186,7 +187,13 @@ def empty_method(): "\n".join(result.stdout.lines), "\n".join(result.stderr.lines), ) - result.assert_outcomes(passed=99, failed=31) + result.stdout.fnmatch_lines_random( + constants.TA_BROKEN_SOURCETYPE_PASSED + constants.TA_BROKEN_SOURCETYPE_FAILED + ) + result.assert_outcomes( + passed=len(constants.TA_BROKEN_SOURCETYPE_PASSED), + failed=len(constants.TA_BROKEN_SOURCETYPE_FAILED), + ) # The test suite should fail as this is a negative test assert result.ret != 0