diff --git a/tests/addons/TA_broken_sourcetype/default/eventtypes.conf b/tests/addons/TA_broken_sourcetype/default/eventtypes.conf index 9418b34b2..f41461b92 100644 --- a/tests/addons/TA_broken_sourcetype/default/eventtypes.conf +++ b/tests/addons/TA_broken_sourcetype/default/eventtypes.conf @@ -1,8 +1,12 @@ +# Component tested: eventtype +# Scenario1: As the eventtype is disabled, no events will be present, the test case will fail. +# Expected result: FAIL [is_splunkd] search = index=_internal sourcetype=splunkd\ disabled = 1 -# As the eventtype is disabled, no events will be present, the test case will fail. +# Component tested: eventtype +# Scenario1: As "does not exist" is not present in any events, the test case will fail. +# Expected result: FAIL [is_splunkd-%host%] search = index=_internal sourcetype=splunkd "does not exist" -# As "does not exist" is not present in any events, the test case will fail. \ No newline at end of file diff --git a/tests/addons/TA_broken_sourcetype/default/props.conf b/tests/addons/TA_broken_sourcetype/default/props.conf index 60bd851f7..b60a485e6 100644 --- a/tests/addons/TA_broken_sourcetype/default/props.conf +++ b/tests/addons/TA_broken_sourcetype/default/props.conf @@ -1,48 +1,83 @@ +# Component tested: sourcetype +# Scenario: The sourcetype doesn't exist so the field will not be extracted +# Expected result: FAIL [notvalid] -## The sourcetype doesn't exist EXTRACT-one = group=(?[^,]+) + [splunkd] -# As EVAL-two field is depend on EVAL-one field, two field events will be never present, the test case will fail for two field. +# Component tested: EVAL +# Scenario: As EVAL-two field is depend on EVAL-one field, two field events will be never present, the test case will fail for two field. +# Expected result: FAIL EVAL-one = "working" EVAL-two = one + +# Component tested: EVAL +# Scenario: source_field is used from EVAL and will not be available in splunk hence this extracting fields from it would fail. +# Expected result: FAIL EVAL-used_in_extract = "use_this_in_extract" EXTRACT-incorrect_extract = (?.*) in used_in_extract -## source_field from EVAL hence this extract would fail -EXTRACT-incorrect_extract = (?.*) in used_in_extract -# Without lookup name +# Component tested: LOOKUP +# Scenario: Without lookup name +# Expected result: FAIL LOOKUP-test_no_lookup = component OUTPUT context_test -# Lookup does not exist +# Component tested: LOOKUP +# Scenario: With non-existing lookup name +# Expected result: FAIL LOOKUP-test_nonexistent_lookup = Lookup_NAN component OUTPUT context_test -# Lookup without input field +# Component tested: LOOKUP +# Scenario: Lookup without input field so the output_fields will not be there. +# Expected result: FAIL LOOKUP-test_no_inputfield = ta_ficition_lookup OUTPUT context_test -# Lookup output field doesn't exist +# Component tested: LOOKUP +# Scenario: non_existing output_field with output/outputnew so the tests for output_field will fail. +# Expected result: FAIL LOOKUP-test_wrong_output = ta_ficition_lookup component output context_test LOOKUP-test_wrong_outputnew = ta_ficition_lookup component outputnew context_test -# Empty lookup file +# Component tested: LOOKUP +# Scenario: Empty lookup file so no tests are generated for the output_fields. +# Expected result: PASS LOOKUP-test_empty_csv = empty_lookup component context_test -# Wrong lookup file + no output fields +# Component tested: LOOKUP +# Scenario: Wrong lookup file and no output fields +# Expected result: PASS LOOKUP-test_lookup_not_found = NaN_lookup component context_test + REPORT-tsc-delim-fields = tsc-delim-fields REPORT-tsc-sk-regex-format = tsc-sk-regex-format REPORT-tsc-sk-delim-format = contact_mode_extract -# If a non_existing stanza is present then no testcases are generated for it +# If a non_existing stanza is present then no testcases are generated for it. REPORT-tsc-regex-format = tsc-regex-format, non_existing_transforms_stanza + +# Component tested: FIELDALIAS +# Scenario: Plugin searches for the original field and one or more alias field names. +# Expected result: PASS FIELDALIAS-one = name AS aliasone + +# Component tested: FIELDALIAS +# Scenario: Plugin searches for the original field and one or more alias field names. +# Expected result: FAIL FIELDALIAS-two = nofield AS aliasone FIELDALIAS-three = one AS aliasthree + +# Component tested: source, sourcetypes +# Expected result: FAIL [source::...notvalid...] sourcetype = notvalid +# Component tested: source, sourcetype +# The settings provided by the pattern [source::...none...] take +# precedence over those provided by [source::...notvalid...], and sourcetype ends up +# with "none" as its value. +# Expected result: PASS [source::...none...] sourcetype = none - diff --git a/tests/addons/TA_broken_sourcetype/default/tags.conf b/tests/addons/TA_broken_sourcetype/default/tags.conf index 27d7451a9..3365bfaf2 100644 --- a/tests/addons/TA_broken_sourcetype/default/tags.conf +++ b/tests/addons/TA_broken_sourcetype/default/tags.conf @@ -1,12 +1,20 @@ +# Component tested: eventtype +# Scenario1: Checking for a random tag in a random event. Should fail. +# Expected result: FAIL [RandomField=RandomValue] RandomTag = enabled # Checking for a random tag in a random event. Should fail. +# Component tested: tags +# Scenario1: # Adding a tag in every events with sourcetype=splunkd +# Expected result: PASS [sourcetype=splunkd] tags_negative_testing = enabled -# Adding a tag in every events with sourcetype=splunkd +# Component tested: tags +# Scenario1: Added a tag in every events with sourcetype=splunkd +## source=/opt/splunk/var/log/splunk/splunkd.log has sourcetype=splunkd. +## Hence the tag will be present, so test case will fail. +# Expected result: FAIL [source=%2Fopt%2Fsplunk%2Fvar%2Flog%2Fsplunk%2Fsplunkd.log] tags_negative_testing = disabled -# source=/opt/splunk/var/log/splunk/splunkd.log has sourcetype=splunkd. -# Hence the tag will be present, so test case will fail. \ No newline at end of file diff --git a/tests/addons/TA_broken_sourcetype/default/transforms.conf b/tests/addons/TA_broken_sourcetype/default/transforms.conf index d9ed4b2be..b21eb6238 100644 --- a/tests/addons/TA_broken_sourcetype/default/transforms.conf +++ b/tests/addons/TA_broken_sourcetype/default/transforms.conf @@ -1,37 +1,51 @@ -# If Delim is not present in the field then the testcases will fail +# Component tested: REPORT, DELIM-FIELDS +# Scenario1: If Delim is not present in the events no field will be extracted and the testcases will fail +# Expected result: FAIL [tsc-delim-fields] SOURCE_KEY = thread DELIMS = "," FIELDS = delim_field1, delim_field2 -# If SOURCE_KEY is not mentioned it will avoid generating testcase for it but for field `comp` it will generate testcases. +# Component tested: REPORT, SOURCE_KEY, REGEX-FORMAT +# Scenario1: If SOURCE_KEY is not mentioned it will avoid generating testcase for it but for field `comp` it will generate testcases. +# Expected result: PASS [tsc-sk-regex-format] SOURCE_KEY = REGEX = (.+) FORMAT = comp::"$1" -# If a non-existing source-key is provided the testcase should fail. +# Component tested: REPORT, DELIM-FIELDS, SOURCE_KEY +# Scenario1: If a non-existing source-key is provided the testcase should fail as no fields will be extracted. +# Expected result: FAIL [contact_mode_extract] SOURCE_KEY = non_existing_field_as_source_key DELIMS = "=" FIELDS = server_contact_mode, dest -# Field3 is not captured by the regex then the test case should fail. +# Component tested: REPORT, DELIM-FIELDS +# Scenario1: An extra field is added into FORMAT Field3 is not captured by the regex then the test case should fail. +# Expected result: FAIL [tsc-regex-format] REGEX = (\w*)=(.*) FORMAT = field1::$1 field2::$2 field3::$3 -# Lookup to test the LOOKUP- test cases +# Component tested: LOOKUP +# Scenario1: Lookup to test the LOOKUP- test cases +# Expected result: PASS [ta_ficition_lookup] filename = ta_fiction_splund_component.csv case_sensitive_match = false -# Lookup file exist but it is empty +# Component tested: LOOKUP +# Scenario1: Lookup file exist but it is empty So the tests will fails +# Expected result: FAIL [empty_lookup] filename = empty.csv case_sensitive_match = false -# The lookup does not exist +# Component tested: LOOKUP +# Scenario1: Non existing Lookup file. +# Expected result: FAIL [NaN_lookup] filename = NaN.csv case_sensitive_match = false diff --git a/tests/addons/TA_fiction/default/eventtypes.conf b/tests/addons/TA_fiction/default/eventtypes.conf index e17527d17..d3cacd69a 100644 --- a/tests/addons/TA_fiction/default/eventtypes.conf +++ b/tests/addons/TA_fiction/default/eventtypes.conf @@ -1,8 +1,21 @@ +# Component tested: eventtype +# Scenario1: This scenario makes an eventtype is_splunkdbased on the search "index=_internal sourcetype=splunkd" +# Expected outcome: PASS [is_splunkd] search = index=_internal sourcetype=splunkd +# Component tested: tags +# Scenario1: eventtype is applied to the events occurring under the search "sourcetype=splunkd" and this is further used to applied tags to the evevttype +# Expected outcome: PASS [for_tags_positive] search = sourcetype=splunkd +# Component tested: eventtype +# Scenario1:# The following example makes an eventtype template because it includes a field name +## surrounded by the percent character (in this case "%code%"). +## The value of "%code%" is substituted into the event type name for that event. +## For example, if the following example event type is instantiated on an event that has a +## "code=432," it becomes "cisco-432". +# Expected outcome: PASS [is_splunkd-%host%] search = index=_internal sourcetype=splunkd diff --git a/tests/addons/TA_fiction/default/props.conf b/tests/addons/TA_fiction/default/props.conf index ce98aa9ae..13c1bcc2b 100644 --- a/tests/addons/TA_fiction/default/props.conf +++ b/tests/addons/TA_fiction/default/props.conf @@ -1,50 +1,94 @@ +######## All the scenarios mentioned here are expected to PASS ######## #Copyright Splunk [source::/opt/splunk/var/log/splunk/splunkd.log*] sourcetype = splunkd EVAL-component = "test" FIELDALIAS-nine = extractone AS aliasthirteen +# Component tested: source, EVAL +# Scenario: Data must be present in the respective source=”/opt/splunk/var/log/splunk/splunkd.log*” +# Scenario-EVAL: The plugin tests that the field generated from EVAL is present in both source and sourcetype +## Plugin covers this by generating searches for the component along with source or sourcetypes. +[source::/opt/splunk/var/log/splunk/splunkd.log*] +EVAL-component = "test" + +# Component tested: source and sourcetype, EVAL +# Scenario1: Sometimes sourcetype is assigned like this stanza so the plugin tests this by generating a combinatory query including both source and sourcetype. +# Scenario2: | is used so the settings mentioned applies to two sources +## source::/opt/splunk/var/log/splunk/metric.log* +## source::/opt/splunk/var/log/splunk/health.log* +## So the plugin handles this kind of scenarios by generating tests for each source. +# Scenario-EVAL: The plugin tests that the field generated from EVAL is present in both source and sourcetype +## Plugin covers this by generating searches for the component along with source and sourcetypes. [source::...(/opt/splunk/var/log/splunk/)(metrics.log*|health.log*)] sourcetype = splunkd EVAL-splunk_server = "server1" -[splunkd] -## "in" used as a regex and not as a keyword +[splunkd] +# Component tested: extract +# Scenario: "in" used as a regex and not as a keyword so the plugin tests if the field `hotWarmPath` is extracted or not using search queries EXTRACT-0 = [i]n hotWarmPath=(?.*) -## single field extraction +# Component tested: extract +# Scenario: single field extraction so the plugin tests if the field `extractone` is extracted. EXTRACT-one = group=(?[^,]+) -## multiple field extractions +# Component tested: extract +# Scenario: multiple field extraction so the plugin generates multiple tests one individual for each extracted field +## and one test with combinatory query which tests all the fields must be extracted from a single event. EXTRACT-two = group=(?[^,]+).*name=(?[^,]+) -## use of "in" keyword to extract from source_field +# Component tested: extract +# Scenario: use of "in" or "IN" keyword to extract from source_field +## tests the fields extracted as mentioned in above scenario extracting multiple fields. +## tests that if source_field is present or not +## and one test with combinatory query containing all the fields and source_field making sure that fields must be extracted from the source_field. EXTRACT-three = (?.*) in hotWarmPath - -## use of "IN" keyword(case insensitivity) to extract from source_field EXTRACT-four = (?.*) IN hotWarmPath -REPORT-g1 = ta_fiction_onefish -REPORT-g2 = ta_fiction_twofish -REPORT-g3 = ta_fiction_redfish, ta_fiction_bluefish +# The explanation of the REPORT scenarios is explained in transforms.conf +## Single transforms stanza associated with REPORT REPORT-tsc-delim-fields = tsc-delim-fields REPORT-tsc-sk-regex-format = tsc-sk-regex-format REPORT-tsc-sk-delim-format = tsc-sk-delim-format -REPORT-tsc-regex = tsc-regex -REPORT-tsc-regex-format = tsc-regex-format +## multiple transforms stanza associated with REPORT +REPORT-tsc-regex-format = tsc-regex, tsc-regex-format + + +# Component tested: sourcetype, EVAL +# Scenario: Data must be present in the respective sourcetype=splunkd +# Scenario-EVAL: The plugin tests that the field generated from EVAL is present in both source and sourcetype +## Plugin covers this by generating searches for the component along with source or sourcetypes. EVAL-myeval = "Working" + +# Component tested: FIELDALIAS +# Scenario: Plugin searches for the original field and one or more alias field names. FIELDALIAS-one = extractone AS aliasone -FIELDALIAS-two = extractone AS aliastwo -FIELDALIAS-three = extractone as aliasthree extractone as aliasfour -FIELDALIAS-four = extractone AS aliasfive, extractone AS aliassix +FIELDALIAS-two = extractone as aliastwo + +# Scenario: use of case-insensiive as/AS +# Scenario: two fields separated by either " " or "," +FIELDALIAS-three = extractone as aliasthree extractone AS aliasfour +FIELDALIAS-four = extractone AS aliasfive, extractone as aliassix + +# Scenario: also covers case-insensitive asnew/ASNEW +# Scenario: two fields separated by either " " or "," FIELDALIAS-five = extractone asnew aliasseven FIELDALIAS-six = extractone ASNEW aliaseight -FIELDALIAS-seven = extractone asnew aliasnine extractone asnew aliasten -FIELDALIAS-eight = extractone ASNEW aliaseleven, extractone ASNEW aliastwelve +FIELDALIAS-seven = extractone asnew aliasnine extractone ASNEW aliasten +FIELDALIAS-eight = extractone ASNEW aliaseleven, extractone asnew aliastwelve + +# Component tested: lookup +# Scenario: The LOOKUP- prefix is actually case-insensitive. Acceptable variants include: +## LOOKUP_ = [...] +## LOOKUP = [...] +## lookup_ = [...] +## lookup = [...] +## The plugin is designed to handle all such scenarios. LOOKUP_test_command_spelling_output1 = ta_ficition_lookup component OUTPUT context_test1 LOOKUPtest_command_spelling_output2 = ta_ficition_lookup component OUTPUT context_test2 LOOKUP-test_command_spelling_output3 = ta_ficition_lookup component OUTPUT context_test1 @@ -57,26 +101,57 @@ LOOKUP-test_command_spelling_outputnew3 = ta_ficition_lookup component OUTPUTNEW lookup_test_command_spelling_outputnew4 = ta_ficition_lookup component OUTPUTNEW context_test2 lookuptest_command_spelling_outputnew5 = ta_ficition_lookup component OUTPUTNEW context_test1 lookup-test_command_spelling_outputnew6 = ta_ficition_lookup component OUTPUTNEW context_test2 + +# Component tested: lookup +# Scenario: To test input_fields, output_fields exists or not. +## If multiple output fields are present then the plugin tests presence of all the fielfs via search queries +## Plugin generates multiple tests one individual for each field before and after OUTPUT keyword. +## and one test with combinatory query which tests all the fields. +## Plugin identifies the fields for all the below scenarios. + +# input_field name aliased as different field LOOKUP-test_as_input = ta_ficition_lookup test_name AS name OUTPUT context_test1 + +# directly providing input_field_name and output_field is aliased LOOKUP-test_as_output = ta_ficition_lookup component OUTPUT context_test AS context_test_alternative + +# both input_field and output_field are aliased LOOKUP-test_as_input_output = ta_ficition_lookup test_name AS name OUTPUT context_test AS context_test_alternative2 + +# multiple output fields are aliased. LOOKUP-test_as_input_output_multiple = ta_ficition_lookup test_name AS name OUTPUT context_test AS context_test_alternative3, status_test AS status2 + +# Component tested: lookup-outputnew +# Scenario: To test input_fields, output_fields exists or not. +## If multiple output fields are present then the plugin tests presence of all the fielfd via search queries +## Plugin generates multiple tests one individual for each field before and after OUTPUTNEW keyword. +## and one test with combinatory query which tests all the fields. LOOKUP-test_as_input_outputnew_multiple = ta_ficition_lookup test_name AS name OUTPUTNEW context_test AS context_test_alternative4 status_test AS status2 + +# Component tested: lookup "as" keyword is case-insensitive +# Scenario: To test input_fields, output_fields exists or not with combinations of "as" and "AS" keywords. +## If multiple output fields are present then the plugin tests presence of all the fielfd via search queries +## Plugin generates multiple tests one individual for each field before and after OUTPUTNEW keyword. +## and one test with combinatory query which tests all the fields. LOOKUP-test_AS_and_as_keyword = ta_ficition_lookup test_name as name OUTPUT context_test AS context_test_alternative5 -LOOKUP-test_as_keyword = ta_ficition_lookup test_name as name OUTPUT context_test1 status_test as status2 LOOKUP-test_AS_keyword = ta_ficition_lookup test_name AS name OUTPUT context_test2 status_test as status2 +LOOKUP-test_as_keyword = ta_ficition_lookup test_name as name OUTPUT context_test1 status_test as status2 LOOKUP-test_AS_and_as_keyword = ta_ficition_lookup test_name AS name OUTPUT status_test as status2 + +# Multiple input field and single output field LOOKUP-test_string_outputfield = ta_ficition_lookup component OUTPUTNEW status.test +# Multiple input field and single output field LOOKUP-test_multiple_input = ta_ficition_lookup component, aliasone OUTPUT context_test2 +# Single input field and multiple output field with "_" LOOKUP-test_multiple_output = ta_ficition_lookup component OUTPUT context_test1, status2 +# Multiple input field and multiple output field LOOKUP-test_multiple_input_output = ta_ficition_lookup component, aliasone OUTPUT context_test1, status2 +# Single input field and multiple output field with OUTPUTNEW LOOKUP-test_multiple_outputnew = ta_ficition_lookup component OUTPUTNEW context_test2, status2, status_test as status2 +# Multiple input field and multiple output field with OUTPUTNEW LOOKUP-test_multiple_input_outputnew = ta_ficition_lookup component, aliasone OUTPUT context_test2, status2 -LOOKUP-test_no_output = ta_ficition_lookup component - -[source::/opt/splunk/var/log/splunk/splunkd.log*] -EVAL-component = "test" +# without OUTPUT/OUTPUTNEW param the plugin checks if the input_fields field exists or not +# And tests are also generated for the output fields present in the csv file . +LOOKUP-test_no_output = ta_ficition_lookup component -[source::...(/opt/splunk/var/log/splunk/)(metrics.log*|health.log*)] -EVAL-splunk_server = "server1" diff --git a/tests/addons/TA_fiction/default/tags.conf b/tests/addons/TA_fiction/default/tags.conf index d3f9f8958..3f1a10c1b 100644 --- a/tests/addons/TA_fiction/default/tags.conf +++ b/tests/addons/TA_fiction/default/tags.conf @@ -1,11 +1,15 @@ +# Component tested: tags +# Scenario1: first tag tags_positive_event will be added to events with eventtype=for_tags_positive +# Scenario2: second tag tags_disabled_event will not be added to events with eventtype=for_tags_positive +## Plugin tests that each tag is applied with the respective field=value or regex mentioned in the stanza if enabled and vice versa. [eventtype=for_tags_positive] tags_positive_event = enabled tags_disabled_event = disabled -# first tag will be added to events with eventtype=for_tags_positive -# second tag will not be added to events with eventtype=for_tags_positive +# Component tested: tags +# Scenario1: It also supports url encoded strings +# first tag will be added to events with source=/opt/splunk/var/log/splunk/splunkd.log +# second tag will not be added to events with source=/opt/splunk/var/log/splunk/splunkd.log [source=%2Fopt%2Fsplunk%2Fvar%2Flog%2Fsplunk%2Fsplunkd.log] tags_positive_event = enabled tags_disabled_event = disabled -# first tag will be added to events with source=/opt/splunk/var/log/splunk/splunkd.log -# second tag will not be added to events with eventtype=for_tags_positive \ No newline at end of file diff --git a/tests/addons/TA_fiction/default/transforms.conf b/tests/addons/TA_fiction/default/transforms.conf index ef88ef81f..9bdcb7c8d 100644 --- a/tests/addons/TA_fiction/default/transforms.conf +++ b/tests/addons/TA_fiction/default/transforms.conf @@ -1,49 +1,45 @@ -[ta_fiction_onefish] -REGEX = group=(?[^,]+) - -[ta_fiction_twofish] -REGEX = group=(?[^,]+).*name=(?[^,]+) - -[ta_fiction_threefish] -REGEX = group=(?[^,]+).*name=(?[^,]+) - -[ta_fiction_redfish] -REGEX = group=(?[^,]+) - -[ta_fiction_bluefish] -REGEX = group=(?[^,]+).*name=(?[^,]+) - +# Component tested: LOOKUP, filename +# Scenario: provides the lookup filename and other related information. [ta_ficition_lookup] filename = ta_fiction_splund_component.csv case_sensitive_match = false -# Delim with field all the fields must be extracted and test scenarios will be -# An individual search for each field and a single searchof all the fields. + +# Component tested: REPORT, DELIM-FIELDS +# Scenario: multiple fields can be extracted using delim and fields parameter +## Similar to multiple field extraction in extract the plugin bevases the same to test the extraxted fields via FIELDS in REPORT +## all the fields must be extracted and test scenarios will be an individual search/test for each field and a single combinatory search comprising all the fields. [tsc-delim-fields] DELIMS = "," FIELDS = day_id, event_id, end_time, start_time -# Source-key with regex and format -# An individual search for each field and SOURCE_KEY and a single searchof all the fields with SOURCE_KEY. +# Component tested: REPORT, DELIM-FIELDS-SOURCE_KEY +# Scenario:# Similar to the above scenario but source-key with delim and fields. +## the fields are extracted from the SOURCE_KEY instead of +## An individual search for SOURCE_KEY and each field mentioned in FIELDS and a single search of all the fields with SOURCE_KEY. +[tsc-sk-delim-format] +SOURCE_KEY = event_id +DELIMS = "=" +FIELDS = server_contact_mode, dest + +# Component tested: REPORT, REGEX-FORMAT-SOURCE_KEY +# Scenario: Source-key with regex and format +## An individual search for SOURCE_KEY and each field extracted in FORMAT and a single search of all the fields with SOURCE_KEY. +## Similar to 'in' scenario in extract [tsc-sk-regex-format] SOURCE_KEY = component REGEX = (.+) FORMAT = comp::"$1" -# Similar to the above scenario but source-key with delim and fields. -# Single search for each field and a combinatory search for all fields. -[tsc-sk-delim-format] -SOURCE_KEY = event_id -DELIMS = "=" -FIELDS = server_contact_mode, dest +# Component tested: REPORT, REGEX-FORMAT +# Scenario: Similar to above just the regex is applied to _raw field if SOURCE_KEY is not mentioned +[tsc-regex-format] +REGEX = (\w*)=(.*) +FORMAT = size1::$1 size2::$2 -# Check for Named captured groups in Regex +# Component tested: REPORT, REGEX +# Scenario: Check for Named captured groups in Regex # Single search for each field and a combinatory search for all fields extracted from regex. [tsc-regex] REGEX = group=(?[^,]+) -# Check for named capturing grps in regex and the fields present in format -# Single search for each field and a combinatory search for all fields. -[tsc-regex-format] -REGEX = (\w*)=(.*) -FORMAT = size1::$1 size2::$2