Feature/acd 4049 documentation for scenarios

tests/addons/TA_broken_sourcetype/default/eventtypes.conf

@@ -1,8 +1,12 @@
+# Component tested: eventtype
+# Scenario1: As the eventtype is disabled, no events will be present, the test case will fail.
+# Expected result: FAIL
 [is_splunkd]
 search = index=_internal sourcetype=splunkd\
 disabled = 1
-# As the eventtype is disabled, no events will be present, the test case will fail.
 
+# Component tested: eventtype
+# Scenario1: As "does not exist" is not present in any events, the test case will fail.
+# Expected result: FAIL
 [is_splunkd-%host%]
 search = index=_internal sourcetype=splunkd "does not exist"
-# As "does not exist" is not present in any events, the test case will fail.

tests/addons/TA_broken_sourcetype/default/props.conf

@@ -1,48 +1,83 @@
+# Component tested: sourcetype
+# Scenario: The sourcetype doesn't exist so the field will not be extracted
+# Expected result: FAIL
 [notvalid]
-## The sourcetype doesn't exist
 EXTRACT-one = group=(?<extractone>[^,]+)
 
+
 [splunkd]
-# As EVAL-two field is depend on EVAL-one field, two field events will be never present, the test case will fail for two field.
+# Component tested: EVAL
+# Scenario: As EVAL-two field is depend on EVAL-one field, two field events will be never present, the test case will fail for two field.
+# Expected result: FAIL
 EVAL-one = "working"
 EVAL-two = one
+
+# Component tested: EVAL
+# Scenario: source_field is used from EVAL and will not be available in splunk hence this extracting fields from it would fail.
+# Expected result: FAIL
 EVAL-used_in_extract = "use_this_in_extract"
 EXTRACT-incorrect_extract = (?<incorrect_extract>.*) in used_in_extract
-## source_field from EVAL hence this extract would fail
-EXTRACT-incorrect_extract = (?<incorrect_extract>.*) in used_in_extract
 
-# Without lookup name
+# Component tested: LOOKUP
+# Scenario: Without lookup name 
+# Expected result: FAIL
 LOOKUP-test_no_lookup = component OUTPUT context_test
 
-# Lookup does not exist 
+# Component tested: LOOKUP
+# Scenario: With non-existing lookup name  
+# Expected result: FAIL
 LOOKUP-test_nonexistent_lookup = Lookup_NAN component OUTPUT context_test
 
-# Lookup without input field
+# Component tested: LOOKUP
+# Scenario: Lookup without input field so the output_fields will not be there.
+# Expected result: FAIL
 LOOKUP-test_no_inputfield = ta_ficition_lookup OUTPUT context_test
 
-# Lookup output field doesn't exist
+# Component tested: LOOKUP
+# Scenario: non_existing output_field with output/outputnew so the tests for output_field will fail.
+# Expected result: FAIL
 LOOKUP-test_wrong_output = ta_ficition_lookup component output context_test
 LOOKUP-test_wrong_outputnew = ta_ficition_lookup component outputnew context_test
 
-# Empty lookup file
+# Component tested: LOOKUP
+# Scenario: Empty lookup file so no tests are generated for the output_fields.
+# Expected result: PASS
 LOOKUP-test_empty_csv = empty_lookup component context_test
 
-# Wrong lookup file + no output fields
+# Component tested: LOOKUP
+# Scenario: Wrong lookup file and no output fields
+# Expected result: PASS
 LOOKUP-test_lookup_not_found = NaN_lookup component context_test
 
+
 REPORT-tsc-delim-fields = tsc-delim-fields
 REPORT-tsc-sk-regex-format = tsc-sk-regex-format
 REPORT-tsc-sk-delim-format = contact_mode_extract
-# If a non_existing stanza is present then no testcases are generated for it
+# If a non_existing stanza is present then no testcases are generated for it.
 REPORT-tsc-regex-format = tsc-regex-format, non_existing_transforms_stanza
 
+
+# Component tested: FIELDALIAS
+# Scenario: Plugin searches for the original field and one or more alias field names.
+# Expected result: PASS
 FIELDALIAS-one = name AS aliasone
+
+# Component tested: FIELDALIAS
+# Scenario: Plugin searches for the original field and one or more alias field names.
+# Expected result: FAIL
 FIELDALIAS-two = nofield AS aliasone
 FIELDALIAS-three = one AS aliasthree
 
+
+# Component tested: source, sourcetypes
+# Expected result: FAIL
 [source::...notvalid...]
 sourcetype = notvalid
 
+# Component tested: source, sourcetype
+# The settings provided by the pattern [source::...none...] take
+# precedence over those provided by [source::...notvalid...], and sourcetype ends up
+# with "none" as its value.
+# Expected result: PASS
 [source::...none...]
 sourcetype = none
-

tests/addons/TA_broken_sourcetype/default/tags.conf

@@ -1,12 +1,20 @@
+# Component tested: eventtype
+# Scenario1: Checking for a random tag in a random event. Should fail.
+# Expected result: FAIL
 [RandomField=RandomValue]
 RandomTag = enabled
 # Checking for a random tag in a random event. Should fail.
 
+# Component tested: tags
+# Scenario1: # Adding a tag in every events with sourcetype=splunkd
+# Expected result: PASS
 [sourcetype=splunkd]
 tags_negative_testing = enabled
-# Adding a tag in every events with sourcetype=splunkd
 
+# Component tested: tags
+# Scenario1: Added a tag in every events with sourcetype=splunkd
+## source=/opt/splunk/var/log/splunk/splunkd.log has sourcetype=splunkd.
+## Hence the tag will be present, so test case will fail.
+# Expected result: FAIL
 [source=%2Fopt%2Fsplunk%2Fvar%2Flog%2Fsplunk%2Fsplunkd.log]
 tags_negative_testing = disabled
-# source=/opt/splunk/var/log/splunk/splunkd.log has sourcetype=splunkd.
-# Hence the tag will be present, so test case will fail.

tests/addons/TA_broken_sourcetype/default/transforms.conf

@@ -1,37 +1,51 @@
-# If Delim is not present in the field then the testcases will fail
+# Component tested: REPORT, DELIM-FIELDS
+# Scenario1: If Delim is not present in the events no field will be extracted and the testcases will fail
+# Expected result: FAIL
 [tsc-delim-fields]
 SOURCE_KEY = thread
 DELIMS = ","
 FIELDS = delim_field1, delim_field2
 
-# If SOURCE_KEY is not mentioned it will avoid generating testcase for it but for field `comp` it will generate testcases.
+# Component tested: REPORT, SOURCE_KEY, REGEX-FORMAT 
+# Scenario1: If SOURCE_KEY is not mentioned it will avoid generating testcase for it but for field `comp` it will generate testcases.
+# Expected result: PASS
 [tsc-sk-regex-format]
 SOURCE_KEY = 
 REGEX = (.+)
 FORMAT = comp::"$1"
 
-# If a non-existing source-key is provided the testcase should fail.
+# Component tested: REPORT, DELIM-FIELDS, SOURCE_KEY
+# Scenario1: If a non-existing source-key is provided the testcase should fail as no fields will be extracted.
+# Expected result: FAIL
 [contact_mode_extract]
 SOURCE_KEY = non_existing_field_as_source_key
 DELIMS = "="
 FIELDS = server_contact_mode, dest
 
-# Field3 is not captured by the regex then the test case should fail.
+# Component tested: REPORT, DELIM-FIELDS
+# Scenario1: An extra field is added into FORMAT Field3 is not captured by the regex then the test case should fail.
+# Expected result: FAIL
 [tsc-regex-format]
 REGEX =  (\w*)=(.*)
 FORMAT = field1::$1 field2::$2 field3::$3
 
-# Lookup to test the LOOKUP- test cases
+# Component tested: LOOKUP
+# Scenario1: Lookup to test the LOOKUP- test cases
+# Expected result: PASS
 [ta_ficition_lookup]
 filename = ta_fiction_splund_component.csv
 case_sensitive_match = false
 
-# Lookup file exist but it is empty 
+# Component tested: LOOKUP
+# Scenario1: Lookup file exist but it is empty So the tests will fails
+# Expected result: FAIL
 [empty_lookup]
 filename = empty.csv
 case_sensitive_match = false
 
-# The lookup does not exist 
+# Component tested: LOOKUP
+# Scenario1: Non existing Lookup file.
+# Expected result: FAIL
 [NaN_lookup]
 filename = NaN.csv
 case_sensitive_match = false

tests/addons/TA_fiction/default/eventtypes.conf

@@ -1,8 +1,21 @@
+# Component tested: eventtype
+# Scenario1: This scenario makes an eventtype is_splunkdbased on the search "index=_internal sourcetype=splunkd"
+# Expected outcome: PASS
 [is_splunkd]
 search = index=_internal sourcetype=splunkd
 
+# Component tested: tags
+# Scenario1: eventtype is applied to the events occurring under the search "sourcetype=splunkd" and this is further used to applied tags to the evevttype
+# Expected outcome: PASS
 [for_tags_positive]
 search = sourcetype=splunkd
 
+# Component tested: eventtype
+# Scenario1:# The following example makes an eventtype template because it includes a field name
+## surrounded by the percent character (in this case "%code%"). 
+## The value of "%code%" is substituted into the event type name for that event. 
+## For example, if the following example event type is instantiated on an event that has a
+## "code=432," it becomes "cisco-432".
+# Expected outcome: PASS
 [is_splunkd-%host%]
 search = index=_internal sourcetype=splunkd

tests/addons/TA_fiction/default/props.conf

@@ -1,50 +1,94 @@
+######## All the scenarios mentioned here are expected to PASS ########
 #Copyright Splunk
 [source::/opt/splunk/var/log/splunk/splunkd.log*]
 sourcetype = splunkd
 EVAL-component = "test"
 FIELDALIAS-nine =  extractone AS aliasthirteen
 
+# Component tested: source, EVAL
+# Scenario: Data must be present in the respective source=”/opt/splunk/var/log/splunk/splunkd.log*”
+# Scenario-EVAL: The plugin tests that the field generated from EVAL is present in both source and sourcetype
+## Plugin covers this by generating searches for the component along with source or sourcetypes.
+[source::/opt/splunk/var/log/splunk/splunkd.log*]
+EVAL-component = "test"
+
+# Component tested: source and sourcetype, EVAL
+# Scenario1: Sometimes sourcetype is assigned like this stanza so the plugin tests this by generating a combinatory query including both source and sourcetype.
+# Scenario2: | is used so the settings mentioned applies to two sources
+## source::/opt/splunk/var/log/splunk/metric.log*
+## source::/opt/splunk/var/log/splunk/health.log*
+## So the plugin handles this kind of scenarios by generating tests for each source.
+# Scenario-EVAL: The plugin tests that the field generated from EVAL is present in both source and sourcetype
+## Plugin covers this by generating searches for the component along with source and sourcetypes.
 [source::...(/opt/splunk/var/log/splunk/)(metrics.log*|health.log*)]
 sourcetype = splunkd
 EVAL-splunk_server = "server1"
 
-[splunkd]
 
-## "in" used as a regex and not as a keyword
+[splunkd]
+# Component tested: extract
+# Scenario: "in" used as a regex and not as a keyword so the plugin tests if the field `hotWarmPath` is extracted or not using search queries
 EXTRACT-0   = [i]n hotWarmPath=(?<hotWarmPath>.*)
 
-## single field extraction
+# Component tested: extract
+# Scenario: single field extraction so the plugin tests if the field `extractone` is extracted.
 EXTRACT-one = group=(?<extractone>[^,]+)
 
-## multiple field extractions
+# Component tested: extract
+# Scenario: multiple field extraction so the plugin generates multiple tests one individual for each extracted field 
+## and one test with combinatory query which tests all the fields must be extracted from a single event. 
 EXTRACT-two = group=(?<extracttwoA>[^,]+).*name=(?<extracttwoB>[^,]+)
 
-## use of "in" keyword to extract from source_field
+# Component tested: extract
+# Scenario: use of "in" or "IN" keyword to extract from source_field
+## tests the fields extracted as mentioned in above scenario extracting multiple fields.
+## tests that if source_field is present or not
+## and one test with combinatory query containing all the fields and source_field making sure that fields must be extracted from the source_field.
 EXTRACT-three = (?<hotWarmPath_in>.*) in hotWarmPath
-
-## use of "IN" keyword(case insensitivity) to extract from source_field
 EXTRACT-four = (?<hotWarmPath_IN>.*) IN hotWarmPath
 
-REPORT-g1 = ta_fiction_onefish
-REPORT-g2 = ta_fiction_twofish
-REPORT-g3 = ta_fiction_redfish, ta_fiction_bluefish
 
+# The explanation of the REPORT scenarios is explained in transforms.conf
+## Single transforms stanza associated with REPORT
 REPORT-tsc-delim-fields = tsc-delim-fields
 REPORT-tsc-sk-regex-format = tsc-sk-regex-format
 REPORT-tsc-sk-delim-format = tsc-sk-delim-format
-REPORT-tsc-regex = tsc-regex
-REPORT-tsc-regex-format = tsc-regex-format
 
+## multiple transforms stanza associated with REPORT
+REPORT-tsc-regex-format = tsc-regex, tsc-regex-format
+
+
+# Component tested: sourcetype, EVAL
+# Scenario: Data must be present in the respective sourcetype=splunkd
+# Scenario-EVAL: The plugin tests that the field generated from EVAL is present in both source and sourcetype
+## Plugin covers this by generating searches for the component along with source or sourcetypes.
 EVAL-myeval = "Working"
+
+# Component tested: FIELDALIAS
+# Scenario: Plugin searches for the original field and one or more alias field names.
 FIELDALIAS-one =  extractone AS aliasone
-FIELDALIAS-two = extractone AS aliastwo
-FIELDALIAS-three = extractone as aliasthree extractone as aliasfour
-FIELDALIAS-four = extractone AS aliasfive, extractone AS aliassix
+FIELDALIAS-two = extractone as aliastwo
+
+# Scenario: use of case-insensiive as/AS
+# Scenario: two fields separated by either " " or ","
+FIELDALIAS-three = extractone as aliasthree extractone AS aliasfour
+FIELDALIAS-four = extractone AS aliasfive, extractone as aliassix
+
+# Scenario: also covers case-insensitive asnew/ASNEW
+# Scenario: two fields separated by either " " or ","
 FIELDALIAS-five = extractone asnew aliasseven
 FIELDALIAS-six = extractone ASNEW aliaseight
-FIELDALIAS-seven = extractone asnew aliasnine extractone asnew aliasten
-FIELDALIAS-eight = extractone ASNEW aliaseleven, extractone ASNEW aliastwelve
+FIELDALIAS-seven = extractone asnew aliasnine extractone ASNEW aliasten
+FIELDALIAS-eight = extractone ASNEW aliaseleven, extractone asnew aliastwelve
+
 
+# Component tested: lookup
+# Scenario: The LOOKUP- prefix is actually case-insensitive. Acceptable variants include:
+##   LOOKUP_<class> = [...]
+##   LOOKUP<class>  = [...]
+##   lookup_<class> = [...]
+##   lookup<class>  = [...]
+## The plugin is designed to handle all such scenarios.
 LOOKUP_test_command_spelling_output1 = ta_ficition_lookup component OUTPUT context_test1
 LOOKUPtest_command_spelling_output2  = ta_ficition_lookup component OUTPUT context_test2
 LOOKUP-test_command_spelling_output3 = ta_ficition_lookup component OUTPUT context_test1
@@ -57,26 +101,57 @@ LOOKUP-test_command_spelling_outputnew3 = ta_ficition_lookup component OUTPUTNEW
 lookup_test_command_spelling_outputnew4 = ta_ficition_lookup component OUTPUTNEW context_test2
 lookuptest_command_spelling_outputnew5  = ta_ficition_lookup component OUTPUTNEW context_test1
 lookup-test_command_spelling_outputnew6  = ta_ficition_lookup component OUTPUTNEW context_test2
+
+# Component tested: lookup
+# Scenario: To test input_fields, output_fields exists or not.
+## If multiple output fields are present then the plugin tests presence of all the fielfs via search queries
+## Plugin generates multiple tests one individual for each field before and after OUTPUT keyword.
+## and one test with combinatory query which tests all the fields.
+## Plugin identifies the fields for all the below scenarios.
+
+# input_field name aliased as different field
 LOOKUP-test_as_input = ta_ficition_lookup test_name AS name OUTPUT context_test1
+
+# directly providing input_field_name and output_field is aliased
 LOOKUP-test_as_output = ta_ficition_lookup component OUTPUT context_test AS context_test_alternative
+
+# both input_field and output_field are aliased
 LOOKUP-test_as_input_output = ta_ficition_lookup test_name AS name OUTPUT context_test AS context_test_alternative2
+
+# multiple output fields are aliased.
 LOOKUP-test_as_input_output_multiple = ta_ficition_lookup test_name AS name OUTPUT context_test AS context_test_alternative3, status_test AS status2
+
+# Component tested: lookup-outputnew
+# Scenario: To test input_fields, output_fields exists or not.
+## If multiple output fields are present then the plugin tests presence of all the fielfd via search queries
+## Plugin generates multiple tests one individual for each field before and after OUTPUTNEW keyword.
+## and one test with combinatory query which tests all the fields.
 LOOKUP-test_as_input_outputnew_multiple = ta_ficition_lookup test_name AS name OUTPUTNEW context_test AS context_test_alternative4 status_test AS status2
+
+# Component tested: lookup "as" keyword is case-insensitive
+# Scenario: To test input_fields, output_fields exists or not with combinations of "as" and "AS" keywords.
+## If multiple output fields are present then the plugin tests presence of all the fielfd via search queries
+## Plugin generates multiple tests one individual for each field before and after OUTPUTNEW keyword.
+## and one test with combinatory query which tests all the fields.
 LOOKUP-test_AS_and_as_keyword = ta_ficition_lookup test_name as name OUTPUT context_test AS context_test_alternative5
-LOOKUP-test_as_keyword = ta_ficition_lookup test_name as name OUTPUT context_test1 status_test as status2
 LOOKUP-test_AS_keyword = ta_ficition_lookup test_name AS name OUTPUT context_test2 status_test as status2
+LOOKUP-test_as_keyword = ta_ficition_lookup test_name as name OUTPUT context_test1 status_test as status2
 LOOKUP-test_AS_and_as_keyword = ta_ficition_lookup test_name AS name OUTPUT status_test as status2
+
+# Multiple input field and single output field
 LOOKUP-test_string_outputfield = ta_ficition_lookup component OUTPUTNEW status.test
+# Multiple input field and single output field
 LOOKUP-test_multiple_input = ta_ficition_lookup component, aliasone OUTPUT context_test2
+# Single input field and multiple output field with "_"
 LOOKUP-test_multiple_output = ta_ficition_lookup component OUTPUT context_test1, status2
+# Multiple input field and multiple output field
 LOOKUP-test_multiple_input_output = ta_ficition_lookup component, aliasone OUTPUT context_test1, status2
+# Single input field and multiple output field with OUTPUTNEW
 LOOKUP-test_multiple_outputnew = ta_ficition_lookup component OUTPUTNEW context_test2, status2, status_test as status2
+# Multiple input field and multiple output field with OUTPUTNEW
 LOOKUP-test_multiple_input_outputnew = ta_ficition_lookup component, aliasone OUTPUT context_test2, status2
-LOOKUP-test_no_output = ta_ficition_lookup component
-
 
-[source::/opt/splunk/var/log/splunk/splunkd.log*]
-EVAL-component = "test"
+# without OUTPUT/OUTPUTNEW param the plugin checks if the input_fields field exists or not
+# And tests are also generated for the output fields present in the csv file .
+LOOKUP-test_no_output = ta_ficition_lookup component
 
-[source::...(/opt/splunk/var/log/splunk/)(metrics.log*|health.log*)]
-EVAL-splunk_server = "server1"