Add Travis check for YAML_TEST_DATA in yaml schedule

Makefile

@@ -98,7 +98,7 @@ test-spec:
 	tools/update_spec --check
 
 .PHONY: test-static
-test-static: tidy-check test-yaml-valid test-modules-in-yaml-schedule test-merge test-dry test-no-wait_idle test-deleted-renamed-referenced-modules test-deleted-renamed-testdata detect-nonexistent-testdata test-unused-modules test-soft_failure-no-reference test-spec test-invalid-syntax
+test-static: tidy-check test-yaml-valid test-modules-in-yaml-schedule test-merge test-dry test-no-wait_idle test-deleted-renamed-referenced-files detect-nonexistent-testdata test-unused-modules test-soft_failure-no-reference test-spec test-invalid-syntax
 .PHONY: test
 ifeq ($(TESTS),compile)
 test: test-compile
@@ -120,17 +120,13 @@ perlcritic: tools/lib/
 test-unused-modules:
 	tools/detect_unused_modules
 
-.PHONY: test-deleted-renamed-referenced-modules
-test-deleted-renamed-referenced-modules:
-	tools/test_deleted_renamed_referenced_modules `git diff --name-only --exit-code --diff-filter=DR $$(git merge-base master HEAD) | grep '^tests/*'`
-
-.PHONY: test-deleted-renamed-testdata
-test-deleted-renamed-testdata:
-	tools/test_deleted_renamed_testdata `git diff --name-only --exit-code --diff-filter=DR $$(git merge-base master HEAD) | grep '^test_data/*'`
+.PHONY: test-deleted-renamed-referenced-files
+test-deleted-renamed-referenced-files:
+	tools/test_deleted_renamed_referenced_files `git diff --name-only --exit-code --diff-filter=DR $$(git merge-base master HEAD) | grep '^test*'`
 
 .PHONY: detect-nonexistent-testdata
 detect-nonexistent-testdata:
-	tools/detect_nonexistent_testdata `git diff --exit-code $$(git merge-base master HEAD) | grep '+  $include: test_data/' | grep 'yaml$$' | awk '{print $$3}'`
+	export PERL5LIB=${PERL5LIB_} ; tools/detect_nonexistent_testdata `git diff --name-only --exit-code --diff-filter=d $$(git merge-base master HEAD) | grep '^schedule/*' | grep '\.ya\?ml$$'`
 
 .PHONY: test-soft_failure-no-reference
 test-soft_failure-no-reference:

tools/detect_nonexistent_testdata

@@ -1,14 +1,53 @@
-#!/bin/sh -e
-
-FILES="${@}"
-success=1
-for FILE in $FILES; do
-	if  ! [ -f "$FILE" ]; then
-		echo "Error! $FILE does not exist."
-		success=0
-	elif ! [ -s "$FILE" ]; then
-		echo "Warning! The test_data file $FILE is empty."
-	fi
-done
-[ $success = 1 ] && echo "SUCCESS" && exit 0
-exit 1
+#!/usr/bin/env perl
+
+# The script verifies if test data files included in yaml schedule exist in the repo.
+
+use strict;
+use warnings;
+use YAML::Tiny;
+use File::Basename;
+use Data::Dumper;
+
+=head2 get_testdata_filename
+
+  get_testdata_paths($schedule_file_path);
+
+Returns yaml test data file name(s) with path, in case yaml schedule includes any, either via $include tag or YAML_TEST_DATA variable.
+
+=cut
+sub get_data_files {
+    my ($schedule_file_path) = @_;
+    my $schedule             = YAML::Tiny::LoadFile($schedule_file_path);
+    my $include_tag          = '$include';
+    my @filename_list;
+    my $test_data = $schedule->{test_data};
+    my @filenames;
+    if (ref($test_data->{$include_tag}) eq 'ARRAY') {
+        @filenames = @{$test_data->{$include_tag}};
+    }
+    else {
+        @filenames = $test_data->{$include_tag};
+    }
+    if ($test_data && @filenames) {
+        foreach my $item (@filenames) {
+            push(@filename_list, $item);
+        }
+    }
+    my $datafile = $schedule->{vars}->{YAML_TEST_DATA};
+    push(@filename_list, $datafile) if $datafile;
+    return @filename_list;
+}
+
+my $success = 1;
+foreach my $schedule_file (@ARGV) {
+    my @data_filenames = get_data_files($schedule_file);
+    if (@data_filenames) {
+        foreach my $item (@data_filenames) {
+            unless (-f $item) {
+                $success = 0;
+                print "Failure: Test data file $item used in $schedule_file does not exist in the repo.\n";
+            }
+        }
+    }
+}
+$success == 1 ? exit 0 : exit 1;

tools/test_deleted_renamed_referenced_files

@@ -0,0 +1,27 @@
+#!/bin/sh -e
+
+FILES="${@}"
+success=1
+for FILE in $FILES; do
+    if [ -f "$FILE" ]; then
+        # if file exists it means it was renamed, and then original file name is retrieved by git log
+        FILE=$(git log --follow -p $FILE | grep 'rename from test' | awk '{print $3}')
+    fi
+    if [ -n "$(echo $FILE | grep '\.pm$')" ]; then
+    # In case file is a module, module name appears in scheduling files excluding 'tests/' and file extension
+    filename=$(echo $FILE | cut -f 2- -d '/' | cut -f 1 -d '.')
+    target_paths='schedule/ products/*/main.pm lib/main_common.pm'
+    # In case file is test_data yaml file, the filename given is the same as in schedule files
+    elif [ -n "$(echo $FILE | grep '\.ya\?ml$')" ]; then
+    filename="$FILE" 
+    target_paths='schedule/'
+    fi
+    if MATCHED_SCHEDULE_FILES="$(grep --recursive --ignore-case --files-with-matches "${filename}\b" $target_paths)"
+    then
+        echo "\"$filename\" was removed or renamed, but it is still used in: \
+        \n$MATCHED_SCHEDULE_FILES\n"
+        success=0
+    fi
+done
+[ $success = 1 ] && echo "SUCCESS" && exit 0
+exit 1