make_tests.pl

#!/usr/bin/perl
#
#  make_tests.pl - generate WPT test cases from the testable statements wiki
#
#  This script assumes that a wiki has testable statement entries
#  in the format described by the specification at
#  https://spec-ops.github.io/atta-api/index.html
#
#  usage: make_tests.pl -f file | -w wiki_title | -s spec -d dir

use strict;

use IO::String ;
use JSON ;
use MediaWiki::API ;
use Getopt::Long;

my %specs = (
    "aria11" => {
      title => "ARIA_1.1_Testable_Statements",
      specURL => "https://www.w3.org/TR/wai-aria11/",
      dir => "aria11"
    },
    "svg" => {
      title => "SVG_Accessibility/Testing/Test_Assertions_with_Tables_for_ATTA",
      specURL => "https://www.w3.org/TR/svg-aam-1.0/",
      dir => "svg",
      fragment => '<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">%code%</svg>'
    }
);

my @apiNames = qw(UIA MSAA ATK IAccessible2 AXAPI);
my $apiNamesRegex = "(" . join("|", @apiNames) . ")";

# the suffix to attach to the automatically generated test case names
my $theSuffix = "-manual.html";

# dir is determined based upon the short name of the spec and is defined
# by the input or on the command line

my $file = undef ;
my $spec = undef ;
my $wiki_title = undef ;
my $dir = undef;
my $theSpecFragment = "%code%";
my $preserveWiki = "";
my $fake = 0;

my $result = GetOptions(
    "f|file=s"   => \$file,
    "p=s" => \$preserveWiki,
    "w|wiki=s"   => \$wiki_title,
    "s|spec=s"   => \$spec,
    "f|fake"    => \$fake,
    "d|dir=s"   => \$dir) || usage();

my $wiki_config = {
  "api_url" => "https://www.w3.org/wiki/api.php"
};

my $io ;
our $theSpecURL = "";

if ($spec) {
  print "Processing spec $spec\n";
  $wiki_title = $specs{$spec}->{title};
  $theSpecURL = $specs{$spec}->{specURL};
  if (!$dir) {
    $dir = "../" . $specs{$spec}->{dir};
  }
  $theSpecFragment = $specs{$spec}->{fragment};
}

if (!$dir) {
  $dir = "../raw";
}

if (!-d $dir) {
  print STDERR "No such directory: $dir\n";
  exit 1;
}

if ($file) {
  open($io, "<", $file) || die("Failed to open $file: " . $@);
} elsif ($wiki_title) {
  my $MW = MediaWiki::API->new( $wiki_config );

  $MW->{config}->{on_error} = \&on_error;

  sub on_error {
    print "Error code: " . $MW->{error}->{code} . "\n";
    print $MW->{error}->{stacktrace}."\n";
    die;
  }
  my $page = $MW->get_page( { title => $wiki_title } );
  my $theContent = $page->{'*'};
  print "Loaded " . length($theContent) . " from $wiki_title\n";
  if ($preserveWiki) {
    if (open(OUTPUT, ">$preserveWiki")) {
      print OUTPUT $theContent;
      close OUTPUT;
      print "Wiki preserved in $preserveWiki\n";
      exit 0;
    } else {
      print "Failed to create $preserveWiki. Terminating.\n";
      exit 1;
    }
  }
  $io = IO::String->new($theContent);
} else {
  usage() ;
}



# Now let's walk through the content and build a test page for every item
#

# iterate over the content

# my $io ;
# open($io, "<", "raw") ;

# data structure:
#
# steps is a list of steps to be performed.
# Each step is an object that has a type property and other properties based upon that type.
#
# Types include:
#
# 'test' - has a property for each ATAPI for which there are tests
# 'attribute' - has a property for the target id, attribute name, and value
# 'event' - has a property for the target id and event name
my $state = 0;   # between items
my $theStep = undef;
my $current = "";
my $theCode = "";
my $theAttributes = {};
my @steps ;
my $theAsserts = {} ;
my $theAssertCount = 0;
my $theAPI = "";
my $typeRows = 0;
my $theType = "";
my $theName = "";
my $theRef = "";
my $lineCounter = 0;
my $skipping = 0;

our $testNames = {} ;

while (<$io>) {
  if (m/<!-- END OF TESTS -->/) {
    last;
  }
  $lineCounter++;
  # look for state
  if (m/^SpecURL: (.*)$/) {
    $theSpecURL = $1;
    $theSpecURL =~ s/^ *//;
    $theSpecURL =~ s/ *$//;
  }
  if ($state == 5 && m/^; \/\/ (.*)/) {
    # we found another test inside a block
    # we were in an item; dump it
    build_test($current, $theAttributes, $theCode, \@steps, $theSpecFragment) ;
    # print "Finished $current and new subblock $1\n";
    $state = 1;
    $theAttributes = {} ;
    $theAPI = "";
    @steps = ();
    $theCode = "";
    $theAsserts = undef;
    $theName = "";
  } elsif (m/^=== +(.*[^ ]) +===/) {
    if ($state != 0) {
      if ($skipping) {
        print STDERR "Flag on assertion $current; skipping\n";
      } else {
        # we were in an item; dump it
        build_test($current, $theAttributes, $theCode, \@steps, $theSpecFragment) ;
        # print "Finished $current\n";
      }
    }
    $state = 1;
    $current = $1;
    $theAttributes = {} ;
    @steps = ();
    $theCode = "";
    $theAsserts = undef;
    $theAPI = "";
    $theName = "";
    if ($current =~ m/\(/) {
      # there is a paren in the name -skip it
      $skipping = 1;
    } else {
      $skipping = 0;
    }
  }

  if ($state == 1) {
    if (m/<pre>/) {
      # we are now in the code block
      $state = 2;
      next;
    } elsif (m/==== +(.*) +====/) {
      # we are in some other block
      $theName = lc($1);
      $theAttributes->{$theName} = "";
      next;
    }
    if (m/^Reference: +(.*)$/) {
      $theAttributes->{reference} = $theSpecURL . "#" . $1;
    } elsif ($theName ne "") {
      # accumulate whatever was in the block under the data for it
      chomp();
      $theAttributes->{$theName} .= $_;
    } elsif (m/TODO/) {
      $state = 0;
    }
  }

  if ($state == 2) {
    if (m/<\/pre>/) {
      # we are done with the code block
      $state = 3;
    } else  {
      if (m/^\s/ && !m/if given/) {
        # trim any trailing whitespace
        $theCode =~ s/ +$//;
        $theCode =~ s/\t/ /g;
        $theCode .= $_;
        # In MediaWiki, to display & symbol escapes as literal text, one
        # must use "&amp;&" for the "&" character. We need to undo that.
        $theCode =~ s/&amp;(\S)/&$1/g;
      }
    }
  } elsif ($state == 3) {
    # look for a table
    if (m/^\{\|/) {
      # table started
      $state = 4;
    }
  } elsif ($state == 4) {
    if (m/^\|-/) {
      if ($theAPI
        && exists($theAsserts->{$theAPI}->[$theAssertCount])
        && scalar(@{$theAsserts->{$theAPI}->[$theAssertCount]})) {
        $theAssertCount++;
      }
      # start of a table row
      if ($theType ne "" && $typeRows) {
        # print qq($theType typeRows was $typeRows\n);
        # we are still processing items for a type
        $typeRows--;
        # populate the first cell
        $theAsserts->{$theAPI}->[$theAssertCount] = [ $theType ] ;
      } else {
        $theType = "";
      }
    } elsif (m/^\|\}/) {
      # ran out of table
      $state = 5;
    # adding processing for additional block types
    # a colspan followed by a keyword triggers a start
    # so |colspan=5|element triggers a new collection
    # |colspan=5|attribute triggers the setting of an attribute
    } elsif (m/^\|colspan="*([0-9])"*\|([^ ]+) (.*)$/) {
      my $type = $2;
      my $params = $3;

      my $obj = {} ;
      if ($type eq "attribute") {
        if ($params =~ m/([^:]+):([^ ]+) +(.*)$/) {
          $obj = {
            type => $type,
            element => $1,
            attribute => $2,
            value => $3
          };
          $theStep = undef;
          push(@steps, $obj);
        } else {
          print STDERR "Malformed attribute instruction at line $lineCounter: " . $_ . "\n";
        }
      } elsif ($type eq "event") {
        if ($params =~ m/([^:]+):([^ ]+).*$/) {
          $obj = {
            type => $type,
            element => $1,
            value => $2
          };
          $theStep = undef;
          push(@steps, $obj);
        } else {
          print STDERR "Malformed event instruction at line $lineCounter: " . $_ . "\n";
        }
      } elsif ($type eq "element") {
        $obj = {
          type => "test",
          element => $3
        };
        push(@steps, $obj);
        $theStep = scalar(@steps) - 1;
        $theAsserts = $steps[$theStep];
      } else {
        print STDERR "Unknown operation type: $type at line " . $lineCounter . "; skipping.\n";
      }
    } elsif (m/($apiNamesRegex)$/) {
      my $theString = $1;
      $theString =~ s/ +$//;
      $theString =~ s/^ +//;
      if ($theString eq "IA2") {
        $theString = "IAccessible2" ;
      }
      my $rows = 1;
      if (m/^\|rowspan="*([0-9])"*\|(.*)$/) {
        $rows = $1
      }
      if (grep { $_ eq $theString } @apiNames) {
        # we found an API name - were we already processing assertions?
        if (!$theAsserts) {
          # nope - now what?
          $theAsserts = {
            type => "test",
            element => "test"
          };
          push(@steps, $theAsserts);
        }
        $theAssertCount = 0;
        # this is a new API section
        $theAPI = $theString ;
        $theAsserts->{$theAPI} = [ [] ] ;
        $theType = "";
      } else {
        # this is a multi-row type
        $theType = $theString;
        $typeRows = $rows;
        # print qq(Found multi-row $theString for $theAPI with $typeRows rows\n);
        $typeRows--;
        # populate the first cell
        if ($theAPI
          && exists($theAsserts->{$theAPI}->[$theAssertCount])
          && scalar(@{$theAsserts->{$theAPI}->[$theAssertCount]})) {
          $theAssertCount++;
        }
        $theAsserts->{$theAPI}->[$theAssertCount] = [ $theType ] ;
      }
    } elsif (m/^\|(.*)$/) {
      my $item = $1;
      $item =~ s/^ *//;
      $item =~ s/ *$//;
      $item =~ s/^['"]//;
      $item =~ s/['"]$//;
      # add into the data structure for the API
      if (!exists $theAsserts->{$theAPI}->[$theAssertCount]) {
        $theAsserts->{$theAPI}->[$theAssertCount] = [ $item ] ;
      } else {
        push(@{$theAsserts->{$theAPI}->[$theAssertCount]}, $item);
      }
    }
  }
};

if ($state != 0) {
  build_test($current, $theAttributes, $theCode, \@steps, $theSpecFragment) ;
  print "Finished $current\n";
}

exit 0;

# build_test
#
# create a test file
#
# attempts to create unique test names

sub build_test() {
  my $title = shift ;
  my $attrs = shift ;
  my $code = shift ;
  my $steps = shift;
  my $frag = shift ;

  if ($title eq "") {
    print STDERR "No name provided!";
    return;
  }

  if ($frag ne "") {
    $frag =~ s/%code%/$code/;
    $code = $frag;
  }

  $code =~ s/ +$//m;
  $code =~ s/\t/ /g;

  my $title_reference = $title;

  if ($code eq "") {
    print STDERR "No code for $title; skipping.\n";
    return;
  }
  if ( $steps eq {}) {
    print STDERR "No assertions for $title; skipping.\n";
    return;
  }

  my $testDef =
  { "title" => $title,
    "steps" => []
  };
  my $stepCount = 0;
  foreach my $asserts (@$steps) {
    $stepCount++;
    my $step =
      {
        "type" => $asserts->{"type"},
        "title"=> "step " . $stepCount,
      };

    if ($asserts->{type} eq "test") {
      # everything in the block is about testing an element
      $step->{"element"} = ( $asserts->{"element"} || "test" );

      my $tests = {};
      if ($fake) {
        $tests->{"WAIFAKE"} = [ [ "property", "role", "is", "ROLE_TABLE_CELL" ], [ "property", "interfaces", "contains", "TableCell" ] ];
      }
      foreach my $name (@apiNames) {
        if (exists $asserts->{$name} && scalar(@{$asserts->{$name}})) {
          $tests->{$name} = $asserts->{$name};
        }
      };

      $step->{test} = $tests;

    } elsif ($asserts->{type} eq "attribute") {
      $step->{type} = "attribute";
      $step->{element} = $asserts->{"element"};
      $step->{attribute} = $asserts->{"attribute"};
      $step->{value} = $asserts->{value};
    } elsif ($asserts->{type} eq "event") {
      $step->{type} = "event";
      $step->{element} = $asserts->{"element"};
      $step->{event} = $asserts->{value};
    } else {
      print STDERR "Invalid step type: " . $asserts->{type} . "\n";
      next;
    }
    push(@{$testDef->{steps}}, $step);
  }


  # populate the rest of the test definition

  if (scalar(keys(%$attrs))) {
    while (my $key = each(%$attrs)) {
      # print "Copying $key \n";
      $testDef->{$key} = $attrs->{$key};
    }
  }

  if (exists $attrs->{reference}) {
    $title_reference = "<a href='" . $attrs->{reference} . "'>" . $title_reference . "</a>" ;
  }

  my $testDef_json = to_json($testDef, { canonical => 1, pretty => 1, utf8 => 1});

  my $fileName = $title;
  $fileName =~ s/\s*$//;
  $fileName =~ s/\///g;
  $fileName =~ s/\s+/_/g;
  $fileName =~ s/[,=:]/_/g;
  $fileName =~ s/['"]//g;

  my $count = 2;
  if ($testNames->{$fileName}) {
    while (exists $testNames->{$fileName . "_$count"}) {
      $count++;
    }
    $fileName .= "_$count";
  }

  $fileName = lc($fileName);

  $testNames->{$fileName} = 1;

  $fileName .= $theSuffix;

  my $template = qq(<!doctype html>
<html>
  <head>
    <title>$title</title>
    <meta content="text/html; charset=utf-8" http-equiv="Content-Type"/>
    <link rel="stylesheet" href="/wai-aria/scripts/manual.css">
    <script src="/resources/testharness.js"></script>
    <script src="/resources/testharnessreport.js"></script>
    <script src="/wai-aria/scripts/ATTAcomm.js"></script>
    <script>
    setup({explicit_timeout: true, explicit_done: true });

    var theTest = new ATTAcomm(
    $testDef_json
    ) ;
    </script>
  </head>
  <body>
  <p>This test examines the ARIA properties for $title_reference.</p>
  $code
  <div id="manualMode"></div>
  <div id="log"></div>
  <div id="ATTAmessages"></div>
  </body>
</html>);

  my $file ;

  if (open($file, ">", "$dir/$fileName")) {
    print $file $template;
    print $file "\n";
    close $file;
  } else {
    print STDERR qq(Failed to create file "$dir/$fileName" $!\n);
  }

  return;
}

sub usage() {
  print STDERR q(usage: make_tests.pl -f file | -w wiki_title | -s spec [-n -v -d dir ]

  -s specname   - the name of a spec known to the system
  -w wiki_title - the TITLE of a wiki page with testable statements
  -f file       - the file from which to read

  -n            - do nothing
  -v            - be verbose
  -d dir        - put generated tests in directory dir
  );
  exit 1;
}

# vim: ts=2 sw=2 ai: