From ed39053ab150a9c703fcbda467226047ddd63a5b Mon Sep 17 00:00:00 2001 From: "George S. Baugh" Date: Wed, 10 Aug 2016 22:47:43 -0500 Subject: [PATCH] Build results of 3e50abf (on master) --- Changes | 8 + MANIFEST | 5 +- META.yml | 19 +- Makefile.PL | 5 +- README | 2 +- TestRail-API-0.036/.travis.yml | 40 --- bin/testrail-bulk-mark-results | 4 +- bin/testrail-cases | 4 +- bin/testrail-lock | 4 +- bin/testrail-report | 6 +- bin/testrail-results | 293 ++++++++++++++++ bin/testrail-runs | 4 +- bin/testrail-tests | 4 +- lib/App/Prove/Plugin/TestRail.pm | 24 +- lib/Test/Rail/Harness.pm | 5 +- lib/Test/Rail/Parser.pm | 48 ++- lib/TestRail/API.pm | 88 ++++- lib/TestRail/Utils.pm | 6 +- lib/TestRail/Utils/Find.pm | 75 ++++- lib/TestRail/Utils/Lock.pm | 4 +- lib/TestRail/Utils/Results.pm | 4 +- t/00-compile.t | 3 +- t/App-Prove-Plugin-Testrail.t | 14 +- t/Test-Rail-Parser.t | 31 +- t/TestRail-API-sections.t | 42 +++ t/TestRail-API.t | 5 +- t/TestRail-Utils-Find.t | 2 + t/author-eol.t | 4 + t/author-no-tabs.t | 4 + t/author-pod-spell.t | 3 + t/data/faketest_cache.json | 1 + t/lib/Test/LWP/UserAgent/TestRailMock.pm | 404 ++++++++++++++++++++++- t/testrail-results.t | 123 +++++++ 33 files changed, 1188 insertions(+), 100 deletions(-) delete mode 100644 TestRail-API-0.036/.travis.yml create mode 100755 bin/testrail-results create mode 100644 t/TestRail-API-sections.t create mode 100644 t/data/faketest_cache.json create mode 100644 t/testrail-results.t diff --git a/Changes b/Changes index c7d13a6..ae5a9d9 100644 --- a/Changes +++ b/Changes @@ -1,5 +1,13 @@ Revision history for Perl module TestRail::API +0.037 2016-04-?? TEODESIAN + - Fix incorrect POD for TestRail::API::createRunInPlan + - Add testrail-results binary and TestRail::Utils::Find::getResults. + - Add TestRail::API::getChildSections, and modify Test::Rail::Parser to recursively search passed sections when spawning runs + - Change TestRail::API::getSections to cache the sections in a project. + - Add notices about problems with duplicate entries to POD. + - Add capability to auto-spawn configurations/groups to App::Prove::Plugin::TestRail and friends when configuration_group is passed + 0.036 2016-04-25 TEODESIAN - Fix using wrong perl during testsuite when running binaries - Silence testsuite for easier diagnosis of issues diff --git a/MANIFEST b/MANIFEST index 60182e0..f24aa6e 100644 --- a/MANIFEST +++ b/MANIFEST @@ -7,11 +7,11 @@ META.yml Makefile.PL README SIGNATURE -TestRail-API-0.036/.travis.yml bin/testrail-bulk-mark-results bin/testrail-cases bin/testrail-lock bin/testrail-report +bin/testrail-results bin/testrail-runs bin/testrail-tests lib/App/Prove/Plugin/TestRail.pm @@ -28,6 +28,7 @@ t/00-compile.t t/App-Prove-Plugin-Testrail.t t/Test-Rail-Parser.t t/TestRail-API-mockOnly.t +t/TestRail-API-sections.t t/TestRail-API.t t/TestRail-Utils-Find.t t/TestRail-Utils-Lock.t @@ -41,6 +42,7 @@ t/author-no-tabs.t t/author-pod-spell.t t/author-pod-syntax.t t/author-test-version.t +t/data/faketest_cache.json t/fake.tap t/fake.test t/faker.test @@ -67,6 +69,7 @@ t/testrail-bulk-mark-results.t t/testrail-cases.t t/testrail-lock.t t/testrail-report.t +t/testrail-results.t t/testrail-runs.t t/testrail-tests.t t/todo_pass.test diff --git a/META.yml b/META.yml index d5abb03..46d43c1 100644 --- a/META.yml +++ b/META.yml @@ -29,28 +29,28 @@ name: TestRail-API provides: App::Prove::Plugin::TestRail: file: lib/App/Prove/Plugin/TestRail.pm - version: 0.036 + version: 0.037 Test::Rail::Harness: file: lib/Test/Rail/Harness.pm - version: 0.036 + version: 0.037 Test::Rail::Parser: file: lib/Test/Rail/Parser.pm - version: 0.036 + version: 0.037 TestRail::API: file: lib/TestRail/API.pm - version: 0.036 + version: 0.037 TestRail::Utils: file: lib/TestRail/Utils.pm - version: 0.036 + version: 0.037 TestRail::Utils::Find: file: lib/TestRail/Utils/Find.pm - version: 0.036 + version: 0.037 TestRail::Utils::Lock: file: lib/TestRail/Utils/Lock.pm - version: 0.036 + version: 0.037 TestRail::Utils::Results: file: lib/TestRail/Utils/Results.pm - version: 0.036 + version: 0.037 requires: Carp: 0 Clone: 0 @@ -69,6 +69,7 @@ requires: POSIX: 0 Pod::Usage: 0 Scalar::Util: 0 + Statistics::Descriptive: 0 Sys::Hostname: 0 TAP::Harness: 0 TAP::Parser: 0 @@ -86,6 +87,6 @@ resources: bugtracker: https://rt.cpan.org/Public/Dist/Display.html?Name=TestRail-API homepage: https://github.com/teodesian/TestRail-Perl repository: git://github.com/teodesian/TestRail-Perl.git -version: 0.036 +version: 0.037 x_contributors: - 'Neil Bowers ' diff --git a/Makefile.PL b/Makefile.PL index 7622eca..fba0fca 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -18,6 +18,7 @@ my %WriteMakefileArgs = ( "bin/testrail-cases", "bin/testrail-lock", "bin/testrail-report", + "bin/testrail-results", "bin/testrail-runs", "bin/testrail-tests" ], @@ -42,6 +43,7 @@ my %WriteMakefileArgs = ( "POSIX" => 0, "Pod::Usage" => 0, "Scalar::Util" => 0, + "Statistics::Descriptive" => 0, "Sys::Hostname" => 0, "TAP::Harness" => 0, "TAP::Parser" => 0, @@ -71,7 +73,7 @@ my %WriteMakefileArgs = ( "Test::More" => 0, "blib" => "1.01" }, - "VERSION" => "0.036", + "VERSION" => "0.037", "test" => { "TESTS" => "t/*.t" } @@ -105,6 +107,7 @@ my %FallbackPrereqs = ( "POSIX" => 0, "Pod::Usage" => 0, "Scalar::Util" => 0, + "Statistics::Descriptive" => 0, "Sys::Hostname" => 0, "TAP::Harness" => 0, "TAP::Parser" => 0, diff --git a/README b/README index 4fb773f..0f5a248 100644 --- a/README +++ b/README @@ -1,7 +1,7 @@ This archive contains the distribution TestRail-API, -version 0.036: +version 0.037: Provides an interface to TestRail's REST api via HTTP diff --git a/TestRail-API-0.036/.travis.yml b/TestRail-API-0.036/.travis.yml deleted file mode 100644 index 199f89f..0000000 --- a/TestRail-API-0.036/.travis.yml +++ /dev/null @@ -1,40 +0,0 @@ -language: perl -perl: - - '5.20' - - '5.18' - - '5.16' - - '5.14' - - '5.12' - - '5.10' - -matrix: - fast_finish: true - include: - - perl: '5.22' - env: COVERAGE=1 - -sudo: false -before_install: - - git clone git://github.com/travis-perl/helpers ~/travis-perl-helpers - - source ~/travis-perl-helpers/init - - build-perl - - perl -V - - build-dist - - cd $BUILD_DIR - -install: - - cpan-install --deps # installs prereqs, including recommends - - cpan-install --coverage # installs converage prereqs, if enabled - - cpanm --quiet --notest Devel::Cover::Report::Coveralls #Make sure to shoot it over to coveralls - - cpanm --quiet --notest Test::Spelling Test::NoTabs Test::EOL #Looks like they were missed by haarg's stuff - -branches: - only: - - /^build/ - -before_script: - - coverage-setup -script: - - prove -l -j$(test-jobs) $(test-files) # parallel testing -after_success: - - coverage-report diff --git a/bin/testrail-bulk-mark-results b/bin/testrail-bulk-mark-results index 0a06dc9..d2d1b0a 100755 --- a/bin/testrail-bulk-mark-results +++ b/bin/testrail-bulk-mark-results @@ -3,7 +3,7 @@ # PODNAME: TestRail::Bin::BulkMarkResults package TestRail::Bin::BulkMarkResults; -$TestRail::Bin::BulkMarkResults::VERSION = '0.036'; +$TestRail::Bin::BulkMarkResults::VERSION = '0.037'; use strict; use warnings; use utf8; @@ -87,7 +87,7 @@ TestRail::Bin::BulkMarkResults - Bulk mark entire runs/plans (or groups of tests =head1 VERSION -version 0.036 +version 0.037 =head1 DESCRIPTION diff --git a/bin/testrail-cases b/bin/testrail-cases index 83e070d..c12d625 100755 --- a/bin/testrail-cases +++ b/bin/testrail-cases @@ -3,7 +3,7 @@ # PODNAME: TestRail::Bin::Cases package TestRail::Bin::Cases; -$TestRail::Bin::Cases::VERSION = '0.036'; +$TestRail::Bin::Cases::VERSION = '0.037'; use strict; use warnings; use utf8; @@ -99,7 +99,7 @@ TestRail::Bin::Cases - get information about cases inside various testsuites/sec =head1 VERSION -version 0.036 +version 0.037 =head1 SYNOPSIS diff --git a/bin/testrail-lock b/bin/testrail-lock index f39f16f..bc2cb66 100755 --- a/bin/testrail-lock +++ b/bin/testrail-lock @@ -3,7 +3,7 @@ # PODNAME: TestRail::Bin::Lock package TestRail::Bin::Lock; -$TestRail::Bin::Lock::VERSION = '0.036'; +$TestRail::Bin::Lock::VERSION = '0.037'; use strict; use warnings; use utf8; @@ -78,7 +78,7 @@ TestRail::Bin::Lock - Lock a test in a TestRail, and return the test name if suc =head1 VERSION -version 0.036 +version 0.037 =head1 SYNOPSIS diff --git a/bin/testrail-report b/bin/testrail-report index 5a041d5..737a7fc 100755 --- a/bin/testrail-report +++ b/bin/testrail-report @@ -3,7 +3,7 @@ # PODNAME: TestRail::Bin::Report package TestRail::Bin::Report; -$TestRail::Bin::Report::VERSION = '0.036'; +$TestRail::Bin::Report::VERSION = '0.037'; use strict; use warnings; @@ -123,7 +123,7 @@ TestRail::Bin::Report - Upload your TAP results to TestRail after they've finish =head1 VERSION -version 0.036 +version 0.037 =head1 SYNOPSIS @@ -132,7 +132,7 @@ version 0.036 prove -v sometest.t | testrail-report [OPTIONS] - prove -PTestRail='apiurl=http://some.testlink.install/,user=someUser,password=somePassword,project=TestProject,run=TestRun,plan=TestPlan,configs=Config1:Config2:Config3,version=0.014' sometest.t + prove -PTestRail='apiurl=http://some.testrail.install/,user=someUser,password=somePassword,project=TestProject,run=TestRun,plan=TestPlan,configs=Config1:Config2:Config3,version=0.014' sometest.t require `which testrail-report`; TestRail::Bin::Report::run('args' => @args); diff --git a/bin/testrail-results b/bin/testrail-results new file mode 100755 index 0000000..1bf0f18 --- /dev/null +++ b/bin/testrail-results @@ -0,0 +1,293 @@ +#!/usr/bin/env perl +# ABSTRACT: List results for specified test(s). +# PODNAME: TestRail::Bin::Results + +package TestRail::Bin::Results; +$TestRail::Bin::Results::VERSION = '0.037'; +use strict; +use warnings; +use utf8; + +use TestRail::API; +use TestRail::Utils; +use TestRail::Utils::Find; + +use Getopt::Long qw{GetOptionsFromArray}; +use File::HomeDir qw{my_home}; +use JSON::MaybeXS (); +use Statistics::Descriptive; + +if ( !caller() ) { + my ( $out, $code ) = run( 'args' => \@ARGV ); + print "$out\n"; + exit $code; +} + +sub run { + my %params = @_; + my $opts = {}; + + #Parse config file if we are missing api url/key or user + my $homedir = my_home() || '.'; + if ( -e $homedir . '/.testrailrc' ) { + $opts = TestRail::Utils::parseConfig($homedir); + } + + GetOptionsFromArray( + $params{'args'}, + 'apiurl=s' => \$opts->{'apiurl'}, + 'password=s' => \$opts->{'password'}, + 'user=s' => \$opts->{'user'}, + 'j|project=s@' => \$opts->{'projects'}, + 'p|plan=s@' => \$opts->{'plans'}, + 'r|run=s@' => \$opts->{'runs'}, + 'e|encoding=s' => \$opts->{'encoding'}, + 'g|grep=s' => \$opts->{'pattern'}, + 'c|cachefile=s' => \$opts->{'cachefile'}, + 'json' => \$opts->{'json'}, + 'h|help' => \$opts->{'help'}, + ); + + if ( $opts->{help} ) { return ( '', TestRail::Utils::help() ); } + + die("No tests passed") unless scalar( @{ $params{'args'} } ); + die("Prior search file passed does not exist") + if $opts->{'cachefile'} && !( -e $opts->{'cachefile'} ); + + $opts->{'browser'} = $params{'browser'}; + + TestRail::Utils::interrogateUser( $opts, qw{apiurl user password} ); + + my $tr = TestRail::Utils::getHandle($opts); + my $prior_search; + my $prior_runs = []; + if ( $opts->{'cachefile'} ) { + my $raw_text = ''; + open( my $fh, '<', $opts->{'cachefile'} ) + or die "Could not open $opts->{cachefile}"; + while (<$fh>) { + $raw_text .= $_; + } + close($fh); + $prior_search = JSON::MaybeXS::decode_json($raw_text); + foreach my $key ( keys(%$prior_search) ) { + push( @$prior_runs, @{ $prior_search->{$key}->{'seen_runs'} } ); + } + } + + my $res = TestRail::Utils::Find::getResults( $tr, $opts, $prior_runs, + @{ $params{'args'} } ); + + my $statuses = $tr->getPossibleTestStatuses(); + my %status_map; + @status_map{ map { $_->{'id'} } @$statuses } = + map { $_->{'label'} } @$statuses; + + my ( $out, $out_json ) = ( '', {} ); + foreach my $case ( keys(%$res) ) { + $out .= "#############################\n"; + my $num_runs = 0; + my $casetotals = {}; + my $total_elapsed = 0; + my $avg_elapsed = 0; + my $median_runtime = 0; + my $elapsetotals = []; + my $seen_runs = []; + + foreach my $casedef ( @{ $res->{$case} } ) { + push( @$seen_runs, $casedef->{run_id} ); + $num_runs++; + + #$out .= "Found case '$case' in run $casedef->{run_id}\n"; + foreach my $result ( @{ $casedef->{results} } ) { + $casetotals->{ $result->{status_id} }++; + push( @$elapsetotals, _elapsed2secs( $result->{'elapsed'} ) ); + } + } + + my $pattern_output = ''; + $out_json->{$case}->{search_string} = $opts->{'pattern'}; + $pattern_output = " using search string '$opts->{pattern}'" + if $opts->{'pattern'}; + + $out .= "$case was present in $num_runs runs$pattern_output.\n"; + $out_json->{$case}->{'num_runs'} = $num_runs; + $out_json->{$case}->{'seen_runs'} = $seen_runs; + + #Collect time statistics + my $timestats = Statistics::Descriptive::Full->new(); + $timestats->add_data(@$elapsetotals); + + $out_json->{$case}->{total_elapsed} = $timestats->sum() || 0; + $out .= + "Total time spent running this test: $out_json->{$case}->{total_elapsed} seconds\n"; + $out_json->{$case}->{median_elapsed} = $timestats->median() || 0; + $out .= + "Median time spent running this test: $out_json->{$case}->{median_elapsed} seconds\n"; + $out_json->{$case}->{average_elapsed} = $timestats->mean() || 0; + $out .= + "Mean time spent running this test: $out_json->{$case}->{average_elapsed} seconds\n"; + $out_json->{$case}->{stdev_elapsed} = + $timestats->standard_deviation() || 0; + $out .= + "Standard deviations of runtime in test: $out_json->{$case}->{stdev_elapsed}\n"; + $out_json->{$case}->{max_elapsed} = $timestats->max() || 0; + $out .= + "Maximum time spent running this test: $out_json->{$case}->{max_elapsed} seconds\n"; + $out_json->{$case}->{min_elapsed} = $timestats->min() || 0; + $out .= + "Minimum time spent running this test: $out_json->{$case}->{min_elapsed} seconds\n"; + $out_json->{$case}->{times_executed} = $timestats->count() || 0; + $out .= + "Num times this test has been executed: $out_json->{$case}->{times_executed}\n"; + + foreach my $status ( keys(%$casetotals) ) { + $out .= "$status_map{$status}: $casetotals->{$status}\n"; + $out_json->{$case}->{ $status_map{$status} } = + $casetotals->{$status}; + } + } + + if ( $opts->{'json'} ) { + my $coder = JSON::MaybeXS->new; + return ( $coder->encode($out_json), 0 ); + } + + $out .= "#############################"; + return ( $out, 0 ); +} + +sub _elapsed2secs { + my $stamp = shift; + return 0 if !$stamp; + my ($seconds) = $stamp =~ m/(\d*)s/; + my ($seconds_minutes) = $stamp =~ m/(\d*)m/; + my ($seconds_hours) = $stamp =~ m/(\d*)h/; + return ( $seconds || 0 ) + + ( $seconds_minutes ? $seconds_minutes * 60 : 0 ) + + ( $seconds_hours ? $seconds_hours * 3600 : 0 ); +} + +1; + +=pod + +=encoding UTF-8 + +=head1 NAME + +TestRail::Bin::Results - List results for specified test(s). + +=head1 VERSION + +version 0.037 + +=head1 SYNOPSIS + + testrail-results [OPTIONS] test1 test2 ... + + require `which testrail-results`; + TestRail::Bin::Results::run('args' => \@args); + +=head1 DESCRIPTION + +testrail-results - List results for specified test(s). + +Searches across multiple runs (and projects) for results for broad-based metrics; especially useful for diagnosing unreliable tests, and establishing defect density for certain features. + +Can be used as the modulino TestRail::Bin::Tests. +Has a single 'run' function which accepts a hash with the 'args' parameter being the array of arguments. + +=head1 WARNING + +Searching across all projects can take a very long time for highly active TestRail Installations. +However, cross project metrics are also very useful. + +As such, the results from prior searches in json mode may be provided, and the runs previously analyzed therein will not be investigated again. + +It is up to the caller to integrate this data into their analysis as may be appropriate. + +=head1 PARAMETERS: + +=head2 MANDATORY PARAMETERS + +=over 4 + +--apiurl : full URL to get to TestRail index document + +--password : Your TestRail Password, or a valid API key (TestRail 4.2 and above). + +--user : Your TestRail User Name. + +=back + +All mandatory options not passed with the above switches, or in your ~/.testrailrc will be prompted for. + +=head2 SEMI-OPTIONAL PARAMETERS + +=over 4 + +-e --encoding : Character encoding of arguments. Defaults to UTF-8. See L for supported encodings. + +=back + +=head2 OPTIONAL PARAMETERS + +=over 4 + +-j --project : Restrict search to provided project name. May be passed multiple times. + +-r --run : Restrict search to runs with the provided name. May be passed multiple times. + +-p --plan : Restrict search to plans with the provided name. May be passed multiple times. + +-g --grep : Restrict results printed to those matching the provided pattern. Great for looking for specific failure conditions. + +-c --cachefile : Load the provided file as a place to pick up your search from. + +--json : Print results as a JSON serialization. + +=back + +=head1 CONFIGURATION FILE + +In your \$HOME, (or the current directory, if your system has no concept of a home directory) put a file called .testrailrc with key=value syntax separated by newlines. +Valid Keys are the same as documented by L. +All options specified thereby are overridden by passing the command-line switches above. + +=head1 MISCELLANEOUS OPTIONS: + +=over 4 + +--help : show this output + +=back + +=head1 SPECIAL THANKS + +Thanks to cPanel Inc, for graciously funding the creation of this distribution. + +=head1 AUTHOR + +George S. Baugh + +=head1 SOURCE + +The development version is on github at L +and may be cloned from L + +=head1 COPYRIGHT AND LICENSE + +This software is copyright (c) 2016 by George S. Baugh. + +This is free software; you can redistribute it and/or modify it under +the same terms as the Perl 5 programming language system itself. + +=cut + +__END__ + +L + +L for the finding of .testrailrc + diff --git a/bin/testrail-runs b/bin/testrail-runs index 2ef4409..1d2a44e 100755 --- a/bin/testrail-runs +++ b/bin/testrail-runs @@ -3,7 +3,7 @@ # PODNAME: TestRail::Bin::Runs package TestRail::Bin::Runs; -$TestRail::Bin::Runs::VERSION = '0.036'; +$TestRail::Bin::Runs::VERSION = '0.037'; use strict; use warnings; use utf8; @@ -71,7 +71,7 @@ TestRail::Bin::Runs - List runs in a TestRail project matching the provided filt =head1 VERSION -version 0.036 +version 0.037 =head1 SYNOPSIS diff --git a/bin/testrail-tests b/bin/testrail-tests index d6e3c5c..06f94e5 100755 --- a/bin/testrail-tests +++ b/bin/testrail-tests @@ -3,7 +3,7 @@ # PODNAME: TestRail::Bin::Tests package TestRail::Bin::Tests; -$TestRail::Bin::Tests::VERSION = '0.036'; +$TestRail::Bin::Tests::VERSION = '0.037'; use strict; use warnings; use utf8; @@ -82,7 +82,7 @@ TestRail::Bin::Tests - List tests in a TestRail run matching the provided filter =head1 VERSION -version 0.036 +version 0.037 =head1 SYNOPSIS diff --git a/lib/App/Prove/Plugin/TestRail.pm b/lib/App/Prove/Plugin/TestRail.pm index 72ffdd6..b95c6ef 100644 --- a/lib/App/Prove/Plugin/TestRail.pm +++ b/lib/App/Prove/Plugin/TestRail.pm @@ -2,7 +2,7 @@ # PODNAME: App::Prove::Plugin::TestRail package App::Prove::Plugin::TestRail; -$App::Prove::Plugin::TestRail::VERSION = '0.036'; +$App::Prove::Plugin::TestRail::VERSION = '0.037'; use strict; use warnings; use utf8; @@ -55,6 +55,7 @@ sub load { $ENV{'TESTRAIL_SECTIONS'} = $params->{sections}; $ENV{'TESTRAIL_AUTOCLOSE'} = $params->{autoclose}; $ENV{'TESTRAIL_ENCODING'} = $params->{encoding}; + $ENV{'TESTRIAL_CGROUP'} = $params->{'configuration_group'}; return $class; } @@ -72,11 +73,11 @@ App::Prove::Plugin::TestRail - Upload your TAP results to TestRail in realtime =head1 VERSION -version 0.036 +version 0.037 =head1 SYNOPSIS -`prove -PTestRail='apiurl=http://some.testlink.install/,user=someUser,password=somePassword,project=TestProject,run=TestRun,plan=TestPlan,configs=Config1:Config2:Config3,version=0.014' sometest.t` +`prove -PTestRail='apiurl=http://some.testrail.install/,user=someUser,password=somePassword,project=TestProject,run=TestRun,plan=TestPlan,configs=Config1:Config2:Config3,version=0.014' sometest.t` =head1 DESCRIPTION @@ -102,8 +103,12 @@ If \$HOME/.testrailrc exists, it will be parsed for any of these values in a new sections=section1:section2:section3: ... :sectionN autoclose=0 encoding=UTF-8 + configuration_group=Operating Systems Note that passing configurations as filters for runs inside of plans are separated by colons. + +If a configuration_group option is passed, it, and any configurations passed will be created automatically for you in the case they do not exist. + Values passed in via query string will override values in \$HOME/.testrailrc. If your system has no concept of user homes, it will look in the current directory for .testrailrc. @@ -114,6 +119,19 @@ See the documentation for the constructor of L as to why you When running prove in multiple job mode (-j), or when breaking out test jobs into multiple prove processes, auto-spawn of plans & runs can race. Be sure to extend your harness to make sure these things are already created if you do either of these things. +Also, all parameters expecting names are vulnerable to duplicate naming issues. Try not to use the same name for: + + * projects + * testsuites within the same project + * sections within the same testsuite that are peers + * test cases + * test plans and runs outside of plans which are not completed + * configurations & configuration groups + +To do so will result in the first of said item found. +This might result in the reuse of an existing run/plan unintentionally, or spawning runs within the wrong project/testsuite or with incorrect test sections. +Similarly, duplicate named tests will result in one of the dupes never being run (as the first found is chosen). + =head1 OVERRIDDEN METHODS =head2 load(parser) diff --git a/lib/Test/Rail/Harness.pm b/lib/Test/Rail/Harness.pm index d7bfc9c..e21317c 100644 --- a/lib/Test/Rail/Harness.pm +++ b/lib/Test/Rail/Harness.pm @@ -1,7 +1,7 @@ # ABSTRACT: TestRail testing harness # PODNAME: Test::Rail::Harness package Test::Rail::Harness; -$Test::Rail::Harness::VERSION = '0.036'; +$Test::Rail::Harness::VERSION = '0.037'; use strict; use warnings; @@ -38,6 +38,7 @@ sub make_parser { $args->{'step_results'} = $ENV{'TESTRAIL_STEPS'}; $args->{'testsuite_id'} = $ENV{'TESTRAIL_SPAWN'}; $args->{'testsuite'} = $ENV{'TESTRAIL_TESTSUITE'}; + $args->{'config_group'} = $ENV{'TESTRAIL_CGROUP'}; @sections = split( /:/, $ENV{'TESTRAIL_SECTIONS'} ) if $ENV{'TESTRAIL_SECTIONS'}; @@ -79,7 +80,7 @@ Test::Rail::Harness - TestRail testing harness =head1 VERSION -version 0.036 +version 0.037 =head1 DESCRIPTION diff --git a/lib/Test/Rail/Parser.pm b/lib/Test/Rail/Parser.pm index 1c3284a..322642b 100644 --- a/lib/Test/Rail/Parser.pm +++ b/lib/Test/Rail/Parser.pm @@ -2,7 +2,7 @@ # PODNAME: Test::Rail::Parser package Test::Rail::Parser; -$Test::Rail::Parser::VERSION = '0.036'; +$Test::Rail::Parser::VERSION = '0.037'; use strict; use warnings; use utf8; @@ -48,6 +48,7 @@ sub new { 'encoding' => delete $opts->{'encoding'}, 'sections' => delete $opts->{'sections'}, 'autoclose' => delete $opts->{'autoclose'}, + 'config_group' => delete $opts->{'config_group'}, #Stubs for extension by subclassers 'result_options' => delete $opts->{'result_options'}, @@ -128,6 +129,31 @@ sub new { #Grab run my ( $run, $plan, $config_ids ); + # See if we have to create a configuration + my $configz2create = $tr->getConfigurations( $tropts->{'project_id'} ); + @$configz2create = grep { + my $c = $_; + ( grep { $_ eq $c->{'name'} } @{ $tropts->{'configs'} } ) + } @$configz2create; + if ( scalar(@$configz2create) && $tropts->{'config_group'} ) { + my $cgroup = + $tr->getConfigurationGroupByName( $tropts->{project_id}, + $tropts->{'config_group'} ); + unless ( ref($cgroup) eq 'HASH' ) { + print "# Adding Configuration Group $tropts->{config_group}...\n"; + $cgroup = + $tr->addConfigurationGroup( $tropts->{project_id}, + $tropts->{'config_group'} ); + } + confess( + "Could neither find nor create the provided configuration group '$tropts->{config_group}'" + ) unless ref($cgroup) eq 'HASH'; + foreach my $cc (@$configz2create) { + print "# Adding Configuration $cc->{name}...\n"; + $tr->addConfiguration( $cgroup->{'id'}, $cc->{'name'} ); + } + } + #check if configs passed are defined for project. If we can't get all the IDs, something's hinky @$config_ids = $tr->translateConfigNamesToIds( $tropts->{'project_id'}, @{ $tropts->{'configs'} } ); @@ -203,6 +229,24 @@ sub new { @{ $tropts->{'sections'} } ); foreach my $section ( @{ $tropts->{'sections'} } ) { + + #Get the child sections, and append them to our section list so we get their cases too. + my $append_sections = $tr->getChildSections( + $tropts->{'project_id'}, + { + 'id' => $section, + 'suite_id' => $tropts->{'testsuite_id'} + } + ); + @$append_sections = grep { + my $sc = $_; + !scalar( grep { $_ == $sc->{'id'} } + @{ $tropts->{'sections'} } ) + } @$append_sections + ; #de-dup in case the user added children to the list + @$append_sections = map { $_->{'id'} } @$append_sections; + push( @{ $tropts->{'sections'} }, @$append_sections ); + my $section_cases = $tr->getCases( $tropts->{'project_id'}, $tropts->{'testsuite_id'}, @@ -673,7 +717,7 @@ Test::Rail::Parser - Upload your TAP results to TestRail =head1 VERSION -version 0.036 +version 0.037 =head1 DESCRIPTION diff --git a/lib/TestRail/API.pm b/lib/TestRail/API.pm index eb71c05..2abeeb6 100644 --- a/lib/TestRail/API.pm +++ b/lib/TestRail/API.pm @@ -2,7 +2,7 @@ # PODNAME: TestRail::API package TestRail::API; -$TestRail::API::VERSION = '0.036'; +$TestRail::API::VERSION = '0.037'; use 5.010; @@ -407,8 +407,13 @@ sub getSections { state $check = compile( Object, Int, Int ); my ( $self, $project_id, $suite_id ) = $check->(@_); - return $self->_doRequest( + #Cache sections to reduce requests in tight loops + return $self->{'sections'}->{$project_id} + if $self->{'sections'}->{$project_id}; + $self->{'sections'}->{$project_id} = $self->_doRequest( "index.php?/api/v2/get_sections/$project_id&suite_id=$suite_id"); + + return $self->{'sections'}->{$project_id}; } sub getSectionByID { @@ -430,6 +435,24 @@ sub getSectionByName { return 0; } +sub getChildSections { + state $check = compile( Object, Int, HashRef ); + my ( $self, $project_id, $section ) = $check->(@_); + + my $sections_orig = $self->getSections( $project_id, $section->{suite_id} ); + return [] + if !$sections_orig || ( reftype($sections_orig) || 'undef' ) ne 'ARRAY'; + my @sections = + grep { $_->{'parent_id'} ? $_->{'parent_id'} == $section->{'id'} : 0 } + @$sections_orig; + foreach my $sec (@sections) { + push( @sections, + grep { $_->{'parent_id'} ? $_->{'parent_id'} == $sec->{'id'} : 0 } + @$sections_orig ); + } + return \@sections; +} + sub sectionNamesToIds { my ( $self, $project_id, $suite_id, @names ) = @_; my $sections = $self->getSections( $project_id, $suite_id ) @@ -1052,8 +1075,11 @@ sub getTestResultFieldByName { sub getPossibleTestStatuses { state $check = compile(Object); my ($self) = $check->(@_); + return $self->{'status_cache'} if $self->{'status_cache'}; - return $self->_doRequest('index.php?/api/v2/get_statuses'); + $self->{'status_cache'} = + $self->_doRequest('index.php?/api/v2/get_statuses'); + return $self->{'status_cache'}; } sub statusNamesToIds { @@ -1156,6 +1182,17 @@ sub getConfigurationGroups { return $self->_doRequest($url); } +sub getConfigurationGroupByName { + state $check = compile( Object, Int, Str ); + my ( $self, $project_id, $name ) = $check->(@_); + + my $cgroups = $self->getConfigurationGroups($project_id); + return 0 if ref($cgroups) ne 'ARRAY'; + @$cgroups = grep { $_->{'name'} eq $name } @$cgroups; + return 0 unless scalar(@$cgroups); + return $cgroups->[0]; +} + sub addConfigurationGroup { state $check = compile( Object, Int, Str ); my ( $self, $project_id, $name ) = $check->(@_); @@ -1251,14 +1288,14 @@ TestRail::API - Provides an interface to TestRail's REST api via HTTP =head1 VERSION -version 0.036 +version 0.037 =head1 SYNOPSIS use TestRail::API; - my ($username,$password,$host) = ('foo','bar','testlink.baz.foo'); - my $tr = TestRail::API->new($username, $password, $host); + my ($username,$password,$host) = ('foo','bar','http://testrail.baz.foo'); + my $tr = TestRail::API->new($host, $username, $password); =head1 DESCRIPTION @@ -1271,6 +1308,17 @@ All the methods aside from the constructor should not die, but return a false va When the server is not responsive, expect a -500 response, and retry accordingly. I recommend using the excellent L module for this purpose. +Also, all *ByName methods are vulnerable to duplicate naming issues. Try not to use the same name for: + + * projects + * testsuites within the same project + * sections within the same testsuite that are peers + * test cases + * test plans and runs outside of plans which are not completed + * configurations + +To do so will result in the first of said item found being returned rather than an array of possibilities to choose from. + =head1 CONSTRUCTOR =head2 B @@ -1573,6 +1621,24 @@ Returns section definition HASHREF. $tr->getSectionByName(1,2,'nugs'); +=head2 B + +Gets desired section's child sections. + +=over 4 + +=item INTEGER C - parent project ID of section. + +=item HASHREF C
- section definition HASHREF. + +=back + +Returns ARRAYREF of section definition HASHREF. ARRAYREF is empty if there are none. + +Recursively searches for children, so the children of child sections will be returned as well. + + $tr->getChildSections($section); + =head2 sectionNamesToIds(project_id,suite_id,names) Convenience method to translate a list of section names to TestRail section IDs. @@ -2042,7 +2108,7 @@ The 'percentages' key has the same, but as a percentage of the total. $tr->getPlanSummary($plan_id); -=head2 B +=head2 B Create a run in a plan. @@ -2236,6 +2302,8 @@ Gets all possible statuses a test can be set to. Returns ARRAYREF of status definition HASHREFs. +Caches the result for the lifetime of the TestRail::API object. + =head2 statusNamesToIds(names) Convenience method to translate a list of statuses to TestRail status IDs. @@ -2355,6 +2423,12 @@ Gets the available configuration groups for a project, with their configurations Returns ARRAYREF of configuration group definition HASHREFs. +=head2 B + +Get the provided configuration group by name. + +Returns false if the configuration group could not be found. + =head2 B New in TestRail 5.2. diff --git a/lib/TestRail/Utils.pm b/lib/TestRail/Utils.pm index 3363727..5940894 100644 --- a/lib/TestRail/Utils.pm +++ b/lib/TestRail/Utils.pm @@ -2,7 +2,7 @@ # PODNAME: TestRail::Utils package TestRail::Utils; -$TestRail::Utils::VERSION = '0.036'; +$TestRail::Utils::VERSION = '0.037'; use strict; use warnings; @@ -34,7 +34,7 @@ sub interrogateUser { my ( $options, @keys ) = @_; foreach my $key (@keys) { if ( !$options->{$key} ) { - print "Type the $key for your testLink install below:\n"; + print "Type the $key for your TestRail install below:\n"; $options->{$key} = TestRail::Utils::userInput(); die "$key cannot be blank!" unless $options->{$key}; } @@ -199,7 +199,7 @@ TestRail::Utils - Utilities for the testrail command line functions, and their m =head1 VERSION -version 0.036 +version 0.037 =head1 SCRIPT HELPER FUNCTIONS diff --git a/lib/TestRail/Utils/Find.pm b/lib/TestRail/Utils/Find.pm index 45cc81e..dc74f5c 100644 --- a/lib/TestRail/Utils/Find.pm +++ b/lib/TestRail/Utils/Find.pm @@ -2,7 +2,7 @@ # ABSTRACT: Find runs and tests according to user specifications. package TestRail::Utils::Find; -$TestRail::Utils::Find::VERSION = '0.036'; +$TestRail::Utils::Find::VERSION = '0.037'; use strict; use warnings; @@ -259,6 +259,71 @@ sub findCases { return $ret; } +sub getResults { + my ( $tr, $opts, $prior_runs, @cases ) = @_; + my $res = {}; + my $projects = $tr->getProjects(); + + #TODO obey status filtering + #TODO obey result notes text grepping + foreach my $project (@$projects) { + next + if $opts->{projects} + && !( grep { $_ eq $project->{'name'} } @{ $opts->{'projects'} } ); + my $runs = $tr->getRuns( $project->{'id'} ); + + #Translate plan names to ids + my $plans = $tr->getPlans( $project->{'id'} ) || []; + $opts->{'runs'} //= []; + my $plan_filters = []; + foreach my $plan (@$plans) { + $plan = $tr->getPlanByID( $plan->{'id'} ); + my $plan_runs = $tr->getChildRuns($plan); + push( @$runs, @$plan_runs ) if $plan_runs; + } + + if ( $opts->{'plans'} ) { + @$plan_filters = map { $_->{'id'} } grep { + my $p = $_; + grep { $p->{'name'} eq $_ } @{ $opts->{'plans'} } + } @$plans; + } + + foreach my $run (@$runs) { + next + if scalar( @{ $opts->{runs} } ) + && !( grep { $_ eq $run->{'name'} } @{ $opts->{'runs'} } ); + next + if scalar(@$plan_filters) + && !( grep { $run->{'plan_id'} ? $_ eq $run->{'plan_id'} : undef } + @$plan_filters ); + next if grep { $run->{id} eq $_ } @$prior_runs; + foreach my $case (@cases) { + my $c = $tr->getTestByName( $run->{'id'}, basename($case) ); + next unless ref $c eq 'HASH'; + + $res->{$case} //= []; + $c->{results} = + $tr->getTestResults( $c->{'id'}, $tr->{'global_limit'}, 0 ); + + #Filter by provided pattern, if any + if ( $opts->{'pattern'} ) { + my $pattern = $opts->{pattern}; + @{ $c->{results} } = grep { + my $comment = $_->{comment} || ''; + $comment =~ m/$pattern/i + } @{ $c->{results} }; + } + + push( @{ $res->{$case} }, $c ) + if scalar( @{ $c->{results} } ) + ; #Make sure they weren't filtered out + } + } + } + return $res; +} + 1; __END__ @@ -273,7 +338,7 @@ TestRail::Utils::Find - Find runs and tests according to user specifications. =head1 VERSION -version 0.036 +version 0.037 =head1 DESCRIPTION @@ -370,6 +435,12 @@ Option hash keys for input are 'no-missing', 'orphans', and 'update'. Returns HASHREF. +=head2 getResults(options, $prior_runs, @cases) + +Get results for tests by name, filtered by the provided options, and skipping any runs found in the provided ARRAYREF of run IDs. + +Probably should have called this findResults, but we all prefer to get results right? + =head1 SPECIAL THANKS Thanks to cPanel Inc, for graciously funding the creation of this module. diff --git a/lib/TestRail/Utils/Lock.pm b/lib/TestRail/Utils/Lock.pm index 6e76232..0704f07 100644 --- a/lib/TestRail/Utils/Lock.pm +++ b/lib/TestRail/Utils/Lock.pm @@ -2,7 +2,7 @@ # PODNAME: TestRail::Utils::Lock package TestRail::Utils::Lock; -$TestRail::Utils::Lock::VERSION = '0.036'; +$TestRail::Utils::Lock::VERSION = '0.037'; use 5.010; use strict; @@ -143,7 +143,7 @@ TestRail::Utils::Lock - Pick high priority cases for execution and lock them via =head1 VERSION -version 0.036 +version 0.037 =head1 DESCRIPTION diff --git a/lib/TestRail/Utils/Results.pm b/lib/TestRail/Utils/Results.pm index db2f373..7ff2246 100644 --- a/lib/TestRail/Utils/Results.pm +++ b/lib/TestRail/Utils/Results.pm @@ -2,7 +2,7 @@ # ABSTRACT: Perform batch operations on test results, and analyze the same. package TestRail::Utils::Results; -$TestRail::Utils::Results::VERSION = '0.036'; +$TestRail::Utils::Results::VERSION = '0.037'; use strict; use warnings; @@ -47,7 +47,7 @@ TestRail::Utils::Results - Perform batch operations on test results, and analyze =head1 VERSION -version 0.036 +version 0.037 =head1 FUNCTIONS diff --git a/t/00-compile.t b/t/00-compile.t index fb220a7..9b46e32 100644 --- a/t/00-compile.t +++ b/t/00-compile.t @@ -6,7 +6,7 @@ use warnings; use Test::More; -plan tests => 14 + ($ENV{AUTHOR_TESTING} ? 1 : 0); +plan tests => 15 + ($ENV{AUTHOR_TESTING} ? 1 : 0); my @module_files = ( 'App/Prove/Plugin/TestRail.pm', @@ -24,6 +24,7 @@ my @scripts = ( 'bin/testrail-cases', 'bin/testrail-lock', 'bin/testrail-report', + 'bin/testrail-results', 'bin/testrail-runs', 'bin/testrail-tests' ); diff --git a/t/App-Prove-Plugin-Testrail.t b/t/App-Prove-Plugin-Testrail.t index 4cb4654..508674b 100644 --- a/t/App-Prove-Plugin-Testrail.t +++ b/t/App-Prove-Plugin-Testrail.t @@ -17,42 +17,42 @@ $ENV{'TESTRAIL_MOCKED'} = 1; #Test the same sort of data as would come from the Test::Rail::Parser case my $prove = App::Prove->new(); -$prove->process_args("-PTestRail=apiurl=http://some.testlink.install/,user=someUser,password=somePassword,project=TestProject,run=TestingSuite,version=0.014",'t/fake.test'); +$prove->process_args("-PTestRail=apiurl=http://some.testrail.install/,user=someUser,password=somePassword,project=TestProject,run=TestingSuite,version=0.014",'t/fake.test'); is (exception { capture { $prove->run() } },undef,"Running TR parser case via plugin functions"); #Check that plan, configs and version also make it through $prove = App::Prove->new(); -$prove->process_args("-PTestRail=apiurl=http://some.testlink.install/,user=someUser,password=somePassword,project=TestProject,run=Executing the great plan,version=0.014,plan=GosPlan,configs=testConfig",'t/fake.test'); +$prove->process_args("-PTestRail=apiurl=http://some.testrail.install/,user=someUser,password=somePassword,project=TestProject,run=Executing the great plan,version=0.014,plan=GosPlan,configs=testConfig",'t/fake.test'); is (exception { capture { $prove->run() } },undef,"Running TR parser case via plugin functions works with configs/plans"); #Check that spawn options make it through $prove = App::Prove->new(); -$prove->process_args("-PTestRail=apiurl=http://some.testlink.install/,user=someUser,password=somePassword,project=TestProject,run=TestingSuite2,version=0.014,testsuite_id=9",'t/skipall.test'); +$prove->process_args("-PTestRail=apiurl=http://some.testrail.install/,user=someUser,password=somePassword,project=TestProject,run=TestingSuite2,version=0.014,testsuite_id=9",'t/skipall.test'); is (exception { capture { $prove->run() } },undef,"Running TR parser case via plugin functions works with configs/plans"); $prove = App::Prove->new(); -$prove->process_args("-PTestRail=apiurl=http://some.testlink.install/,user=someUser,password=somePassword,project=TestProject,plan=bogoPlan,run=bogoRun,version=0.014,testsuite=HAMBURGER-IZE HUMANITY",'t/skipall.test'); +$prove->process_args("-PTestRail=apiurl=http://some.testrail.install/,user=someUser,password=somePassword,project=TestProject,plan=bogoPlan,run=bogoRun,version=0.014,testsuite=HAMBURGER-IZE HUMANITY",'t/skipall.test'); is (exception { capture { $prove->run() } },undef,"Running TR parser spawns both runs and plans"); $prove = App::Prove->new(); -$prove->process_args("-PTestRail=apiurl=http://some.testlink.install/,user=someUser,password=somePassword,project=TestProject,run=bogoRun,version=0.014,testsuite_id=9,sections=fake.test:CARBON LIQUEFACTION",'t/fake.test'); +$prove->process_args("-PTestRail=apiurl=http://some.testrail.install/,user=someUser,password=somePassword,project=TestProject,run=bogoRun,version=0.014,testsuite_id=9,sections=fake.test:CARBON LIQUEFACTION",'t/fake.test'); is (exception { capture { $prove->run() } },undef,"Running TR parser can discriminate by sections correctly"); $prove = App::Prove->new(); -$prove->process_args("-PTestRail=apiurl=http://some.testlink.install/,user=someUser,password=somePassword,project=TestProject,plan=FinalPlan,run=FinalRun,configs=testConfig,version=0.014,autoclose=1",'t/fake.test'); +$prove->process_args("-PTestRail=apiurl=http://some.testrail.install/,user=someUser,password=somePassword,project=TestProject,plan=FinalPlan,run=FinalRun,configs=testConfig,version=0.014,autoclose=1",'t/fake.test'); is (exception { capture { $prove->run() } },undef,"Running TR parser with autoclose works correctly"); #Test multi-job upload shizz #Note that I don't care if it even uploads, just that it *would have* done so correctly. $prove = App::Prove->new(); -$prove->process_args("-PTestRail=apiurl=http://some.testlink.install/,user=someUser,password=somePassword,project=TestProject,plan=FinalPlan,run=FinalRun,configs=testConfig,step_results=step_results", '-j2', 't/fake.test', 't/skipall.test'); +$prove->process_args("-PTestRail=apiurl=http://some.testrail.install/,user=someUser,password=somePassword,project=TestProject,plan=FinalPlan,run=FinalRun,configs=testConfig,step_results=step_results", '-j2', 't/fake.test', 't/skipall.test'); is (exception { capture { $prove->run() } },undef,"Running TR parser -j2 works"); diff --git a/t/Test-Rail-Parser.t b/t/Test-Rail-Parser.t index 12b4e2f..6298cef 100644 --- a/t/Test-Rail-Parser.t +++ b/t/Test-Rail-Parser.t @@ -10,7 +10,7 @@ use Scalar::Util qw{reftype}; use TestRail::API; use Test::LWP::UserAgent::TestRailMock; use Test::Rail::Parser; -use Test::More 'tests' => 115; +use Test::More 'tests' => 118; use Test::Fatal qw{exception}; use Test::Deep qw{cmp_deeply}; use Capture::Tiny qw{capture}; @@ -546,3 +546,32 @@ if (!$res) { my $srs = $tap->{'tr_opts'}->{'result_custom_options'}->{'step_results'}; is($srs->[-1]->{'content'},"Bail Out!.","Bailout noted in step results"); } + +#Check section spawn recursion is done correctly +undef $opts->{'tap'}; +$opts->{'source'} = 't/pass.test'; +$opts->{'testsuite_id'} = 5; +$opts->{'project_id'} = 3; +$opts->{'run'} = 'zippyRun'; +$opts->{'sections'} = ['Recursing section','grandchild']; + +$res = exception { $tap = Test::Rail::Parser->new($opts) }; +is($res,undef,"TR Parser runs all the way through when recursing sections"); + +if (!$res) { + $tap->run(); + is($tap->{'errors'},0,"No errors encountered uploading case results"); +} + +#Check configuration group spawn is done correctly +undef $opts->{'tap'}; +$opts->{'source'} = 't/pass.test'; +$opts->{'project_id'} = 9; +$opts->{'run'} = 'TestingSuite'; +$opts->{'plan'} = 'mah dubz plan'; +$opts->{'config_group'} = 'noSuchGroup'; +$opts->{'configs'} = ['noSuchConfig']; +$opts->{'sections'} = []; + +$res = exception { $tap = Test::Rail::Parser->new($opts) }; +is($res,undef,"TR Parser runs all the way through when spawning configurations"); diff --git a/t/TestRail-API-sections.t b/t/TestRail-API-sections.t new file mode 100644 index 0000000..d20c251 --- /dev/null +++ b/t/TestRail-API-sections.t @@ -0,0 +1,42 @@ +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::Bin/lib"; + +use TestRail::API; +use Test::LWP::UserAgent::TestRailMock; + +use Test::More tests => 2; +use Test::Fatal; +use Test::Deep; +use Scalar::Util (); +use Capture::Tiny qw{capture}; + +my $apiurl = $ENV{'TESTRAIL_API_URL'}; +my $login = $ENV{'TESTRAIL_USER'}; +my $pw = $ENV{'TESTRAIL_PASSWORD'}; + +#Mock if nothing is provided +my $is_mock = (!$apiurl && !$login && !$pw); +($apiurl,$login,$pw) = ('http://testrail.local','teodesian@cpan.org','fake') if $is_mock; + +my $tr = new TestRail::API($apiurl,$login,$pw,undef,1); + +#Mock if necesary +$tr->{'debug'} = 0; +$tr->{'browser'} = $Test::LWP::UserAgent::TestRailMock::mockObject if $is_mock; + +#This is a mock-only test. +my $project = $tr->getProjectByName('zippy'); +my $suite = $tr->getTestSuiteByName($project->{'id'},'Master'); +my $section = $tr->getSectionByName($project->{'id'},$suite->{'id'},'Recursing section'); + +my $children = $tr->getChildSections($project->{'id'},$section); + +my @expected = qw{child grandchild great-grandchild}; +my @actual = map {$_->{'name'} } @$children; +cmp_bag(\@actual,\@expected,"Got child suites recursively"); +cmp_bag($tr->getChildSections($project->{'id'},{ 'suite_id' => 999999999999999, 'id' => 9999999999999999 }),[],"Nothing returned when bogus section passed"); + +1; diff --git a/t/TestRail-API.t b/t/TestRail-API.t index a5cc5aa..80c1bdb 100644 --- a/t/TestRail-API.t +++ b/t/TestRail-API.t @@ -7,7 +7,7 @@ use lib "$FindBin::Bin/lib"; use TestRail::API; use Test::LWP::UserAgent::TestRailMock; -use Test::More tests => 87; +use Test::More tests => 88; use Test::Fatal; use Test::Deep; use Scalar::Util (); @@ -231,6 +231,9 @@ is_deeply(\@config_ids,\@t_config_ids, "Can correctly translate Project names to my $grp = $tr->addConfigurationGroup($new_project->{'id'},"zippy"); is($grp->{'name'},'zippy',"Can add configuration group successfully"); +my $fetchedgrp = $tr->getConfigurationGroupByName($new_project->{'id'},'zippy'); +is($fetchedgrp->{'id'},$grp->{'id'},"Can get configuration group by name"); + my $newgrp = $tr->editConfigurationGroup($grp->{'id'},"doodah"); is($newgrp->{'name'},'doodah',"Can edit configuration group successfully"); diff --git a/t/TestRail-Utils-Find.t b/t/TestRail-Utils-Find.t index 4a6f11a..e81d697 100644 --- a/t/TestRail-Utils-Find.t +++ b/t/TestRail-Utils-Find.t @@ -231,4 +231,6 @@ like(exception {TestRail::Utils::Find::findCases($opts,@$cases)},qr/Directory pa $opts->{'directory'} = 'bogoDir/'; like(exception {TestRail::Utils::Find::findCases($opts,@$cases)},qr/No such directory/i,"Bad directory being passed results in error"); +#XXX Deliberately omitting the tests for getResults. It's adequately covered (for now) by testrail-results unit test + #Test synchronize diff --git a/t/author-eol.t b/t/author-eol.t index cec6c61..0b668e8 100644 --- a/t/author-eol.t +++ b/t/author-eol.t @@ -19,6 +19,7 @@ my @files = ( 'bin/testrail-cases', 'bin/testrail-lock', 'bin/testrail-report', + 'bin/testrail-results', 'bin/testrail-runs', 'bin/testrail-tests', 'lib/App/Prove/Plugin/TestRail.pm', @@ -34,6 +35,7 @@ my @files = ( 't/App-Prove-Plugin-Testrail.t', 't/Test-Rail-Parser.t', 't/TestRail-API-mockOnly.t', + 't/TestRail-API-sections.t', 't/TestRail-API.t', 't/TestRail-Utils-Find.t', 't/TestRail-Utils-Lock.t', @@ -47,6 +49,7 @@ my @files = ( 't/author-pod-spell.t', 't/author-pod-syntax.t', 't/author-test-version.t', + 't/data/faketest_cache.json', 't/fake.tap', 't/fake.test', 't/faker.test', @@ -73,6 +76,7 @@ my @files = ( 't/testrail-cases.t', 't/testrail-lock.t', 't/testrail-report.t', + 't/testrail-results.t', 't/testrail-runs.t', 't/testrail-tests.t', 't/todo_pass.test', diff --git a/t/author-no-tabs.t b/t/author-no-tabs.t index ff7d1b3..0ae6ae7 100644 --- a/t/author-no-tabs.t +++ b/t/author-no-tabs.t @@ -19,6 +19,7 @@ my @files = ( 'bin/testrail-cases', 'bin/testrail-lock', 'bin/testrail-report', + 'bin/testrail-results', 'bin/testrail-runs', 'bin/testrail-tests', 'lib/App/Prove/Plugin/TestRail.pm', @@ -34,6 +35,7 @@ my @files = ( 't/App-Prove-Plugin-Testrail.t', 't/Test-Rail-Parser.t', 't/TestRail-API-mockOnly.t', + 't/TestRail-API-sections.t', 't/TestRail-API.t', 't/TestRail-Utils-Find.t', 't/TestRail-Utils-Lock.t', @@ -47,6 +49,7 @@ my @files = ( 't/author-pod-spell.t', 't/author-pod-syntax.t', 't/author-test-version.t', + 't/data/faketest_cache.json', 't/fake.tap', 't/fake.test', 't/faker.test', @@ -73,6 +76,7 @@ my @files = ( 't/testrail-cases.t', 't/testrail-lock.t', 't/testrail-report.t', + 't/testrail-results.t', 't/testrail-runs.t', 't/testrail-tests.t', 't/todo_pass.test', diff --git a/t/author-pod-spell.t b/t/author-pod-spell.t index 76af7bf..75d40e3 100644 --- a/t/author-pod-spell.t +++ b/t/author-pod-spell.t @@ -104,6 +104,9 @@ getPlanSummary getRunSummary judgements bailoutCallback +findResults +cachefile +getChildSections George Baugh teodesian diff --git a/t/data/faketest_cache.json b/t/data/faketest_cache.json new file mode 100644 index 0000000..ca22cd8 --- /dev/null +++ b/t/data/faketest_cache.json @@ -0,0 +1 @@ +{"t/fake.test":{"median_elapsed":1.5,"times_executed":1030,"stdev_elapsed":0.500242895326447,"average_elapsed":1.5,"num_runs":515,"max_elapsed":2,"Retest":515,"search_string":null,"total_elapsed":1545,"min_elapsed":1,"Failed":515,"seen_runs":[22,1815,1814,1813,1812,1811,1810,1809,1808,1807,1806,1805,1804,1803,1802,1801,1800,1799,1798,1797,1796,1795,1794,1793,1792,1791,1790,1789,1788,1787,1786,1785,1784,1783,1782,1781,1780,1779,1778,1777,1776,1775,1774,1773,1772,1771,1770,1769,1768,1767,1766,1765,1764,1763,1762,1761,1760,1759,1758,1757,1756,1755,1754,1753,1752,1751,1750,1749,1748,1747,1746,1745,1744,1743,1742,1741,1740,1739,1738,1737,1736,1735,1734,1733,1732,1731,1730,1729,1728,1727,1726,1725,1724,1723,1722,1721,1720,1719,1718,1717,1716,1715,1714,1713,1712,1711,1710,1709,1708,1707,1706,1705,1704,1703,1702,1701,1700,1699,1698,1697,1696,1695,1694,1693,1692,1691,1690,1689,1688,1687,1686,1685,1684,1683,1682,1681,1680,1679,1678,1677,1676,1675,1674,1673,1672,1671,1670,1669,1668,1667,1666,1665,1664,1663,1662,1661,1660,1659,1658,1657,1656,1655,1654,1653,1652,1651,1650,1649,1648,1647,1646,1645,1644,1643,1642,1641,1640,1639,1638,1637,1636,1635,1634,1633,1632,1631,1630,1629,1628,1627,1626,1625,1624,1623,1622,1621,1620,1619,1618,1617,1616,1615,1614,1613,1612,1611,1610,1609,1608,1607,1606,1605,1604,1603,1602,1601,1600,1599,1598,1597,1596,1595,1594,1593,1592,1591,1590,1589,1588,1587,1586,1585,1584,1583,1582,1581,1580,1579,1578,1577,1576,1575,1574,1573,1572,1571,1570,1569,1568,1567,1566,1562,"999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999","999",1,2,3,1099,9999,"999","999","999","999","999",32]}} \ No newline at end of file diff --git a/t/lib/Test/LWP/UserAgent/TestRailMock.pm b/t/lib/Test/LWP/UserAgent/TestRailMock.pm index 11dddb6..ec9bbec 100644 --- a/t/lib/Test/LWP/UserAgent/TestRailMock.pm +++ b/t/lib/Test/LWP/UserAgent/TestRailMock.pm @@ -170,7 +170,7 @@ $VAR4 = bless( { 'content-type' => 'application/json; charset=utf-8', 'server' => 'Apache/2.4.7 (Ubuntu)' }, 'HTTP::Headers' ); -$VAR5 = '[{"id":9,"name":"CRUSH ALL HUMANS","announcement":"Robo-Signed Soviet 5 Year Project","show_announcement":false,"is_completed":false,"completed_on":null,"suite_mode":3,"url":"http:\\/\\/testrail.local\\/\\/index.php?\\/projects\\/overview\\/9"},{"id":10,"name":"TestProject","is_completed":false}]'; +$VAR5 = '[{"id":9,"name":"CRUSH ALL HUMANS","announcement":"Robo-Signed Soviet 5 Year Project","show_announcement":false,"is_completed":false,"completed_on":null,"suite_mode":3,"url":"http:\\/\\/testrail.local\\/\\/index.php?\\/projects\\/overview\\/9"},{"id":10,"name":"TestProject","is_completed":false},{"id":3,"name":"zippy","announcement":null,"show_announcement":false,"is_completed":false,"completed_on":null,"suite_mode":2,"url":"http:\\/\\/testrail.local\\/index.php?\\/projects\\/overview\\/3"}]'; $mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, $VAR5)); } @@ -1431,7 +1431,7 @@ $VAR4 = bless( { 'content-type' => 'application/json; charset=utf-8', 'server' => 'Apache/2.4.7 (Ubuntu)' }, 'HTTP::Headers' ); -$VAR5 = '[{"display_order":1,"system_name":"custom_step_results","name":"step_results","description":"Step by step results","is_active":1,"type_id":11,"configs":[{"options":{"is_required":0,"format":"markdown","has_actual":1,"has_expected":1},"context":{"project_ids":[5],"is_global":1},"id":"43410543-edaf-44d2-91fc-58a6f9b3f743"},{"options":{"is_required":1,"format":"markdown","has_actual":1,"has_expected":1},"context":{"project_ids":[1],"is_global":1},"id":"0ab86184-0468-40d8-a385-a9b3a1ec41a4"},{"options":{"is_required":0,"format":"markdown","has_actual":1,"has_expected":1},"context":{"project_ids":[10],"is_global":1},"id":"43ebdf1f-c9b9-4b91-a729-5c9f21252f00"}],"id":6,"label":"Step Results"}]'; +$VAR5 = '[{"display_order":1,"system_name":"custom_step_results","name":"step_results","description":"Step by step results","is_active":1,"type_id":11,"configs":[{"options":{"is_required":0,"format":"markdown","has_actual":1,"has_expected":1},"context":{"project_ids":[5,3,9],"is_global":1},"id":"43410543-edaf-44d2-91fc-58a6f9b3f743"},{"options":{"is_required":1,"format":"markdown","has_actual":1,"has_expected":1},"context":{"project_ids":[1],"is_global":1},"id":"0ab86184-0468-40d8-a385-a9b3a1ec41a4"},{"options":{"is_required":0,"format":"markdown","has_actual":1,"has_expected":1},"context":{"project_ids":[10],"is_global":1},"id":"43ebdf1f-c9b9-4b91-a729-5c9f21252f00"}],"id":6,"label":"Step Results"}]'; $mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, $VAR5)); } @@ -1938,6 +1938,18 @@ $VAR5 = '[ "id": 2, "name": "Operating Systems", "project_id": 1 + }, + { + "configs": [ + { + "group_id": 3, + "id": 666, + "name": "noSuchConfig" + } + ], + "id": 3, + "name": "zippy", + "project_id": 1 } ]'; $mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, $VAR5)); @@ -2908,6 +2920,394 @@ $mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, } +######################## +# getChildSections mocks +######################## + +{ + +$VAR1 = 'index.php?/api/v2/get_suites/3'; +$VAR2 = '200'; +$VAR3 = 'OK'; +$VAR4 = bless( { + 'date' => 'Wed, 10 Aug 2016 03:07:51 GMT', + 'connection' => 'close', + 'x-powered-by' => 'PHP/5.5.9-1ubuntu4.14', + 'content-length' => '199', + '::std_case' => { + 'client-date' => 'Client-Date', + 'client-response-num' => 'Client-Response-Num', + 'client-peer' => 'Client-Peer', + 'x-powered-by' => 'X-Powered-By' + }, + 'client-date' => 'Wed, 10 Aug 2016 03:07:51 GMT', + 'client-response-num' => 1, + 'server' => 'Apache/2.4.7 (Ubuntu)', + 'client-peer' => '192.168.122.217:80', + 'content-type' => 'application/json; charset=utf-8' + }, 'HTTP::Headers' ); +$VAR5 = '[{"id":5,"name":"Master","description":null,"project_id":3,"is_master":true,"is_baseline":false,"is_completed":false,"completed_on":null,"url":"http:\\/\\/testrail.local\\/index.php?\\/suites\\/view\\/5"}]'; +$mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, $VAR5)); + +} + +{ + +$VAR1 = 'index.php?/api/v2/get_sections/3&suite_id=5'; +$VAR2 = '200'; +$VAR3 = 'OK'; +$VAR4 = bless( { + '::std_case' => { + 'client-response-num' => 'Client-Response-Num', + 'client-peer' => 'Client-Peer', + 'x-powered-by' => 'X-Powered-By', + 'client-date' => 'Client-Date' + }, + 'client-date' => 'Wed, 10 Aug 2016 03:07:52 GMT', + 'date' => 'Wed, 10 Aug 2016 03:07:51 GMT', + 'connection' => 'close', + 'x-powered-by' => 'PHP/5.5.9-1ubuntu4.14', + 'content-length' => '835', + 'content-type' => 'application/json; charset=utf-8', + 'client-peer' => '192.168.122.217:80', + 'client-response-num' => 1, + 'server' => 'Apache/2.4.7 (Ubuntu)' + }, 'HTTP::Headers' ); +$VAR5 = '[{"id":6,"suite_id":5,"name":"Column A","description":null,"parent_id":null,"display_order":1,"depth":0},{"id":8,"suite_id":5,"name":"zippy","description":null,"parent_id":6,"display_order":2,"depth":1},{"id":7,"suite_id":5,"name":"Column B","description":null,"parent_id":null,"display_order":3,"depth":0},{"id":9,"suite_id":5,"name":"zippy","description":null,"parent_id":7,"display_order":4,"depth":1},{"id":11,"suite_id":5,"name":"Recursing section","description":null,"parent_id":null,"display_order":5,"depth":0},{"id":12,"suite_id":5,"name":"child","description":null,"parent_id":11,"display_order":6,"depth":1},{"id":13,"suite_id":5,"name":"grandchild","description":null,"parent_id":12,"display_order":7,"depth":2},{"id":14,"suite_id":5,"name":"great-grandchild","description":null,"parent_id":13,"display_order":8,"depth":3}]'; +$mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, $VAR5)); + +} + +{ + +$VAR1 = 'index.php?/api/v2/get_sections/9&suite_id=5'; +$VAR2 = '200'; +$VAR3 = 'OK'; +$VAR4 = bless( { + '::std_case' => { + 'client-response-num' => 'Client-Response-Num', + 'client-peer' => 'Client-Peer', + 'x-powered-by' => 'X-Powered-By', + 'client-date' => 'Client-Date' + }, + 'client-date' => 'Wed, 10 Aug 2016 03:07:52 GMT', + 'date' => 'Wed, 10 Aug 2016 03:07:51 GMT', + 'connection' => 'close', + 'x-powered-by' => 'PHP/5.5.9-1ubuntu4.14', + 'content-length' => '835', + 'content-type' => 'application/json; charset=utf-8', + 'client-peer' => '192.168.122.217:80', + 'client-response-num' => 1, + 'server' => 'Apache/2.4.7 (Ubuntu)' + }, 'HTTP::Headers' ); +$VAR5 = '[{"id":6,"suite_id":5,"name":"Column A","description":null,"parent_id":null,"display_order":1,"depth":0},{"id":8,"suite_id":5,"name":"zippy","description":null,"parent_id":6,"display_order":2,"depth":1},{"id":7,"suite_id":5,"name":"Column B","description":null,"parent_id":null,"display_order":3,"depth":0},{"id":9,"suite_id":5,"name":"zippy","description":null,"parent_id":7,"display_order":4,"depth":1},{"id":11,"suite_id":5,"name":"Recursing section","description":null,"parent_id":null,"display_order":5,"depth":0},{"id":12,"suite_id":5,"name":"child","description":null,"parent_id":11,"display_order":6,"depth":1},{"id":13,"suite_id":5,"name":"grandchild","description":null,"parent_id":12,"display_order":7,"depth":2},{"id":14,"suite_id":5,"name":"great-grandchild","description":null,"parent_id":13,"display_order":8,"depth":3}]'; +$mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, $VAR5)); + +} + + +{ + +$VAR1 = 'index.php?/api/v2/get_configs/3'; +$VAR2 = '200'; +$VAR3 = 'OK'; +$VAR4 = bless( { + 'client-response-num' => 1, + 'content-length' => '2', + 'connection' => 'close', + 'content-type' => 'application/json; charset=utf-8', + '::std_case' => { + 'client-date' => 'Client-Date', + 'x-powered-by' => 'X-Powered-By', + 'client-peer' => 'Client-Peer', + 'client-response-num' => 'Client-Response-Num' + }, + 'client-peer' => '192.168.122.217:80', + 'server' => 'Apache/2.4.7 (Ubuntu)', + 'client-date' => 'Wed, 10 Aug 2016 03:39:47 GMT', + 'x-powered-by' => 'PHP/5.5.9-1ubuntu4.14', + 'date' => 'Wed, 10 Aug 2016 03:39:47 GMT' + }, 'HTTP::Headers' ); +$VAR5 = '[]'; +$mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, $VAR5)); + +} + +{ + +$VAR1 = 'index.php?/api/v2/get_runs/3&offset=0&limit=250'; +$VAR2 = '200'; +$VAR3 = 'OK'; +$VAR4 = bless( { + 'date' => 'Wed, 10 Aug 2016 03:39:47 GMT', + 'client-date' => 'Wed, 10 Aug 2016 03:39:47 GMT', + 'x-powered-by' => 'PHP/5.5.9-1ubuntu4.14', + 'server' => 'Apache/2.4.7 (Ubuntu)', + 'client-peer' => '192.168.122.217:80', + 'content-type' => 'application/json; charset=utf-8', + '::std_case' => { + 'client-peer' => 'Client-Peer', + 'client-response-num' => 'Client-Response-Num', + 'client-date' => 'Client-Date', + 'x-powered-by' => 'X-Powered-By' + }, + 'connection' => 'close', + 'content-length' => '588', + 'client-response-num' => 1 + }, 'HTTP::Headers' ); +$VAR5 = '[{"id":32,"suite_id":5,"name":"Master Shake","description":null,"milestone_id":null,"assignedto_id":null,"include_all":true,"is_completed":false,"completed_on":null,"config":null,"config_ids":[],"passed_count":1,"blocked_count":0,"untested_count":3,"retest_count":0,"failed_count":0,"custom_status1_count":0,"custom_status2_count":0,"custom_status3_count":0,"custom_status4_count":0,"custom_status5_count":0,"custom_status6_count":0,"custom_status7_count":0,"project_id":3,"plan_id":null,"created_on":1470345740,"created_by":1,"url":"http:\\/\\/testrail.local\\/index.php?\\/runs\\/view\\/32"}]'; +$mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, $VAR5)); + +} + +{ + +$VAR1 = 'index.php?/api/v2/get_sections/3&suite_id=5'; +$VAR2 = '200'; +$VAR3 = 'OK'; +$VAR4 = bless( { + 'date' => 'Wed, 10 Aug 2016 03:39:47 GMT', + 'server' => 'Apache/2.4.7 (Ubuntu)', + 'client-peer' => '192.168.122.217:80', + 'x-powered-by' => 'PHP/5.5.9-1ubuntu4.14', + 'client-date' => 'Wed, 10 Aug 2016 03:39:47 GMT', + '::std_case' => { + 'client-response-num' => 'Client-Response-Num', + 'client-peer' => 'Client-Peer', + 'x-powered-by' => 'X-Powered-By', + 'client-date' => 'Client-Date' + }, + 'content-type' => 'application/json; charset=utf-8', + 'client-response-num' => 1, + 'content-length' => '835', + 'connection' => 'close' + }, 'HTTP::Headers' ); +$VAR5 = '[{"id":6,"suite_id":5,"name":"Column A","description":null,"parent_id":null,"display_order":1,"depth":0},{"id":8,"suite_id":5,"name":"zippy","description":null,"parent_id":6,"display_order":2,"depth":1},{"id":7,"suite_id":5,"name":"Column B","description":null,"parent_id":null,"display_order":3,"depth":0},{"id":9,"suite_id":5,"name":"zippy","description":null,"parent_id":7,"display_order":4,"depth":1},{"id":11,"suite_id":5,"name":"Recursing section","description":null,"parent_id":null,"display_order":5,"depth":0},{"id":12,"suite_id":5,"name":"child","description":null,"parent_id":11,"display_order":6,"depth":1},{"id":13,"suite_id":5,"name":"grandchild","description":null,"parent_id":12,"display_order":7,"depth":2},{"id":14,"suite_id":5,"name":"great-grandchild","description":null,"parent_id":13,"display_order":8,"depth":3}]'; +$mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, $VAR5)); + +} + +{ + +$VAR1 = 'index.php?/api/v2/get_cases/3&suite_id=5§ion_id=11'; +$VAR2 = '200'; +$VAR3 = 'OK'; +$VAR4 = bless( { + 'date' => 'Wed, 10 Aug 2016 03:39:47 GMT', + 'server' => 'Apache/2.4.7 (Ubuntu)', + 'client-peer' => '192.168.122.217:80', + 'x-powered-by' => 'PHP/5.5.9-1ubuntu4.14', + 'client-date' => 'Wed, 10 Aug 2016 03:39:48 GMT', + '::std_case' => { + 'x-powered-by' => 'X-Powered-By', + 'client-date' => 'Client-Date', + 'client-response-num' => 'Client-Response-Num', + 'client-peer' => 'Client-Peer' + }, + 'content-type' => 'application/json; charset=utf-8', + 'client-response-num' => 1, + 'content-length' => '2', + 'connection' => 'close' + }, 'HTTP::Headers' ); +$VAR5 = '[]'; +$mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, $VAR5)); + +} + +{ + +$VAR1 = 'index.php?/api/v2/get_cases/3&suite_id=5§ion_id=14'; +$VAR2 = '200'; +$VAR3 = 'OK'; +$VAR4 = bless( { + 'date' => 'Wed, 10 Aug 2016 03:39:48 GMT', + 'client-date' => 'Wed, 10 Aug 2016 03:39:48 GMT', + 'x-powered-by' => 'PHP/5.5.9-1ubuntu4.14', + 'client-peer' => '192.168.122.217:80', + 'server' => 'Apache/2.4.7 (Ubuntu)', + 'content-type' => 'application/json; charset=utf-8', + '::std_case' => { + 'client-peer' => 'Client-Peer', + 'client-response-num' => 'Client-Response-Num', + 'client-date' => 'Client-Date', + 'x-powered-by' => 'X-Powered-By' + }, + 'connection' => 'close', + 'content-length' => '321', + 'client-response-num' => 1 + }, 'HTTP::Headers' ); +$VAR5 = '[{"id":16,"title":"pass.test","section_id":14,"template_id":1,"type_id":6,"priority_id":4,"milestone_id":null,"refs":null,"created_by":1,"created_on":1470799296,"updated_by":1,"updated_on":1470799296,"estimate":null,"estimate_forecast":null,"suite_id":5,"custom_preconds":null,"custom_steps":null,"custom_expected":null}]'; +$mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, $VAR5)); + +} + +{ + +$VAR1 = 'index.php?/api/v2/get_cases/3&suite_id=5§ion_id=12'; +$VAR2 = '200'; +$VAR3 = 'OK'; +$VAR4 = bless( { + 'date' => 'Wed, 10 Aug 2016 03:39:48 GMT', + 'client-peer' => '192.168.122.217:80', + 'server' => 'Apache/2.4.7 (Ubuntu)', + 'x-powered-by' => 'PHP/5.5.9-1ubuntu4.14', + 'client-date' => 'Wed, 10 Aug 2016 03:39:48 GMT', + '::std_case' => { + 'client-response-num' => 'Client-Response-Num', + 'client-peer' => 'Client-Peer', + 'x-powered-by' => 'X-Powered-By', + 'client-date' => 'Client-Date' + }, + 'content-type' => 'application/json; charset=utf-8', + 'client-response-num' => 1, + 'content-length' => '321', + 'connection' => 'close' + }, 'HTTP::Headers' ); +$VAR5 = '[{"id":17,"title":"fake.test","section_id":12,"template_id":1,"type_id":6,"priority_id":4,"milestone_id":null,"refs":null,"created_by":1,"created_on":1470799305,"updated_by":1,"updated_on":1470799305,"estimate":null,"estimate_forecast":null,"suite_id":5,"custom_preconds":null,"custom_steps":null,"custom_expected":null}]'; +$mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, $VAR5)); + +} + +{ + +$VAR1 = 'index.php?/api/v2/get_cases/3&suite_id=5§ion_id=13'; +$VAR2 = '200'; +$VAR3 = 'OK'; +$VAR4 = bless( { + 'date' => 'Wed, 10 Aug 2016 03:39:48 GMT', + 'client-peer' => '192.168.122.217:80', + 'server' => 'Apache/2.4.7 (Ubuntu)', + 'client-date' => 'Wed, 10 Aug 2016 03:39:48 GMT', + 'x-powered-by' => 'PHP/5.5.9-1ubuntu4.14', + 'content-type' => 'application/json; charset=utf-8', + '::std_case' => { + 'client-date' => 'Client-Date', + 'x-powered-by' => 'X-Powered-By', + 'client-peer' => 'Client-Peer', + 'client-response-num' => 'Client-Response-Num' + }, + 'client-response-num' => 1, + 'connection' => 'close', + 'content-length' => '321' + }, 'HTTP::Headers' ); +$VAR5 = '[{"id":18,"title":"skip.test","section_id":13,"template_id":1,"type_id":6,"priority_id":4,"milestone_id":null,"refs":null,"created_by":1,"created_on":1470799317,"updated_by":1,"updated_on":1470799317,"estimate":null,"estimate_forecast":null,"suite_id":5,"custom_preconds":null,"custom_steps":null,"custom_expected":null}]'; +$mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, $VAR5)); + +} + +{ + +$VAR1 = 'index.php?/api/v2/add_run/3'; +$VAR2 = '200'; +$VAR3 = 'OK'; +$VAR4 = bless( { + 'date' => 'Wed, 10 Aug 2016 03:39:48 GMT', + 'x-powered-by' => 'PHP/5.5.9-1ubuntu4.14', + 'client-date' => 'Wed, 10 Aug 2016 03:39:48 GMT', + 'client-peer' => '192.168.122.217:80', + 'server' => 'Apache/2.4.7 (Ubuntu)', + '::std_case' => { + 'client-peer' => 'Client-Peer', + 'client-response-num' => 'Client-Response-Num', + 'client-date' => 'Client-Date', + 'x-powered-by' => 'X-Powered-By' + }, + 'content-type' => 'application/json; charset=utf-8', + 'connection' => 'close', + 'content-length' => '625', + 'client-response-num' => 1 + }, 'HTTP::Headers' ); +$VAR5 = '{"id":36,"suite_id":5,"name":"zippyRun","description":"Automatically created Run from TestRail::API","milestone_id":null,"assignedto_id":null,"include_all":false,"is_completed":false,"completed_on":null,"config":null,"config_ids":[],"passed_count":0,"blocked_count":0,"untested_count":3,"retest_count":0,"failed_count":0,"custom_status1_count":0,"custom_status2_count":0,"custom_status3_count":0,"custom_status4_count":0,"custom_status5_count":0,"custom_status6_count":0,"custom_status7_count":0,"project_id":3,"plan_id":null,"created_on":1470800388,"created_by":1,"url":"http:\\/\\/testrail.local\\/index.php?\\/runs\\/view\\/36"}'; +$mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, $VAR5)); + +} + +{ + +$VAR1 = 'index.php?/api/v2/get_tests/36'; +$VAR2 = '200'; +$VAR3 = 'OK'; +$VAR4 = bless( { + 'date' => 'Wed, 10 Aug 2016 03:39:49 GMT', + 'client-date' => 'Wed, 10 Aug 2016 03:39:49 GMT', + 'x-powered-by' => 'PHP/5.5.9-1ubuntu4.14', + 'server' => 'Apache/2.4.7 (Ubuntu)', + 'client-peer' => '192.168.122.217:80', + 'content-type' => 'application/json; charset=utf-8', + '::std_case' => { + 'client-response-num' => 'Client-Response-Num', + 'client-peer' => 'Client-Peer', + 'x-powered-by' => 'X-Powered-By', + 'client-date' => 'Client-Date' + }, + 'content-length' => '820', + 'connection' => 'close', + 'client-response-num' => 1 + }, 'HTTP::Headers' ); +$VAR5 = '[{"id":43,"case_id":17,"status_id":3,"assignedto_id":null,"run_id":36,"title":"fake.test","template_id":1,"type_id":6,"priority_id":4,"estimate":null,"estimate_forecast":null,"refs":null,"milestone_id":null,"custom_preconds":null,"custom_steps":null,"custom_expected":null},{"id":44,"case_id":18,"status_id":3,"assignedto_id":null,"run_id":36,"title":"skip.test","template_id":1,"type_id":6,"priority_id":4,"estimate":null,"estimate_forecast":null,"refs":null,"milestone_id":null,"custom_preconds":null,"custom_steps":null,"custom_expected":null},{"id":42,"case_id":16,"status_id":3,"assignedto_id":null,"run_id":36,"title":"pass.test","template_id":1,"type_id":6,"priority_id":4,"estimate":null,"estimate_forecast":null,"refs":null,"milestone_id":null,"custom_preconds":null,"custom_steps":null,"custom_expected":null}]'; +$mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, $VAR5)); + +} + +{ + +$VAR1 = 'index.php?/api/v2/add_result/42'; +$VAR2 = '200'; +$VAR3 = 'OK'; +$VAR4 = bless( { + 'client-response-num' => 1, + 'connection' => 'close', + 'content-length' => '194', + 'content-type' => 'application/json; charset=utf-8', + '::std_case' => { + 'client-response-num' => 'Client-Response-Num', + 'client-peer' => 'Client-Peer', + 'x-powered-by' => 'X-Powered-By', + 'client-date' => 'Client-Date' + }, + 'client-peer' => '192.168.122.217:80', + 'server' => 'Apache/2.4.7 (Ubuntu)', + 'client-date' => 'Wed, 10 Aug 2016 03:39:49 GMT', + 'x-powered-by' => 'PHP/5.5.9-1ubuntu4.14', + 'date' => 'Wed, 10 Aug 2016 03:39:49 GMT' + }, 'HTTP::Headers' ); +$VAR5 = '{"id":516,"test_id":42,"status_id":1,"created_by":1,"created_on":1470800389,"assignedto_id":null,"comment":"[22:39:48 Aug 9 2016 (0s)] ok 1 - yay!","version":null,"elapsed":null,"defects":null}'; +$mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, $VAR5)); + +} + +{ + +$VAR1 = 'index.php?/api/v2/get_plans/3'; +$VAR2 = '200'; +$VAR3 = 'OK'; +$VAR4 = bless( { + 'connection' => 'close', + 'x-powered-by' => 'PHP/5.5.9-1ubuntu4.5', + 'client-response-num' => 1, + 'date' => 'Tue, 23 Dec 2014 20:02:10 GMT', + 'client-peer' => '192.168.122.217:80', + 'content-length' => '554', + '::std_case' => { + 'client-date' => 'Client-Date', + 'x-powered-by' => 'X-Powered-By', + 'client-response-num' => 'Client-Response-Num', + 'client-peer' => 'Client-Peer' + }, + 'client-date' => 'Tue, 23 Dec 2014 20:02:10 GMT', + 'content-type' => 'application/json; charset=utf-8', + 'server' => 'Apache/2.4.7 (Ubuntu)' + }, 'HTTP::Headers' ); +$VAR5 = '[]'; +$mockObject->map_response(qr/\Q$VAR1\E/,HTTP::Response->new($VAR2, $VAR3, $VAR4, $VAR5)); + +} + + +########### +#Lock mocks +########### + sub lockMockStep0 { $VAR1 = 'index.php?/api/v2/get_tests/1099'; diff --git a/t/testrail-results.t b/t/testrail-results.t new file mode 100644 index 0000000..e729e11 --- /dev/null +++ b/t/testrail-results.t @@ -0,0 +1,123 @@ +use strict; +use warnings; + +use FindBin; + +use lib $FindBin::Bin.'/../bin'; +require 'testrail-results'; + +use lib $FindBin::Bin.'/lib'; +use Test::LWP::UserAgent::TestRailMock; + +use Test::More 'tests' => 22; +use Capture::Tiny qw{capture_merged}; + +no warnings qw{redefine once}; +*TestRail::API::getTests = sub { + my ($self,$run_id) = @_; + return [ + { + 'id' => 666, + 'title' => 'fake.test', + 'run_id' => $run_id + } + ]; +}; + +*TestRail::API::getTestResults = sub { + return [ + { + 'elapsed' => '1s', + 'status_id' => 5 + }, + { + 'elapsed' => '2s', + 'status_id' => 4, + 'comment' => 'zippy' + } + ]; +}; + +*TestRail::API::getPlanByID = sub { + return { + 'id' => 40000, + 'name' => 'mah dubz plan', + 'entries' => [{ + 'runs' => [ + { + 'name' => 'planrun', + 'id' => '999', + 'plan_id' => 40000 + } + ] + }] + }; +}; + +use warnings; + +#check doing things over all projects/plans/runs +my @args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake t/fake.test }; +my ($out,$code) = TestRail::Bin::Results::run('browser' => $Test::LWP::UserAgent::TestRailMock::mockObject, 'args' => \@args); +is($code, 0, "Exit code OK looking for results of fake.test"); +like($out,qr/fake\.test was present in 515 runs/,"Gets correct # of runs with test inside it"); + +#check project filters +@args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake --project TestProject t/fake.test }; +($out,$code) = TestRail::Bin::Results::run('browser' => $Test::LWP::UserAgent::TestRailMock::mockObject, 'args' => \@args); +is($code, 0, "Exit code OK looking for results of fake.test"); +like($out,qr/fake\.test was present in 10 runs/,"Gets correct # of runs with test inside it when filtering by project name"); + +#check plan filters +@args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake --plan }; +push(@args,'mah dubz plan', 't/fake.test'); +($out,$code) = TestRail::Bin::Results::run('browser' => $Test::LWP::UserAgent::TestRailMock::mockObject, 'args' => \@args); +is($code, 0, "Exit code OK looking for results of fake.test"); +like($out,qr/fake\.test was present in 258 runs/,"Gets correct # of runs with test inside it when filtering by plan name"); + +#check run filters +@args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake --run FinalRun t/fake.test}; +($out,$code) = TestRail::Bin::Results::run('browser' => $Test::LWP::UserAgent::TestRailMock::mockObject, 'args' => \@args); +is($code, 0, "Exit code OK looking for results of fake.test"); +like($out,qr/fake\.test was present in 1 runs/,"Gets correct # of runs with test inside it when filtering by run name"); + +#check pattern filters +@args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake --grep zippy t/fake.test}; +($out,$code) = TestRail::Bin::Results::run('browser' => $Test::LWP::UserAgent::TestRailMock::mockObject, 'args' => \@args); +is($code, 0, "Exit code OK looking for results of fake.test"); +like($out,qr/Retest: 515/,"Gets correct # & status of runs with test inside it when grepping"); +unlike($out,qr/Failed: 515/,"Gets correct # & status of runs with test inside it when grepping"); + +@args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake --json t/fake.test }; +($out,$code) = TestRail::Bin::Results::run('browser' => $Test::LWP::UserAgent::TestRailMock::mockObject, 'args' => \@args); +is($code, 0, "Exit code OK looking for results of fake.test in json mode"); +like($out,qr/num_runs/,"Gets # of runs with test inside it in json mode"); + +#For making the test data to test the caching +#open(my $fh, '>', "t/data/faketest_cache.json"); +#print $fh $out; +#close($fh); + +#Check caching +@args = qw{--apiurl http://testrail.local --user test@fake.fake --password fake --json --cachefile t/data/faketest_cache.json t/fake.test }; +($out,$code) = TestRail::Bin::Results::run('browser' => $Test::LWP::UserAgent::TestRailMock::mockObject, 'args' => \@args); +is($code, 0, "Exit code OK looking for results of fake.test in json mode"); +chomp $out; +is($out,"{}","Caching mode works"); + +#Check time parser +is(TestRail::Bin::Results::_elapsed2secs('1s'),1,"elapsed2secs works : seconds"); +is(TestRail::Bin::Results::_elapsed2secs('1m'),60,"elapsed2secs works : minutes"); +is(TestRail::Bin::Results::_elapsed2secs('1h'),3600,"elapsed2secs works : hours"); +is(TestRail::Bin::Results::_elapsed2secs('1s1m1h'),3661,"elapsed2secs works :smh"); + +#Check help output +@args = qw{--help}; +$0 = $FindBin::Bin.'/../bin/testrail-runs'; +($out,(undef,$code)) = capture_merged {TestRail::Bin::Results::run('args' => \@args)}; +is($code, 0, "Exit code OK asking for help"); +like($out,qr/encoding of arguments/i,"Help output OK"); + +#Make sure that the binary itself processes args correctly +$out = `$^X $0 --help`; +like($out,qr/encoding of arguments/i,"Appears we can run binary successfully");