Add repeat parameter to clone a job multiple times

to better support statistical investigation, we want to provide an
immediate way to clone a job multiple times. Till now it has been done
with a for loop similar to

for i in {01..50}; do openqa-clone-job … TEST+=-$i; done

But we can avoid the API overhead of copying the same parameters and
assets again and again from the original job, simply by cloning it once and
posting the new job(s) N times. TEST name is automatically numbered, so the
user is not forced to remember any shell syntax.

As the default is --repeat=1, the API remains compatible with any shell
script using the openqa-clone-job program.

docs/UsersGuide.asciidoc

@@ -1113,6 +1113,25 @@ timeouts by performing the task in background.
 This is recommended on big instances but means that the results (and
 possible errors) need to be polled via `openqa-cli api isos/$scheduled_product_id`.
 
+
+===== Statistical investigation ===== 
+In case issues appear sporadically and are therefore hard to reproduce it can 
+help to trigger many more jobs on a production instance to gather more data 
+first, for example the failure ratio.
+
+Example of triggering 50 jobs in a development group so that the result of 
+passed/failed jobs is counted by openQA itself on the corresponding overview page:
+[source,sh]
+--------------------------------------------------------------------------------
+openqa-clone-job --skip-chained-deps --repeat=50 --within-instance \
+https://openqa.opensuse.org 123456  BUILD=poo32242_investigation \
+_GROUP="Test Development:openSUSE Tumbleweed" 
+--------------------------------------------------------------------------------
+
+To get an overview about the fail ratio and confidence interval of sporadically 
+failing applications you can also use a script like 
+https://github.com/okurz/scripts/blob/master/count_fail_ratio[this].
+
 [id="scenarios_yaml"]
 ===== Defining test scenarios in YAML =====
 Instead of relying on the tables for machines, mediums/products, test suites and

lib/OpenQA/Script/CloneJob.pm

@@ -226,11 +226,32 @@ sub handle_tx ($tx, $url_handler, $options, $jobs) {
     }
 }
 
+# append a formatted "-NN" to the TEST parameter
+sub append_idx_to_test_name ($n, $post_params) {
+    my $number_suffix = sprintf("-%03d", $n);
+    foreach my $job_key (keys %$post_params) {
+        my $job = $post_params->{$job_key};
+        my $test_name = $job->{TEST};
+        if ($n == 1) {
+            $test_name .= $number_suffix;    # just append at the first time
+        }
+        else {
+            # from the second onwards, replace the suffix
+            substr($test_name, -length($number_suffix)) = $number_suffix;
+        }
+        $job->{TEST} = $test_name;
+    }
+}
+
 sub clone_jobs ($jobid, $options) {
     my $url_handler = create_url_handler($options);
+    my $repeat = delete $options->{'repeat'} || 1;
     clone_job($jobid, $url_handler, $options, my $post_params = {}, my $jobs = {});
-    my $tx = post_jobs($post_params, $url_handler, $options);
-    handle_tx($tx, $url_handler, $options, $jobs) if $tx;
+    for my $counter (1 .. $repeat) {
+        append_idx_to_test_name($counter, $post_params) if $repeat > 1;
+        my $tx = post_jobs($post_params, $url_handler, $options);
+        handle_tx($tx, $url_handler, $options, $jobs) if $tx;
+    }
 }
 
 sub clone_job ($jobid, $url_handler, $options, $post_params = {}, $jobs = {}, $depth = 1, $relation = '') {

script/openqa-clone-job

@@ -133,6 +133,10 @@ are cloned. Use C<0> to denote infinity.
 A shortcut for C<--skip-download --from HOST --host HOST> to clone a job within
 a local or remote instance.
 
+=item B<--repeat> N
+
+Do the same operation N times. Allowed values are from 1 to 999
+
 =item B<--show-progress>
 
 Displays a progress bar when downloading assets.
@@ -190,7 +194,7 @@ sub parse_options () {
         "apikey:s", "apisecret:s", "verbose|v", "json-output|j",
         "skip-deps", "skip-chained-deps", "skip-download", "parental-inheritance",
         "help|h", "show-progress", "within-instance|w=s", "clone-children",
-        "max-depth:i", "ignore-missing-assets", "export-command",
+        "max-depth:i", "repeat|r:i", "ignore-missing-assets", "export-command",
     ) or usage(1);
     usage(0) if $options{help};
     usage(1) if $options{help} || ($options{'within-instance'} && $options{from});
@@ -207,6 +211,8 @@ sub parse_options () {
     die "missing job reference, see --help for usage\n" unless $jobid;
     ($options{'from'}, $jobid) = split_jobid($jobid) unless $options{'from'};
     usage(1) unless ($jobid && $options{'from'});
+    $options{'repeat'} ||= 1;
+    die "invalid repeat count, should be between 1 and 999" if ($options{'repeat'} < 1 or $options{'repeat'} > 999);
     $options{'dir'} ||= assetdir();
     $options{'host'} ||= 'localhost';
     $options{'args'} = \@ARGV;

t/35-script_clone_job.t

@@ -244,6 +244,27 @@ subtest 'export command' => sub {
     combined_like { clone_jobs(42, \%options) } $expected_cmd, 'openqa-cli command printed';
 };
 
+subtest 'cloning with repeat count' => sub {
+    my $ua_mock = Test::MockModule->new('Mojo::UserAgent');
+    my $clone_mock = Test::MockModule->new('OpenQA::Script::CloneJob');
+    my @post_args;
+    my $tx_handled = 0;
+    $ua_mock->redefine(post => sub { push @post_args, [@_] });
+    $clone_mock->redefine(handle_tx => sub ($tx, $url_handler, $options, $jobs) { $tx_handled = 1 });
+    my %fake_jobs = (42 => {id => 42, name => 'myjob', settings => {TEST => 'myjob'}, children => {Parallel => [42]}});
+    $clone_mock->redefine(clone_job_get_job => sub ($job_id, @args) { $fake_jobs{$job_id} });
+    my %options = (host => 'foo', from => 'bar', repeat => 3, apikey => 'bar', apisecret => 'bar');
+    OpenQA::Script::CloneJob::clone_jobs(42, \%options);
+    is $options{repeat}, undef, 'repeat count has been removed from options';
+    subtest 'post args' => sub {
+        is scalar @post_args, 3, 'exactly 3 post calls made';
+        # check test name ends with -NNN
+        is $post_args[0]->[3]->{'TEST:42'}, 'myjob-001', '1st index has been appended to test name';
+        is $post_args[1]->[3]->{'TEST:42'}, 'myjob-002', '2nd index has been appended to test name';
+        is $post_args[2]->[3]->{'TEST:42'}, 'myjob-003', '3rd index has been appended to test name';
+    } or diag explain \@post_args;
+};
+
 subtest 'overall cloning with parallel and chained dependencies' => sub {
     # do not actually post any jobs, assume jobs can be cloned (clone ID = original ID + 100)
     my $ua_mock = Test::MockModule->new('Mojo::UserAgent');