/
bisect-runner.pl
executable file
·3863 lines (3305 loc) · 127 KB
/
bisect-runner.pl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/perl -w
use strict;
use Getopt::Long qw(:config bundling no_auto_abbrev);
use Pod::Usage;
use Config;
use File::Temp qw(tempdir);
use File::Spec;
my @targets
= qw(none config.sh config.h miniperl lib/Config.pm Fcntl perl test_prep);
my %options =
(
'expect-pass' => 1,
clean => 1, # mostly for debugging this
);
# We accept #!./miniperl and #!./perl
# We don't accept #!miniperl and #!perl as their intent is ambiguous
my $run_with_our_perl = qr{\A#!(\./(?:mini)?perl)\b};
my $linux64 = `uname -sm` eq "Linux x86_64\n" ? '64' : '';
my @paths;
if ($^O eq 'linux') {
# This is the search logic for a multi-arch library layout
# added to linux.sh in commits 40f026236b9959b7 and dcffd848632af2c7.
my $gcc = -x '/usr/bin/gcc' ? '/usr/bin/gcc' : 'gcc';
foreach (`$gcc -print-search-dirs`) {
next unless /^libraries: =(.*)/;
foreach (split ':', $1) {
next if m/gcc/;
next unless -d $_;
s!/$!!;
push @paths, $_;
}
}
push @paths, map {$_ . $linux64} qw(/usr/local/lib /lib /usr/lib)
if $linux64;
}
my %defines =
(
usedevel => '',
optimize => '-g',
ld => 'cc',
(@paths ? (libpth => \@paths) : ()),
);
# Needed for the 'ignore_versioned_solibs' emulation below.
push @paths, qw(/usr/local/lib /lib /usr/lib)
unless $linux64;
unless(GetOptions(\%options,
'target=s', 'make=s', 'jobs|j=i', 'crash', 'expect-pass=i',
'expect-fail' => sub { $options{'expect-pass'} = 0; },
'clean!', 'one-liner|e=s@', 'c', 'l', 'w', 'match=s',
'no-match=s' => sub {
$options{match} = $_[1];
$options{'expect-pass'} = 0;
},
'force-manifest', 'force-regen', 'setpgrp!', 'timeout=i',
'test-build', 'validate',
'all-fixups', 'early-fixup=s@', 'late-fixup=s@', 'valgrind',
'check-args', 'check-shebang!', 'usage|help|?', 'gold=s',
'module=s', 'with-module=s', 'cpan-config-dir=s',
'test-module=s', 'no-module-tests',
'A=s@',
'D=s@' => sub {
my (undef, $val) = @_;
if ($val =~ /\A([^=]+)=(.*)/s) {
$defines{$1} = length $2 ? $2 : "\0";
} else {
$defines{$val} = '';
}
},
'U=s@' => sub {
$defines{$_[1]} = undef;
},
)) {
pod2usage(exitval => 255, verbose => 1);
}
my ($target, $match) = @options{qw(target match)};
@ARGV = ('sh', '-c', 'cd t && ./perl TEST base/*.t')
if $options{validate} && !@ARGV;
pod2usage(exitval => 0, verbose => 2) if $options{usage};
# This needs to be done before the next arguments check, as it's populating
# @ARGV
if (defined $target && $target =~ /\.t\z/) {
# t/TEST don't have a reliable way to run the test script under valgrind
# The $ENV{VALGRIND} code was only added after v5.8.0, and is more
# geared to logging than to exiting on failure if errors are found.
# I guess one could fudge things by replacing the symlink t/perl with a
# wrapper script which invokes valgrind, but leave doing that until
# someone needs it. (If that's you, then patches welcome.)
foreach (qw(valgrind match validate test-build one-liner)) {
die_255("$0: Test-case targets can't be run with --$_")
if $options{$_};
}
die_255("$0: Test-case targets can't be combined with an explicit test")
if @ARGV;
# Needing this unless is a smell suggesting that this implementation of
# test-case targets is not really in the right place.
unless ($options{'check-args'}) {
# The top level sanity tests refuse to start or end a test run at a
# revision which skips, hence this test ensures reasonable sanity at
# automatically picking a suitable start point for both normal operation
# and --expect-fail
skip("Test case $target is not a readable file")
unless -f $target && -r _;
}
# t/TEST runs from and takes pathnames relative to t/, so need to strip
# out a leading t, or add ../ otherwise
unless ($target =~ s!\At/!!) {
$target = "../$target";
}
@ARGV = ('sh', '-c', "cd t && ./perl TEST " . quotemeta $target);
$target = 'test_prep';
}
pod2usage(exitval => 255, verbose => 1)
unless @ARGV || $match || $options{'test-build'}
|| defined $options{'one-liner'} || defined $options{module}
|| defined $options{'test-module'};
pod2usage(exitval => 255, verbose => 1)
if !$options{'one-liner'} && ($options{l} || $options{w});
if ($options{'no-module-tests'} && $options{module}) {
print STDERR "--module and --no-module-tests are exclusive.\n\n";
pod2usage(exitval => 255, verbose => 1)
}
if ($options{'no-module-tests'} && $options{'test-module'}) {
print STDERR "--test-module and --no-module-tests are exclusive.\n\n";
pod2usage(exitval => 255, verbose => 1)
}
if ($options{module} && $options{'test-module'}) {
print STDERR "--module and --test-module are exclusive.\n\n";
pod2usage(exitval => 255, verbose => 1)
}
check_shebang($ARGV[0])
if $options{'check-shebang'} && @ARGV && !$options{match};
exit 0 if $options{'check-args'};
=head1 NAME
bisect.pl - use git bisect to pinpoint changes
=head1 SYNOPSIS
# When did this become an error?
.../Porting/bisect.pl -e 'my $a := 2;'
# When did this stop being an error?
.../Porting/bisect.pl --expect-fail -e '1 // 2'
# When did this test start failing?
.../Porting/bisect.pl --target t/op/sort.t
# When were all lines matching this pattern removed from all files?
.../Porting/bisect.pl --match '\b(?:PL_)hash_seed_set\b'
# When was some line matching this pattern added to some file?
.../Porting/bisect.pl --expect-fail --match '\buseithreads\b'
# When did this test program stop exiting 0?
.../Porting/bisect.pl -- ./perl -Ilib ../test_prog.pl
# When did this test program start crashing (any signal or coredump)?
.../Porting/bisect.pl --crash -- ./perl -Ilib ../test_prog.pl
# When did this first become valid syntax?
.../Porting/bisect.pl --target=miniperl --end=v5.10.0 \
--expect-fail -e 'my $a := 2;'
# What was the last revision to build with these options?
.../Porting/bisect.pl --test-build -Dd_dosuid
# When did this test program start generating errors from valgrind?
.../Porting/bisect.pl --valgrind ../test_prog.pl
# When did these cpan modules start failing to compile/pass tests?
.../Porting/bisect.pl --module=autobox,Moose
# When did this code stop working in blead with these modules?
.../Porting/bisect.pl --with-module=Moose,Moo -e 'use Moose; 1;'
# Like the above 2 but with custom CPAN::MyConfig
.../Porting/bisect.pl --module=Moo --cpan-config-dir=/home/blah/custom/
=head1 DESCRIPTION
Together F<bisect.pl> and F<bisect-runner.pl> attempt to automate the use
of C<git bisect> as much as possible. With one command (and no other files)
it's easy to find out
=over 4
=item *
Which commit caused this example code to break?
=item *
Which commit caused this example code to start working?
=item *
Which commit added the first file to match this regex?
=item *
Which commit removed the last file to match this regex?
=back
usually without needing to know which versions of perl to use as start and
end revisions.
By default F<bisect.pl> will process all options, then use the rest of the
command line as arguments to list C<system> to run a test case. By default,
the test case should pass (exit with 0) on earlier perls, and fail (exit
non-zero) on I<blead>. F<bisect.pl> will use F<bisect-runner.pl> to find the
earliest stable perl version on which the test case passes, check that it
fails on blead, and then use F<bisect-runner.pl> with C<git bisect run> to
find the commit which caused the failure.
Many of perl's own test scripts exit 0 even if their TAP reports test
failures, and some need particular setup (such as running from the right
directory, or adding C<-T> to the command line). Hence if you want to bisect
a test script, you can specify it with the I<--target> option, and it will
be invoked using F<t/TEST> which performs all the setup, and exits non-zero
if the TAP reports failures. This works for any file ending C<.t>, so you can
use it with a file outside of the working checkout, for example to test a
particular version of a test script, as a path inside the repository will
(of course) be testing the version of the script checked out for the current
revision, which may be too early to have the test you are interested in.
Because the test case is the complete argument to C<system>, it is easy to
run something other than the F<perl> built, if necessary. If you need to run
the perl built, you'll probably need to invoke it as C<./perl -Ilib ...>.
As a special case, if the first argument of the test case is a readable file
(whether executable or not), matching C<qr{\A#!./(?:mini)?perl\b}> then it
will have C<./perl> <-Ilib> (or C<./miniperl>) prepended to it.
You need a clean checkout to run a bisect. You can use the checkout
containing F<Porting/bisect.pl> if you wish - in this case
F<Porting/bisect.pl> will copy F<Porting/bisect-runner.pl> to a temporary
file generated by C<File::Temp::tempfile()>. If doing this, beware that when
the bisect ends (or you abort it) then your checkout is no longer at
C<blead>, so you will need to C<git checkout blead> before restarting, to
get the current version of F<Porting/bisect.pl> again. It's often easier
either to copy F<Porting/bisect.pl> and F<Porting/bisect-runner.pl> to
another directory (I<e.g.> F<~/bin>, if you have one), or to create a second
git repository for running bisect. To create a second local repository, if
your working checkout is called F<perl>, a simple solution is to make a
local clone, and run from that. I<i.e.>:
cd ..
git clone perl perl2
cd perl2
../perl/Porting/bisect.pl ...
By default, F<bisect-runner.pl> will automatically disable the build of
L<DB_File> for commits earlier than ccb44e3bf3be2c30, as it's not practical
to patch DB_File 1.70 and earlier to build with current Berkeley DB headers.
(ccb44e3bf3be2c30 was in September 1999, between 5.005_62 and 5.005_63.)
If your F<db.h> is old enough you can override this with C<-Unoextensions>.
=head1 OPTIONS
=over 4
=item *
--start I<commit-ish>
Earliest revision to test, as a I<commit-ish> (a tag, commit or anything
else C<git> understands as a revision). If not specified, F<bisect.pl> will
search stable .0 perl releases until it finds one where the test case
passes. The default is to search from 5.002 to the most recent tagged stable
release (v5.18.0 at the time of writing). If F<bisect.pl> detects that the
checkout is on a case insensitive file system, it will search from 5.005 to
the most recent tagged stable release. Only .0 stable releases are used
because these are the only stable releases that are parents of blead, and
hence suitable for a bisect run.
=item *
--end I<commit-ish>
Most recent revision to test, as a I<commit-ish>. If not specified, defaults
to I<blead>.
=item *
--target I<target>
F<Makefile> target (or equivalent) needed, to run the test case. If specified,
this should be one of
=over 4
=item *
I<none>
Don't build anything - just run the user test case against a clean checkout.
Using this gives a couple of features that a plain C<git bisect run> can't
offer - automatic start revision detection, and test case C<--timeout>.
=item *
I<config.sh>
Just run F<./Configure>
=item *
I<config.h>
Run the various F<*.SH> files to generate F<Makefile>, F<config.h>, I<etc>.
=item *
I<miniperl>
Build F<miniperl>.
=item *
I<lib/Config.pm>
Use F<miniperl> to build F<lib/Config.pm>
=item *
I<Fcntl>
Build F<lib/auto/Fcntl/Fnctl.so> (strictly, C<.$Config{so}>). As L<Fcntl>
is simple XS module present since 5.000, this provides a fast test of
whether XS modules can be built. Note, XS modules are built by F<miniperl>,
hence this target will not build F<perl>.
=item *
I<perl>
Build F<perl>. This also builds pure-Perl modules in F<cpan>, F<dist> and
F<ext>. XS modules (such as L<Fcntl>) are not built.
=item *
I<test_prep>
Build everything needed to run the tests. This is the default if we're
running test code, but is time consuming, as it means building all
XS modules. For older F<Makefile>s, the previous name of C<test-prep>
is automatically substituted. For very old F<Makefile>s, C<make test> is
run, as there is no target provided to just get things ready, and for 5.004
and earlier the tests run very quickly.
=item *
A file ending C<.t>
Build everything needed to run the tests, and then run this test script using
F<t/TEST>. This is actually implemented internally by using the target
I<test_prep>, and setting the test case to "sh", "-c", "cd t && ./TEST ..."
=back
=item *
--one-liner 'code to run'
=item *
-e 'code to run'
Example code to run, just like you'd use with C<perl -e>.
This prepends C<./perl -Ilib -e 'code to run'> to the test case given,
or F<./miniperl> if I<target> is C<miniperl>.
(Usually you'll use C<-e> instead of providing a test case in the
non-option arguments to F<bisect.pl>. You can repeat C<-e> on the command
line, just like you can with C<perl>)
C<-E> intentionally isn't supported, as it's an error in 5.8.0 and earlier,
which interferes with detecting errors in the example code itself.
=item *
-c
Add C<-c> to the command line, to cause perl to exit after syntax checking.
=item *
-l
Add C<-l> to the command line with C<-e>
This will automatically append a newline to every output line of your testcase.
Note that you can't specify an argument to F<perl>'s C<-l> with this, as it's
not feasible to emulate F<perl>'s somewhat quirky switch parsing with
L<Getopt::Long>. If you need the full flexibility of C<-l>, you need to write
a full test case, instead of using C<bisect.pl>'s C<-e> shortcut.
=item *
-w
Add C<-w> to the command line with C<-e>
It's not valid to pass C<-c>, C<-l> or C<-w> to C<bisect.pl> unless you are
also using C<-e>
=item *
--expect-fail
The test case should fail for the I<start> revision, and pass for the I<end>
revision. The bisect run will find the first commit where it passes.
=item *
--crash
Treat any non-crash as success, any crash as failure. (Crashing defined
as exiting with a signal or a core dump.)
=item *
-D I<config_arg=value>
=item *
-U I<config_arg>
=item *
-A I<config_arg=value>
Arguments (C<-A>, C<-D>, C<-U>) to pass to F<Configure>. The C<-D>, C<-A> and
C<-U> switches should be spelled as if you were normally giving them to
F<./Configure>. For example,
-Dnoextensions=Encode
-Uusedevel
-Accflags=-DNO_MATHOMS
Repeated C<-A> arguments are passed
through as is. C<-D> and C<-U> are processed in order, and override
previous settings for the same parameter. F<bisect-runner.pl> emulates
C<-Dnoextensions> when F<Configure> itself does not provide it, as it's
often very useful to be able to disable some XS extensions.
=item *
--make I<make-prog>
The C<make> command to use. If this not set, F<make> is used. If this is
set, it also adds a C<-Dmake=...> else some recursive make invocations
in extensions may fail. Typically one would use this as C<--make gmake>
to use F<gmake> in place of the system F<make>.
=item *
--jobs I<jobs>
=item *
-j I<jobs>
Number of C<make> jobs to run in parallel. A value of 0 suppresses
parallelism. If F</proc/cpuinfo> exists and can be parsed, or F</sbin/sysctl>
exists and reports C<hw.ncpu>, or F</usr/bin/getconf> exists and reports
C<_NPROCESSORS_ONLN> defaults to 1 + I<number of CPUs>. On HP-UX with the
system make defaults to 0, otherwise defaults to 2.
=item *
--match pattern
=item *
--no-match pattern
Instead of running a test program to determine I<pass> or I<fail>,
C<--match> will pass if the given regex matches, and hence search for the
commit that removes the last matching file. C<--no-match> inverts the test,
to search for the first commit that adds files that match.
The remaining command line arguments are treated as glob patterns for files
to match against. If none are specified, then they default as follows:
=over 4
=item *
If no I<target> is specified, the match is against all files in the
repository (which is fast).
=item *
If a I<target> is specified, that target is built, and the match is against
only the built files.
=back
Treating the command line arguments as glob patterns should not cause
problems, as the perl distribution has never shipped or built files with
names that contain characters which are globbing metacharacters.
Anything which is not a readable file is ignored, instead of generating an
error. (If you want an error, run C<grep> or C<ack> as a test case). This
permits one to easily search in a file that changed its name. For example:
.../Porting/bisect.pl --match 'Pod.*Functions' 'pod/buildtoc*'
C<--no-match ...> is implemented as C<--expect-fail --match ...>
=item *
--valgrind
Run the test program under C<valgrind>. If you need to test for memory
errors when parsing invalid programs, the default parser fail exit code of
255 will always override C<valgrind>, so try putting the test case invalid
code inside a I<string> C<eval>, so that the perl interpreter will exit with 0.
(Be sure to check the output of $@, to avoid missing mistakes such as
unintended C<eval> failures due to incorrect C<@INC>)
Specifically, this option prepends C<valgrind> C<--error-exitcode=124> to
the command line that runs the testcase, to cause valgrind to exit non-zero
if it detects errors, with the assumption that the test program itself
always exits with zero. If you require more flexibility than this, either
specify your C<valgrind> invocation explicitly as part of the test case, or
use a wrapper script to control the command line or massage the exit codes.
In order for the test program to be seen as a perl script to valgrind
(rather than a shell script), the first line must be one of the following
#!./perl
#!./miniperl
=item *
--test-build
Test that the build completes, without running any test case.
By default, if the build for the desired I<target> fails to complete,
F<bisect-runner.pl> reports a I<skip> back to C<git bisect>, the assumption
being that one wants to find a commit which changed state "builds && passes"
to "builds && fails". If instead one is interested in which commit broke the
build (possibly for particular F<Configure> options), use I<--test-build>
to treat a build failure as a failure, not a "skip".
Often this option isn't as useful as it first seems, because I<any> build
failure will be reported to C<git bisect> as a failure, not just the failure
that you're interested in. Generally, to debug a particular problem, it's
more useful to use a I<target> that builds properly at the point of interest,
and then a test case that runs C<make>. For example:
.../Porting/bisect.pl --start=perl-5.000 --end=perl-5.002 \
--expect-fail --force-manifest --target=miniperl make perl
will find the first revision capable of building L<DynaLoader> and then
F<perl>, without becoming confused by revisions where F<miniperl> won't
even link.
=item *
--module module1,module2,...
Install this (or these) module(s), die when it (the last of those)
cannot be updated to the current version.
Misnomer. the argument can be any argument that can be passed to CPAN
shell's install command. B<But>: since we only have the uptodate
command to verify that an install has taken place, we are unable to
determine success for arguments like
MSCHWERN/Test-Simple-1.005000_005.tar.gz.
In so far, it is not such a misnomer.
Note that this and I<--with-module> will both require a C<CPAN::MyConfig>.
If F<$ENV{HOME}/.cpan/CPAN/MyConfig.pm> does not exist, a CPAN shell will
be started up for you so you can configure one. Feel free to let
CPAN pick defaults for you. Enter 'quit' when you are done, and
then everything should be all set. Alternatively, you may
specify a custom C<CPAN::MyConfig> by using I<--cpan-config-dir>.
Also, if you want to bisect a module that needs a display (like
TK) and you don't want random screens appearing and disappearing
on your computer while you're working, you can do something like
this:
In a terminal:
$ while true; do date ; if ! ps auxww | grep -v grep \
| grep -q Xvfb; then Xvfb :121 & fi; echo -n 'sleeping 60 '; \
sleep 60; done
And then:
DISPLAY=":121" .../Porting/bisect.pl --module=TK
(Some display alternatives are vncserver and Xnest.)
=item *
--with-module module1,module2,...
Like I<--module> above, except this simply installs the requested
modules and they can then be used in other tests.
For example:
.../Porting/bisect.pl --with-module=Moose -e 'use Moose; ...'
=item *
--no-module-tests
Use in conjunction with I<--with-module> to install the modules without
running their tests. This can be a big time saver.
For example:
.../Porting/bisect.pl --with-module=Moose --no-module-tests \
-e 'use Moose; ...'
=item *
--test-module
This is like I<--module>, but just runs the module's tests, instead of
installing it.
WARNING: This is a somewhat experimental option, known to work on recent
CPAN shell versions. If you use this option and strange things happen,
please report them.
Usually, you can just use I<--module>, but if you are getting inconsistent
installation failures and you just want to see when the tests started
failing, you might find this option useful.
=item *
--cpan-config-dir /home/blah/custom
If defined, this will cause L<CPAN> to look for F<CPAN/MyConfig.pm> inside of
the specified directory, instead of using the default config of
F<$ENV{HOME}/.cpan/>.
If no default config exists, a L<CPAN> shell will be fired up for you to
configure things. Letting L<CPAN> automatically configure things for you
should work well enough. You probably want to choose I<manual> instead of
I<local::lib> if it asks. When you're finished with configuration, just
type I<q> and hit I<ENTER> and the bisect should continue.
=item *
--force-manifest
By default, a build will "skip" if any files listed in F<MANIFEST> are not
present. Usually this is useful, as it avoids false-failures. However, there
are some long ranges of commits where listed files are missing, which can
cause a bisect to abort because all that remain are skipped revisions.
In these cases, particularly if the test case uses F<miniperl> and no modules,
it may be more useful to force the build to continue, even if files
F<MANIFEST> are missing.
=item *
--force-regen
Run C<make regen_headers> before building F<miniperl>. This may fix a build
that otherwise would skip because the generated headers at that revision
are stale. It's not the default because it conceals this error in the true
state of such revisions.
=item *
--expect-pass [0|1]
C<--expect-pass=0> is equivalent to C<--expect-fail>. I<1> is the default.
=item *
--timeout I<seconds>
Run the testcase with the given timeout. If this is exceeded, kill it (and
by default all its children), and treat it as a failure.
=item *
--setpgrp
Run the testcase in its own process group. Specifically, call C<setpgrp 0, 0>
just before C<exec>-ing the user testcase. The default is not to set the
process group, unless a timeout is used.
=item *
--all-fixups
F<bisect-runner.pl> will minimally patch various files on a platform and
version dependent basis to get the build to complete. Normally it defers
doing this as long as possible - C<.SH> files aren't patched until after
F<Configure> is run, and C<C> and C<XS> code isn't patched until after
F<miniperl> is built. If C<--all-fixups> is specified, all the fixups are
done before running C<Configure>. In rare cases adding this may cause a
bisect to abort, because an inapplicable patch or other fixup is attempted
for a revision which would usually have already I<skip>ped. If this happens,
please report it as a bug, giving the OS and problem revision.
=item *
--early-fixup file
=item *
--late-fixup file
Specify a file containing a patch or other fixup for the source code. The
action to take depends on the first line of the fixup file
=over 4
=item *
C<#!perl>
If the first line starts C<#!perl> then the file is run using C<$^X>
=item *
C<#!/absolute/path>
If a shebang line is present the file is executed using C<system>
=item *
C<I<filename> =~ /I<pattern>/>
=item *
C<I<filename> !~ /I<pattern>/>
If I<filename> does not exist then the fixup file's contents are ignored.
Otherwise, for C<=~>, if it contains a line matching I<pattern>, then the
file is fed to C<patch -p1> on standard input. For C<=~>, the patch is
applied if no lines match the pattern.
As the empty pattern in Perl is a special case (it matches the most recent
successful match) which is not useful here, the treatment of an empty pattern
is special-cased. C<I<filename> =~ //> applies the patch if filename is
present. C<I<filename> !~ //> applies the patch if filename missing. This
makes it easy to unconditionally apply patches to files, and to use a patch
as a way of creating a new file.
=item *
Otherwise, the file is assumed to be a patch, and always applied.
=back
I<early-fixup>s are applied before F<./Configure> is run. I<late-fixup>s are
applied just after F<./Configure> is run.
These options can be specified more than once. I<file> is actually expanded
as a glob pattern. Globs that do not match are errors, as are missing files.
=item *
--no-clean
Tell F<bisect-runner.pl> not to clean up after the build. This allows one
to use F<bisect-runner.pl> to build the current particular perl revision for
interactive testing, or for debugging F<bisect-runner.pl>.
Passing this to F<bisect.pl> will likely cause the bisect to fail badly.
=item *
--validate
Test that all stable (.0) revisions can be built. By default, attempts to
build I<blead>, then tagged stable releases in reverse order down to
I<perl-5.002> (or I<perl5.005> on a case insensitive file system). Stops at
the first failure, without cleaning the checkout. Use I<--start> to specify
the earliest revision to test, I<--end> to specify the most recent. Useful
for validating a new OS/CPU/compiler combination. For example
../perl/Porting/bisect.pl --validate -le 'print "Hello from $]"'
If no testcase is specified, the default is to use F<t/TEST> to run
F<t/base/*.t>
=item *
--check-args
Validate the options and arguments, and exit silently if they are valid.
=item *
--check-shebang
Validate that the test case isn't an executable file with a
C<#!/usr/bin/perl> line (or similar). As F<bisect-runner.pl> does B<not>
automatically prepend C<./perl> to the test case, a I<#!> line specifying an
external F<perl> binary will cause the test case to always run with I<that>
F<perl>, not the F<perl> built by the bisect runner. Likely this is not what
you wanted. If your test case is actually a wrapper script to run other
commands, you should run it with an explicit interpreter, to be clear. For
example, instead of C<../perl/Porting/bisect.pl ~/test/testcase.pl> you'd
run C<../perl/Porting/bisect.pl /usr/bin/perl ~/test/testcase.pl>
=item *
--gold
Revision to use when checking out known-good recent versions of files,
such as F<makedepend.SH>. F<bisect-runner.pl> defaults this to I<blead>,
but F<bisect.pl> will default it to the most recent stable release.
=item *
--usage
=item *
--help
=item *
-?
Display the usage information and exit.
=back
=head1 EXAMPLES
=head2 Code has started to crash under C<miniperl>
=over 4
=item * Problem
Under C<make minitest> (but not under C<make test_harness>), F<t/re/pat.t> was
failing to compile. What was the first commit at which that compilation
failure could be observed?
=item * Solution
Extract code from the test file at the point where C<./miniperl -Ilib -c> was
showing a compilation failure. Use that in bisection with the C<miniperl>
target.
.../Porting/bisect.pl --target=miniperl --start=2ec4590e \
-e 'q|ace| =~ /c(?=.$)/; $#{^CAPTURE} == -1); exit 0;'
=item * Reference
L<GH issue 17293|https://github.com/Perl/perl5/issues/17293>
=back
=head2 Blead breaks CPAN on threaded builds only
=over 4
=item * Problem
Tests in CPAN module XML::Parser's test suite had begun to fail when tested
against blead in threaded builds only.
=item * Solution
Provide F<Configure>-style switch to bisection program. Straightforward use
of the C<--module> switch.
.../Porting/bisect.pl -Duseithreads \
--start=6256cf2c \
--end=f6f85064 \
--module=XML::Parser
=item * Reference
L<GH issue 16918|https://github.com/Perl/perl5/issues/16918>
=back
=head2 Point in time where code started to segfault is unknown
=over 4
=item * Problem
User submitted code sample which when run caused F<perl> to segfault, but did
not claim that this was a recent change.
=item * Solution
Used locally installed production releases of perl (previously created by
F<perlbrew>) to identify the first production release at which the code would
not compile. Used that information to shorten bisection time.
.../perl Porting/bisect.pl \
--start=v5.14.4 \
--end=v5.16.3 \
--crash -- ./perl -Ilib /tmp/gh-17333-map.pl
$ cat gh-17333-map.pl
@N = 1..5;
map { pop @N } @N;
=item * Reference
L<GH issue 17333|https://github.com/Perl/perl5/issues/17333>
=back
=head2 When did perl start failing to build on a certain platform using C<g++> as the C-compiler?
=over 4
=item * Problem
On NetBSD-8.0, C<perl> had never been smoke-tested using C<g++> as the
C-compiler. Once this was done, it became evident that changes in that
version of the operating system's code were incompatible with some C<perl>
source written long before that OS version was ever released!
=item * Solution
Bisection range was first narrowed using existing builds at release tags.
Then, bisection specified the C-compiler via C<Configure>-style switch and
used C<--test-build> to identify the commit which "broke" the build.
.../perl Porting/bisect.pl \
-Dcc=g++ \
--test-build \
--start=v5.21.6 \
--end=v5.21.7
Then, problem was discussed with knowledgeable NetBSD user.
=item * Reference
L<GH issue 17381|https://github.com/Perl/perl5/issues/17381>
=back
=head2 When did a test file start to emit warnings?
=over 4
=item * Problem
When F<dist/Tie-File/t/43_synopsis> was run as part of C<make test>, we
observed warnings not previously seen. At what commit were those warnings
first emitted?
=item * Solution
We know that when this test file was first committed to blead, no warnings
were observed and there was no output to C<STDERR>. So that commit becomes
the value for C<--start>.
Since the test file in question is for a CPAN distribution maintained by core,
we must prepare to run that test by including C<--target=test_prep> in the
bisection invocation. We then run the test file in a way that captures
C<STDERR> in a file. If that file has non-zero size, then we have presumably
captured the newly seen warnings.
export ERR="/tmp/err"
.../perl Porting/bisect.pl \
--start=507614678018ae1abd55a22e9941778c65741ba3 \
--end=d34b46d077dcfc479c36f65b196086abd7941c76 \
--target=test_prep \
-e 'chdir("t");
system(
"./perl harness ../dist/Tie-File/t/43_synopsis.t
2>$ENV{ERR}"
);
-s $ENV{ERR} and die "See $ENV{ERR} for warnings thrown";'
Bisection pointed to a commit where strictures and warnings were first turned
on throughout the F<dist/Tie-File/> directory.
=item * Reference