filter-webkit-build does not catch build preparation, preferences, in…

…spector protocol bindings and settings generators

https://bugs.webkit.org/show_bug.cgi?id=245412
rdar://problem/100159026

Reviewed by Alexey Proskuryakov.

Also adds a debug mode so debugging the script is easier.

* Tools/Scripts/filter-build-webkit:
(usageAndExit):
(main):
(printLine):
(setOutputFormatOption):
(shouldIgnoreLine):
* Tools/Scripts/webkitperl/filter-build-webkit_unittest/shouldIgnoreLine_unittests.pl:

Canonical link: https://commits.webkit.org/254719@main

Tools/Scripts/filter-build-webkit

@@ -23,6 +23,19 @@
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 # Filters the output of build-webkit into a more human-readable format.
+
+# To develop:
+#   make debug VERBOSITY=noisy >build-output.txt 
+#   cat build-output.txt | Tools/Scripts/filter-build-webkit --format=debug > formatted.txt
+# Remove an spurious alert (red line):
+# Inspect formatted.txt, locate ALERT: lines that are not real alerts
+#  A) If it is a real build task, add a matcher and printer in main function.
+#  B) If it is a spurious line
+#    - Locate the original sequence from build-output.txt
+#    - Add the sequence as a test in shouldIgnoreLine_unittests.pl
+#    - Add a ignore in the shouldIgnoreLine()
+#    - perl -I Tools/Scripts/ Tools/Scripts/webkitperl/filter-build-webkit_unittest/shouldIgnoreLine_unittests.pl
+
 package FilterBuildWebKit;
 
 BEGIN {
@@ -108,6 +121,7 @@ Output Options:
                   text: Plain text
                   color: Plain text with colors
                   oneline: Plain text with colors on one line
+                  debug: Prefix lines with message class 
 Unfiltered Logging Options:
   -l|--log      Save unfiltered output to file (see --log-file)
   --logfile     Path to save unfiltered output (implies --log, default: $unfilteredOutputPath)
@@ -154,7 +168,7 @@ sub main() {
             printLine($line, STYLE_HEADER);
         } elsif ($line =~ /^note: [Uu]sing/) {
             printLine($line, STYLE_HEADER);
-        } elsif ($line =~ /Checking Dependencies|Check dependencies|Create product structure|Write auxiliary files|LinkStoryboards/) {
+        } elsif ($line =~ /Prepare packages|Computing target dependency graph and provisioning inputs|Create build description|Checking Dependencies|Check dependencies|Create product structure|Write auxiliary files|LinkStoryboards/) {
             printLine($line, STYLE_PLAIN);
         } elsif ($line =~ /\*\* BUILD SUCCEEDED \*\*/) {
             printLine("Build Succeeded", STYLE_SUCCESS);
@@ -209,16 +223,21 @@ sub main() {
             printLine("Generating bytecode list", STYLE_PLAIN);
         } elsif ($line =~ /^ruby JavaScriptCore\/b3\/air\/opcode_generator.rb JavaScriptCore\/b3\/air\/AirOpcode.opcodes$/) {
             printLine("Generating opcodes", STYLE_PLAIN);
-        } elsif ($line =~ /^ruby WebCore\/Scripts\/GenerateSettings.rb --input .*/) {
-            printLine("Generating settings", STYLE_PLAIN);
+        } elsif ($line =~ /^.*ruby WebCore\/Scripts\/GenerateSettings.rb\s+.*\s+--template\s+.*(\/\S+).erb/) {
+            my $path = basename($1);
+            printLine("Generating settings $path", STYLE_PLAIN);
         } elsif ($line =~ /^ruby "?WebCore\/domjit\/generate-abstract-heap.rb"? (\S+) (\S+)/) {
             printLine("Generating abstract heap", STYLE_PLAIN);
+        } elsif ($line =~ /^.*\/generate-inspector-protocol-bindings.py\s+--framework\s+(\S+).*(\/\S+).json/) {
+            my $path = basename($2);
+            printLine("Generating inspector protocol bindings $1 $path", STYLE_PLAIN);
         } elsif ($line =~ /^bash -c "perl JavaScriptCorePrivateHeaders\/xxd.pl .* \<\(gzip -cn .*\) .*"/) {
             printLine("Converting WHLSLStandardLibrary", STYLE_PLAIN);
         } elsif ($line =~ /^sh .*\/generate-https-upgrade-database\.sh .*\/HTTPSUpgradeList.txt HTTPSUpgradeList.db/) {
             printLine("Converting HTTPSUpgradeList", STYLE_PLAIN);
-        } elsif ($line =~ /^.*\/GeneratePreferences.rb --input .*\.yaml/) {
-            printLine("Generating preferences", STYLE_PLAIN);
+        } elsif ($line =~ /^ruby .*\/GeneratePreferences\.rb\s+--frontend\s+(\S+)\s+.*\s+--template\s+.*(\/\S+).erb/) {
+            my $path = basename($2);
+            printLine("Generating preferences $1 $path", STYLE_PLAIN);
         } elsif ($line =~ /^### (Generating \.xcfilelists for .*)$/) {
             printLine("$1", STYLE_PLAIN);
         } elsif ($line =~ /^(Pre-processing InspectorBackendCommands\.\.\.)$/) {
@@ -281,6 +300,10 @@ sub printLine($$)
                 $endl = "\r";
             }
         }
+    } elsif ($outputFormat eq "debug") {
+        if ($style == STYLE_HEADER)    { $color = "HEADER:"; }
+        elsif ($style == STYLE_SUCCESS) { $color = "SUCCESS:"; }
+        elsif ($style == STYLE_ALERT)   { $color = "ALERT:"; }
     }
     print OUTPUT_HANDLE $erase, $color, $line, $reset, $endl;
     if ($outputFormat eq "oneline") {
@@ -299,8 +322,8 @@ sub setOutputFormatOption($$)
 {
     my ($opt, $value) = @_;
     $value = lc($value);
-    if ($value ne "html" && $value ne "text" && $value ne "color" && $value ne "oneline") {
-        die "The $opt option must be either \"html\", \"text\", \”color\" or \”oneline\"";
+    if ($value ne "html" && $value ne "text" && $value ne "color" && $value ne "oneline" && $value ne "debug") {
+        die "The $opt option must be either \"html\", \"text\", \”color\", \”oneline\" or \"debug\"";
     }
     $outputFormat = $value;
 }
@@ -412,6 +435,10 @@ sub shouldIgnoreLine($$)
     return 1 if $line =~ /^.* Run script build phase '/;
     return 1 if $line =~ /^.* Traditional headermap style is no longer supported;/;
     return 1 if $line =~ /^.* Skipping duplicate build file in/;
+    return 1 if $line =~ /^Build description signature:/;
+    return 1 if $line =~ /^Build description path:/;
+    return 1 if $line =~ /^note: Building targets in dependency order/;
+    return 1 if $line =~ /^warning: .*libtool: archive library: .* the table of contents is empty \(no object file members in the library define global symbols\)/;
     # Investigate these in https://bugs.webkit.org/show_bug.cgi?id=245263.
     return 1 if $line =~ /warning: Could not read serialized diagnostics file: error\(\"Failed to open diagnostics file\"\)/;
     # Remove two below once https://bugs.webkit.org/show_bug.cgi?id=175997 is fixed.

Tools/Scripts/webkitperl/filter-build-webkit_unittest/shouldIgnoreLine_unittests.pl

@@ -240,6 +240,58 @@ END
     }
 }
 
+my @buildDescriptionLines = split(/$INPUT_RECORD_SEPARATOR/, <<'END');
+Create build description
+Build description signature: 6df54043eda5ce9a5bc62735755fc500
+Build description path: /Users/u/Build/XCBuildData/6df54043eda5ce9a5bc62735755fc500-desc.xcbuild
+note: Building targets in dependency order
+Create build description
+END
+for my $i (0..scalar(@buildDescriptionLines) - 1) {
+    my $previousLine = $i ? $dsymNoObjectFileLines[$i - 1] : "";
+    my $line = $buildDescriptionLines[$i];
+    if ($line =~ /Create build description/) {
+        is(shouldIgnoreLine($previousLine, $line), 0, description("Printed: " . $line));
+    } else {
+        is(shouldIgnoreLine($previousLine, $line), 1, description("Ignored: " . $line));
+    }
+}
+my @libtoolEmptyTOC = split(/$INPUT_RECORD_SEPARATOR/, <<'END');
+Libtool libWebKitAdditions
+warning: /Volumes/Somevolume/Xcode.app/Contents/Developer/Toolchains/OSX13.0.xctoolchain/usr/bin/libtool: archive library: /Users/u/Build/Debug/libWebKitAdditions.a the table of contents is empty (no object file members in the library define global symbols)
+Libtool endtest
+END
+for my $i (0..scalar(@libtoolEmptyTOC) - 1) {
+    my $previousLine = $i ? $libtoolEmptyTOC[$i - 1] : "";
+    my $line = $libtoolEmptyTOC[$i];
+    if ($line =~ /Libtool/) {
+        is(shouldIgnoreLine($previousLine, $line), 0, description("Printed: " . $line));
+    } else {
+        is(shouldIgnoreLine($previousLine, $line), 1, description("Ignored: " . $line));
+    }
+}
+
+my @generatePrefsLines = split(/$INPUT_RECORD_SEPARATOR/, <<'END');
+PhaseScriptExecution Run\ Script /Users/u/Build/DumpRenderTree.build/Debug/Derived\ Sources.build/Script-0F18E7011D6B9CC60027E547.sh (in target 'Derived Sources' from project 'DumpRenderTree')
+    cd /Users/u/WebKit/OpenSource/Tools/DumpRenderTree
+    export ACTION\=build
+    /bin/sh -c /Users/u/Build/DumpRenderTree.build/Debug/Derived\\\ Sources.build/Script-0F18E7011D6B9CC60027E547.sh
+Generating bindings for UIScriptController...
+ruby /Users/u/Build/Debug/usr/local/include/wtf/Scripts/GeneratePreferences.rb --frontend WebKitLegacy --base /Users/u/Build/Debug/usr/local/include/wtf/Scripts/Preferences/WebPreferences.yaml --debug /Users/u/Build/Debug/usr/local/include/wtf/Scripts/Preferences/WebPreferencesDebug.yaml --experimental /Users/u/Build/Debug/usr/local/include/wtf/Scripts/Preferences/WebPreferencesExperimental.yaml --internal /Users/u/Build/Debug/usr/local/include/wtf/Scripts/Preferences/WebPreferencesInternal.yaml --template /Users/u/WebKit/OpenSource/Tools/DumpRenderTree/Scripts/PreferencesTemplates/TestOptionsGeneratedWebKitLegacyKeyMapping.cpp.erb
+perl -I /Users/u/Build/Debug/WebCore.framework/Versions/A/PrivateHeaders -I /Users/u/WebKit/OpenSource/Tools/DumpRenderTree/../TestRunnerShared/UIScriptContext/Bindings -I /Users/u/WebKit/OpenSource/Tools/DumpRenderTree/Bindings /Users/u/Build/Debug/WebCore.framework/Versions/A/PrivateHeaders/generate-bindings.pl --defines "ENABLE_3D_TRANSFORMS WTF_PLATFORM_COCOA WTF_PLATFORM_MAC" --include /Users/u/WebKit/OpenSource/Tools/DumpRenderTree/../TestRunnerShared/UIScriptContext/Bindings --outputDir . --generator DumpRenderTree --idlAttributesFile /Users/u/Build/Debug/WebCore.framework/Versions/A/PrivateHeaders/IDLAttributes.json /Users/u/WebKit/OpenSource/Tools/DumpRenderTree/../TestRunnerShared/UIScriptContext/Bindings/UIScriptController.idl
+ruby /Users/u/Build/Debug/usr/local/include/wtf/Scripts/GeneratePreferences.rb --frontend WebKitLegacy --base /Users/u/Build/Debug/usr/local/include/wtf/Scripts/Preferences/WebPreferences.yaml --debug /Users/u/Build/Debug/usr/local/include/wtf/Scripts/Preferences/WebPreferencesDebug.yaml --experimental /Users/u/Build/Debug/usr/local/include/wtf/Scripts/Preferences/WebPreferencesExperimental.yaml --internal /Users/u/Build/Debug/usr/local/include/wtf/Scripts/Preferences/WebPreferencesInternal.yaml --template /Users/u/WebKit/OpenSource/Tools/DumpRenderTree/Scripts/PreferencesTemplates/TestOptionsGeneratedKeys.h.erb
+PhaseScriptExecution endtest
+END
+for my $i (0..scalar(@generatePrefsLines) - 1) {
+    my $previousLine = $i ? $generatePrefsLines[$i - 1] : "";
+    my $line = $generatePrefsLines[$i];
+    if ($line =~ /GeneratePreferences.rb|PhaseScriptExecution|Generating bindings/) {
+        is(shouldIgnoreLine($previousLine, $line), 0, description("Printed: " . $line));
+    } else {
+        is(shouldIgnoreLine($previousLine, $line), 1, description("Ignored: " . $line));
+    }
+}
+
 done_testing();
 
 sub description($)