Add "Do not translate" and "XML Broken" sections to revcheck

scripts/broken.php

@@ -22,6 +22,8 @@
 This tool also cares for directories marked with .xmlfragmentdir, so
 theses files are tested in relaxed semantics for XML Fragments.       */
 
+require_once __DIR__ . '/translation/lib/XmlErrorFilter.php';
+
 ini_set( 'display_errors' , 1 );
 ini_set( 'display_startup_errors' , 1 );
 error_reporting( E_ALL );
@@ -69,32 +71,6 @@ function print_usage_exit( $cmd )
     exit;
 }
 
-function setup( string & $prefix , string & $suffix , string & $extra )
-{
-    // Undefined entities generate TWO different error messages on libxml
-    // - "Entity '?' not defined" (for entity inside elements)
-    // - "Extra content at the end of the document" (entity outside elements)
-
-    $inside = "<x>&ZZZ;</x>";
-    $outside = "<x/>&ZZZ;";
-
-    $doc = new DOMDocument();
-    $doc->recover            = true;
-    $doc->resolveExternals   = false;
-    $doc->substituteEntities = false;
-    libxml_use_internal_errors( true );
-
-    $doc->loadXML( $inside );
-    $message = trim( libxml_get_errors()[0]->message );
-    $message = str_replace( "ZZZ" , "\f" , $message );
-    [ $prefix , $suffix ] = explode( "\f" , $message );
-    libxml_clear_errors();
-
-    $doc->loadXML( $outside );
-    $extra = trim( libxml_get_errors()[0]->message );
-    libxml_clear_errors();
-}
-
 function testFile( string $filename , bool $checkDnt , bool $fragmentDir = false )
 {
     $contents = file_get_contents( $filename );
@@ -125,13 +101,8 @@ function testFile( string $filename , bool $checkDnt , bool $fragmentDir = false
         echo "  Issue: Manual build may fail.\n";
         echo "  Path:  $filename\n";
         echo "\n";
-        autofix_dos2unix( $filename );
     }
 
-    static $prefix = "", $suffix = "", $extra = "";
-    if ( $extra == "" )
-        setup( $prefix , $suffix , $extra );
-
     $doc = new DOMDocument();
     $doc->recover            = true;
     $doc->resolveExternals   = false;
@@ -150,11 +121,11 @@ function testFile( string $filename , bool $checkDnt , bool $fragmentDir = false
         $message = trim( $error->message );
         $hintFragDir = false;
 
-        if ( str_starts_with( $message , $prefix ) && str_ends_with( $message , $suffix ) )
+        if ( XmlErrorFilter::isUndefinedEntity( $message ) )
             continue;
-        //if ( $message == $extra ) // Disabled as unnecessary. Also, this indicates that some
-        //    continue;             // some entity reference is used at an unusual position.
-        if ( $message == $extra )
+        // Extra content is not skipped, as it indicates that some entity
+        // reference is used at an unusual position.
+        if ( XmlErrorFilter::isExtraContent( $message ) )
             $hintFragDir = true;
 
         $lin = $error->line;

scripts/revcheck.php

@@ -63,6 +63,8 @@ function print_html_all( RevcheckData $data )
     print_html_notinen( $data );
     print_html_revtag( $data );
     print_html_untranslated( $data );
+    print_html_donottranslate( $data );
+    print_html_xmlbroken( $data );
     print_html_footer();
 }
 
@@ -113,6 +115,8 @@ function print_html_menu( string $href )
 | <a href="#notinen">Not in EN tree</a>
 | <a href="#revtag">Missing or invalid revtag</a>
 | <a href="#untranslated">Untranslated files</a>
+| <a href="#donottranslate">Do not translate</a>
+| <a href="#xmlbroken">Broken XML</a>
 </p><p/>
 HTML;
 }
@@ -178,24 +182,24 @@ function print_html_translators( RevcheckData $data )
 </tr>
 HTML;
 
+    // Files that must not be translated are listed, but kept out of the
+    // totals: counting them would lower the completion rate of every
+    // translation, for files no translation is expected to ever have.
+
     $filesTotal = 0;
-    foreach ( $data->fileSummary as $count )
-        $filesTotal += $count;
+    foreach ( $data->fileSummary as $status => $count )
+        if ( $status != RevcheckStatus::DoNotTranslate->value )
+            $filesTotal += $count;
+
+    $labels = $data->getSummaryLabels();
 
     foreach( RevcheckStatus::cases() as $key )
     {
-        $label = "";
+        $label = $labels[ $key->value ] ?? "";
         $count = $data->fileSummary[ $key->value ];
-        $perc = number_format( $count / $filesTotal * 100 , 2 ) . "%";
-        switch( $key )
-        {
-            case RevcheckStatus::TranslatedOk:  $label = "Up to date files"; break;
-            case RevcheckStatus::TranslatedOld: $label = "Outdated files"; break;
-            case RevcheckStatus::TranslatedWip: $label = "Work in progress"; break;
-            case RevcheckStatus::RevTagProblem: $label = "Revision tag missing/problem"; break;
-            case RevcheckStatus::NotInEnTree:   $label = "Not in EN tree"; break;
-            case RevcheckStatus::Untranslated:  $label = "Available for translation"; break;
-        }
+        $perc = $key == RevcheckStatus::DoNotTranslate || $filesTotal == 0
+              ? "n/a"
+              : number_format( $count / $filesTotal * 100 , 2 ) . "%";
 
         print <<<HTML
 <tr>
@@ -428,6 +432,95 @@ function print_html_untranslated( RevcheckData $data )
     print "</table>\n\n";
 }
 
+function print_html_donottranslate( RevcheckData $data )
+{
+    print_html_menu("donottranslate");
+    if ( $data->fileSummary[ RevcheckStatus::DoNotTranslate->value ] == 0 )
+    {
+        echo "<p>No source file is marked do not translate.</p>\n\n";
+        return;
+    }
+
+    print <<<HTML
+<table class="c">
+ <tr>
+  <th>Files marked do not translate</th>
+  <th>kb</th>
+ </tr>
+HTML;
+
+    $path = null;
+    foreach ( $data->fileDetail as $file )
+    {
+        if ( $file->status != RevcheckStatus::DoNotTranslate )
+            continue;
+
+        if ( $path !== $file->path )
+        {
+            $path = $file->path;
+            $header = $path == '' ? '/' : $path;
+            print " <tr><th colspan='2'>$header</th></tr>";
+        }
+
+        $name = $file->name;
+        $size = round( $file->size / 1024 );
+
+        print <<<HTML
+ <tr class="bggray">
+  <td class="c">$name</td>
+  <td class="c">$size</td>
+ </tr>
+HTML;
+    }
+    print "</table>\n\n";
+}
+
+function print_html_xmlbroken( RevcheckData $data )
+{
+    print_html_menu("xmlbroken");
+    if ( $data->fileSummary[ RevcheckStatus::XmlBroken->value ] == 0 )
+    {
+        echo "<p>Good, all translated files are valid XML.</p>\n\n";
+        return;
+    }
+
+    print <<<HTML
+<table class="c">
+ <tr>
+  <th>Broken XML files</th>
+  <th>Error</th>
+  <th>kb</th>
+ </tr>
+HTML;
+
+    $path = null;
+    foreach ( $data->fileDetail as $file )
+    {
+        if ( $file->status != RevcheckStatus::XmlBroken )
+            continue;
+
+        if ( $path !== $file->path )
+        {
+            $path = $file->path;
+            $header = $path == '' ? '/' : $path;
+            print " <tr><th colspan='3'>$header</th></tr>";
+        }
+
+        $name = $file->name;
+        $size = round( $file->size / 1024 );
+        $error = htmlspecialchars( $file->xmlError );
+
+        print <<<HTML
+ <tr class="bgorange">
+  <td class="c">$name</td>
+  <td>$error</td>
+  <td class="c">$size</td>
+ </tr>
+HTML;
+    }
+    print "</table>\n\n";
+}
+
 function print_html_footer()
 {
     print_html_menu("");

scripts/translation/genrevdb.php

@@ -93,18 +93,25 @@ function generate( SQLite3 $db , string $lang )
                 , $file->hashLast
                 , $file->hashDiff
                 , $file->hashRvtg
+                , $file->xmlError
             );
 
+        // Same total as scripts/revcheck.php: files that must not be
+        // translated are listed, but never counted against a translation.
+
         $filesTotal = 0;
-        foreach( $data->fileSummary as $count )
-            $filesTotal += $count;
+        foreach( $data->fileSummary as $status => $count )
+            if ( $status != RevcheckStatus::DoNotTranslate->value )
+                $filesTotal += $count;
         $labels = $data->getSummaryLabels();
         foreach( $data->fileSummary as $status => $count )
             db_insert( $db , "summary", $data->lang
                 , $status
                 , $labels[ $status ]
                 , $count
-                , number_format( $count / $filesTotal * 100 , 2 ) . "%"
+                , $status == RevcheckStatus::DoNotTranslate->value || $filesTotal == 0
+                    ? "n/a"
+                    : number_format( $count / $filesTotal * 100 , 2 ) . "%"
             );
 
         $db->exec( 'COMMIT TRANSACTION' );
@@ -204,6 +211,7 @@ function db_create( $path ) : SQLite3
     hashLast TEXT,
     hashDiff TEXT,
     hashRvtg TEXT,
+    xmlError TEXT,
     UNIQUE ( lang , path , name ) );
 SQL;
 

scripts/translation/lib/RevcheckData.php

@@ -25,6 +25,8 @@ enum RevcheckStatus : string
     case RevTagProblem = 'RevTagProblem';
     case NotInEnTree   = 'NotInEnTree';
     case Untranslated  = 'Untranslated';
+    case DoNotTranslate = 'DoNotTranslate';
+    case XmlBroken      = 'XmlBroken';
 }
 
 class RevcheckData
@@ -68,6 +70,8 @@ public function getSummaryLabels() : array
         $ret[ RevcheckStatus::RevTagProblem->value ] = "Revision tag missing/problem";
         $ret[ RevcheckStatus::NotInEnTree->value   ] = "Not in EN tree";
         $ret[ RevcheckStatus::Untranslated->value  ] = "Available for translation";
+        $ret[ RevcheckStatus::DoNotTranslate->value ] = "Marked do not translate";
+        $ret[ RevcheckStatus::XmlBroken->value      ] = "Broken XML files";
         return $ret;
     }
 }
@@ -100,4 +104,5 @@ class RevcheckDataFile
     public string $hashLast;      // The most recent commit hash, skipped or not
     public string $hashDiff;      // The most recent, non [skip-revcheck] commit hash
     public string $hashRvtg = ""; // Revtag hash, if any
+    public string $xmlError = ""; // First real XML error, if any
 }

scripts/translation/lib/RevcheckFileItem.php

@@ -29,6 +29,7 @@ class RevcheckFileItem
 
     public RevcheckStatus  $status; // target only
     public RevtagInfo|null $revtag; // target only
+    public string $xmlError = "";   // set on target by RevtagParser, copied to source by RevcheckRun
 
     private array $hashList;        // source only
     private bool  $hashStop;        // source only

scripts/translation/lib/RevcheckRun.php

@@ -33,6 +33,8 @@ class RevcheckRun
     public array $filesUntranslated = [];
     public array $filesNotInEn = [];
     public array $filesWip = [];
+    public array $filesDoNotTranslate = [];
+    public array $filesXmlBroken = [];
 
     public array $qaList = [];
     public RevcheckData $revData;
@@ -83,14 +85,33 @@ private function calculateStatus()
             if ( $target == null )
             {
                 if ( RevcheckIgnore::byMark( "{$this->sourceDir}/{$source->file}" ) )
+                {
+                    $source->status = RevcheckStatus::DoNotTranslate;
+                    $this->filesDoNotTranslate[] = $source;
+                    $this->addData( $source , null );
                     continue;
+                }
 
                 $source->status = RevcheckStatus::Untranslated;
                 $this->filesUntranslated[] = $source;
                 $this->addData( $source , null );
                 continue;
             }
 
+            // XmlBroken
+            //
+            // Checked before the revtag, as a broken file makes every other
+            // check on it unreliable, revtag parsing included.
+
+            if ( $target->xmlError != "" )
+            {
+                $source->status = RevcheckStatus::XmlBroken;
+                $source->xmlError = $target->xmlError;
+                $this->filesXmlBroken[] = $source;
+                $this->addData( $source , $target->revtag );
+                continue;
+            }
+
             // RevTagProblem
 
             if ( $target->revtag == null || strlen( $target->revtag->revision ) != 40 )
@@ -168,6 +189,7 @@ private function addData( RevcheckFileItem $info , RevtagInfo|null $revtag = nul
         $file->status = $info->status;
         $file->hashLast = $info->hashLast;
         $file->hashDiff = $info->hashDiff;
+        $file->xmlError = $info->xmlError;
 
         $this->revData->addFile( $info->file , $file );
 

scripts/translation/lib/RevtagParser.php

@@ -33,7 +33,22 @@ class RevtagParser
     static function parseDir( string $lang , RevcheckFileList $list )
     {
         foreach( $list->iterator() as $entry )
+        {
             $entry->revtag = RevtagParser::parseFile( $lang . '/' . $entry->file );
+
+            // Files are parsed here anyway, so reuse the errors already
+            // collected by XmlUtil, instead of loading everything again.
+            //
+            // Only .xml files are checked. Entity files are DTD fragments,
+            // never standalone XML, so they always fail to parse as such.
+
+            if ( str_ends_with( $entry->file , '.xml' ) == false )
+                continue;
+
+            $error = XmlUtil::$lastErrors[0] ?? null;
+            if ( $error != null )
+                $entry->xmlError = trim( $error->message ) . " [{$error->line},{$error->column}]";
+        }
     }
 
     public static function parseFile( string $filename ): RevtagInfo|null